<commit_before>//
// Package jsonp impements JSONp middleware
//
// Some documentation on this in ../../middleware/jsonp/README.md
//
// Copyright (C) Philip Schlump, 2015
// License LICENSE.apache.txt or LICENSE.mit.txt
//

package jsonp

import (
	"net/http"
	"net/url"

	"github.com/mholt/caddy/middleware"
	"github.com/mholt/caddy/middleware/jsonp/bufferhtml"
)

type JsonPHandlerType struct {
	Paths                []string
	Next                 middleware.Handler
	ResponseBodyRecorder *bufferhtml.BufferHTML
}

func (jph JsonPHandlerType) ServeHTTP(www http.ResponseWriter, req *http.Request) (int, error) {
	for _, prefix := range jph.Paths {
		if middleware.Path(req.URL.Path).Matches(prefix) {
			ResponseBodyRecorder := bufferhtml.NewBufferHTML()
			status, err := jph.Next.ServeHTTP(ResponseBodyRecorder, req)
			if status == 200 {
				// if there is a "callback" argument then use that to format the JSONp response.
				Prefix := ""
				Postfix := ""
				u, err := url.ParseRequestURI(req.RequestURI)
				if err != nil {
					ResponseBodyRecorder.FlushAtEnd(www, "/*1*/", "")
					return status, nil
				}
				m, err := url.ParseQuery(u.RawQuery)
				if err != nil {
					ResponseBodyRecorder.FlushAtEnd(www, "/*2*/", "")
					return status, nil
				}
				callback := m.Get("callback")
				if callback != "" {
					www.Header().Set("Content-Type", "application/javascript")
					Prefix = callback + "("
					Postfix = ");"
				}
				ResponseBodyRecorder.FlushAtEnd(www, Prefix, Postfix)
			} else {
				ResponseBodyRecorder.FlushAtEnd(www, "/*3*/", "")
			}
			return status, err
		}
	}
	return jph.Next.ServeHTTP(www, req)
}
<commit_msg>removed returned to client<commit_after>//
// Package jsonp impements JSONp middleware
//
// Some documentation on this in ../../middleware/jsonp/README.md
//
// Copyright (C) Philip Schlump, 2015
// License LICENSE.apache.txt or LICENSE.mit.txt
//

package jsonp

import (
	"net/http"
	"net/url"

	"github.com/mholt/caddy/middleware"
	"github.com/mholt/caddy/middleware/jsonp/bufferhtml"
)

type JsonPHandlerType struct {
	Paths                []string
	Next                 middleware.Handler
	ResponseBodyRecorder *bufferhtml.BufferHTML
}

func (jph JsonPHandlerType) ServeHTTP(www http.ResponseWriter, req *http.Request) (int, error) {
	for _, prefix := range jph.Paths {
		if middleware.Path(req.URL.Path).Matches(prefix) {
			ResponseBodyRecorder := bufferhtml.NewBufferHTML()
			status, err := jph.Next.ServeHTTP(ResponseBodyRecorder, req)
			if status == 200 {
				// if there is a "callback" argument then use that to format the JSONp response.
				Prefix := ""
				Postfix := ""
				u, err := url.ParseRequestURI(req.RequestURI)
				if err != nil {
					ResponseBodyRecorder.FlushAtEnd(www, "", "")
					return status, nil
				}
				m, err := url.ParseQuery(u.RawQuery)
				if err != nil {
					ResponseBodyRecorder.FlushAtEnd(www, "", "")
					return status, nil
				}
				callback := m.Get("callback")
				if callback != "" {
					www.Header().Set("Content-Type", "application/javascript")
					Prefix = callback + "("
					Postfix = ");"
				}
				ResponseBodyRecorder.FlushAtEnd(www, Prefix, Postfix)
			} else {
				ResponseBodyRecorder.FlushAtEnd(www, "", "")
			}
			return status, err
		}
	}
	return jph.Next.ServeHTTP(www, req)
}
<|endoftext|>


<commit_before>package gist7480523

import (
	"fmt"
	"go/build"
	"os"
	"path/filepath"
	"strings"

	"github.com/shurcooL/go/exp/12"
	"golang.org/x/tools/go/vcs"

	. "github.com/shurcooL/go/gists/gist5504644"
	. "github.com/shurcooL/go/gists/gist7802150"
)

type GoPackageStringer func(*GoPackage) string

// A GoPackage describes a single package found in a directory.
// This is partially a copy of "cmd/go".Package, except it can be imported and reused. =.=
// https://code.google.com/p/go/source/browse/src/cmd/go/pkg.go?name=release#24
type GoPackage struct {
	Bpkg    *build.Package
	BpkgErr error

	Dir *exp12.Directory
}

func GoPackageFromImportPathFound(importPathFound ImportPathFound) *GoPackage {
	bpkg, err := BuildPackageFromSrcDir(importPathFound.FullPath())
	return goPackageFromBuildPackage(bpkg, err)
}

func GoPackageFromImportPath(importPath string) *GoPackage {
	bpkg, err := BuildPackageFromImportPath(importPath)
	return goPackageFromBuildPackage(bpkg, err)
}

func GoPackageFromPath(path, srcDir string) (*GoPackage, error) {
	bpkg, err := BuildPackageFromPath(path, srcDir)
	if err != nil {
		if _, noGo := err.(*build.NoGoError); noGo || bpkg.Dir == "" {
			return nil, err
		}
	}
	return goPackageFromBuildPackage(bpkg, err), nil
}

func goPackageFromBuildPackage(bpkg *build.Package, bpkgErr error) *GoPackage {
	if bpkgErr != nil {
		if _, noGo := bpkgErr.(*build.NoGoError); noGo || bpkg.Dir == "" {
			return nil
		}
	}

	if bpkg.ConflictDir != "" {
		fmt.Fprintf(os.Stderr, "warning: ConflictDir=%q (Dir=%q)\n", bpkg.ConflictDir, bpkg.Dir)
		return nil
	}

	goPackage := &GoPackage{
		Bpkg:    bpkg,
		BpkgErr: bpkgErr,

		Dir: exp12.LookupDirectory(bpkg.Dir),
	}

	/*if goPackage.Bpkg.Goroot == false { // Optimization that assume packages under Goroot are not under vcs
		// TODO: markAsNotNeedToUpdate() because of external insight?
	}*/

	return goPackage
}

// This is okay to call concurrently (a mutex is used internally).
// Actually, not completely okay because MakeUpdated technology is not thread-safe.
func (this *GoPackage) UpdateVcs() {
	if this.Bpkg.Goroot == false { // Optimization that assume packages under Goroot are not under vcs
		MakeUpdated(this.Dir)
	}
}

func (this *GoPackage) UpdateVcsFields() {
	if this.Dir.Repo == nil {
		return
	}

	MakeUpdated(this.Dir.Repo.VcsLocal)
	MakeUpdated(this.Dir.Repo.VcsRemote)

	repoImportPath := GetRepoImportPath(this.Dir.Repo.Vcs.RootPath(), this.Bpkg.SrcRoot)
	if repoRoot, err := vcs.RepoRootForImportPath(repoImportPath, false); err == nil {
		this.Dir.Repo.RepoRoot = repoRoot
	}
}

func GetRepoImportPath(repoPath, srcRoot string) string {
	if s, err := filepath.EvalSymlinks(repoPath); err == nil {
		repoPath = s
	} else {
		fmt.Fprintln(os.Stderr, "warning: GetRepoImportPath: can't resolve symlink")
	}
	if s, err := filepath.EvalSymlinks(srcRoot); err == nil {
		srcRoot = s
	} else {
		fmt.Fprintln(os.Stderr, "warning: GetRepoImportPath: can't resolve symlink")
	}

	// Detect and handle case mismatch in prefix.
	if prefixLen := len(srcRoot + "/"); len(repoPath) >= prefixLen && srcRoot+"/" != repoPath[:prefixLen] && strings.EqualFold(srcRoot+"/", repoPath[:prefixLen]) {
		fmt.Fprintln(os.Stderr, "warning: GetRepoImportPath: prefix case doesn't match")
		return repoPath[prefixLen:]
	}

	return strings.TrimPrefix(repoPath, srcRoot+"/")
}
func GetRepoImportPathPattern(repoPath, srcRoot string) string {
	return GetRepoImportPath(repoPath, srcRoot) + "/..."
}

func (this *GoPackage) String() string {
	return this.Bpkg.ImportPath
}

// =====

// GoPackageRepo represents a collection of Go packages contained by one VCS repository.
type GoPackageRepo struct {
	rootPath   string
	goPackages []*GoPackage
}

func NewGoPackageRepo(rootPath string, goPackages []*GoPackage) GoPackageRepo {
	return GoPackageRepo{rootPath, goPackages}
}

// RepoImportPath returns what would be the import path of the root folder of the repository. It may or may not
// be an actual Go package. E.g.,
//
//	"github.com/owner/repo"
func (repo GoPackageRepo) RepoImportPath() string {
	return GetRepoImportPath(repo.rootPath, repo.goPackages[0].Bpkg.SrcRoot)
}

// ImportPathPattern returns an import path pattern that matches all of the Go packages in this repo.
// E.g.,
//
//	"github.com/owner/repo/..."
func (repo GoPackageRepo) ImportPathPattern() string {
	return GetRepoImportPathPattern(repo.rootPath, repo.goPackages[0].Bpkg.SrcRoot)
}

// RootPath returns the path to the root workdir folder of the repository.
func (repo GoPackageRepo) RootPath() string         { return repo.rootPath }
func (repo GoPackageRepo) GoPackages() []*GoPackage { return repo.goPackages }

// ImportPaths returns a newline separated list of all import paths.
func (repo GoPackageRepo) ImportPaths() string {
	var importPaths []string
	for _, goPackage := range repo.goPackages {
		importPaths = append(importPaths, goPackage.Bpkg.ImportPath)
	}
	return strings.Join(importPaths, "\n")
}
<commit_msg>GetRepoImportPath: Include error details in warning.<commit_after>package gist7480523

import (
	"fmt"
	"go/build"
	"os"
	"path/filepath"
	"strings"

	"github.com/shurcooL/go/exp/12"
	"golang.org/x/tools/go/vcs"

	. "github.com/shurcooL/go/gists/gist5504644"
	. "github.com/shurcooL/go/gists/gist7802150"
)

type GoPackageStringer func(*GoPackage) string

// A GoPackage describes a single package found in a directory.
// This is partially a copy of "cmd/go".Package, except it can be imported and reused. =.=
// https://code.google.com/p/go/source/browse/src/cmd/go/pkg.go?name=release#24
type GoPackage struct {
	Bpkg    *build.Package
	BpkgErr error

	Dir *exp12.Directory
}

func GoPackageFromImportPathFound(importPathFound ImportPathFound) *GoPackage {
	bpkg, err := BuildPackageFromSrcDir(importPathFound.FullPath())
	return goPackageFromBuildPackage(bpkg, err)
}

func GoPackageFromImportPath(importPath string) *GoPackage {
	bpkg, err := BuildPackageFromImportPath(importPath)
	return goPackageFromBuildPackage(bpkg, err)
}

func GoPackageFromPath(path, srcDir string) (*GoPackage, error) {
	bpkg, err := BuildPackageFromPath(path, srcDir)
	if err != nil {
		if _, noGo := err.(*build.NoGoError); noGo || bpkg.Dir == "" {
			return nil, err
		}
	}
	return goPackageFromBuildPackage(bpkg, err), nil
}

func goPackageFromBuildPackage(bpkg *build.Package, bpkgErr error) *GoPackage {
	if bpkgErr != nil {
		if _, noGo := bpkgErr.(*build.NoGoError); noGo || bpkg.Dir == "" {
			return nil
		}
	}

	if bpkg.ConflictDir != "" {
		fmt.Fprintf(os.Stderr, "warning: ConflictDir=%q (Dir=%q)\n", bpkg.ConflictDir, bpkg.Dir)
		return nil
	}

	goPackage := &GoPackage{
		Bpkg:    bpkg,
		BpkgErr: bpkgErr,

		Dir: exp12.LookupDirectory(bpkg.Dir),
	}

	/*if goPackage.Bpkg.Goroot == false { // Optimization that assume packages under Goroot are not under vcs
		// TODO: markAsNotNeedToUpdate() because of external insight?
	}*/

	return goPackage
}

// This is okay to call concurrently (a mutex is used internally).
// Actually, not completely okay because MakeUpdated technology is not thread-safe.
func (this *GoPackage) UpdateVcs() {
	if this.Bpkg.Goroot == false { // Optimization that assume packages under Goroot are not under vcs
		MakeUpdated(this.Dir)
	}
}

func (this *GoPackage) UpdateVcsFields() {
	if this.Dir.Repo == nil {
		return
	}

	MakeUpdated(this.Dir.Repo.VcsLocal)
	MakeUpdated(this.Dir.Repo.VcsRemote)

	repoImportPath := GetRepoImportPath(this.Dir.Repo.Vcs.RootPath(), this.Bpkg.SrcRoot)
	if repoRoot, err := vcs.RepoRootForImportPath(repoImportPath, false); err == nil {
		this.Dir.Repo.RepoRoot = repoRoot
	}
}

func GetRepoImportPath(repoPath, srcRoot string) string {
	if s, err := filepath.EvalSymlinks(repoPath); err == nil {
		repoPath = s
	} else {
		fmt.Fprintln(os.Stderr, "warning: GetRepoImportPath: can't resolve symlink:", err)
	}
	if s, err := filepath.EvalSymlinks(srcRoot); err == nil {
		srcRoot = s
	} else {
		fmt.Fprintln(os.Stderr, "warning: GetRepoImportPath: can't resolve symlink:", err)
	}

	// Detect and handle case mismatch in prefix.
	if prefixLen := len(srcRoot + "/"); len(repoPath) >= prefixLen && srcRoot+"/" != repoPath[:prefixLen] && strings.EqualFold(srcRoot+"/", repoPath[:prefixLen]) {
		fmt.Fprintln(os.Stderr, "warning: GetRepoImportPath: prefix case doesn't match")
		return repoPath[prefixLen:]
	}

	return strings.TrimPrefix(repoPath, srcRoot+"/")
}
func GetRepoImportPathPattern(repoPath, srcRoot string) string {
	return GetRepoImportPath(repoPath, srcRoot) + "/..."
}

func (this *GoPackage) String() string {
	return this.Bpkg.ImportPath
}

// =====

// GoPackageRepo represents a collection of Go packages contained by one VCS repository.
type GoPackageRepo struct {
	rootPath   string
	goPackages []*GoPackage
}

func NewGoPackageRepo(rootPath string, goPackages []*GoPackage) GoPackageRepo {
	return GoPackageRepo{rootPath, goPackages}
}

// RepoImportPath returns what would be the import path of the root folder of the repository. It may or may not
// be an actual Go package. E.g.,
//
//	"github.com/owner/repo"
func (repo GoPackageRepo) RepoImportPath() string {
	return GetRepoImportPath(repo.rootPath, repo.goPackages[0].Bpkg.SrcRoot)
}

// ImportPathPattern returns an import path pattern that matches all of the Go packages in this repo.
// E.g.,
//
//	"github.com/owner/repo/..."
func (repo GoPackageRepo) ImportPathPattern() string {
	return GetRepoImportPathPattern(repo.rootPath, repo.goPackages[0].Bpkg.SrcRoot)
}

// RootPath returns the path to the root workdir folder of the repository.
func (repo GoPackageRepo) RootPath() string         { return repo.rootPath }
func (repo GoPackageRepo) GoPackages() []*GoPackage { return repo.goPackages }

// ImportPaths returns a newline separated list of all import paths.
func (repo GoPackageRepo) ImportPaths() string {
	var importPaths []string
	for _, goPackage := range repo.goPackages {
		importPaths = append(importPaths, goPackage.Bpkg.ImportPath)
	}
	return strings.Join(importPaths, "\n")
}
<|endoftext|>


<commit_before>package agent

import (
	"crypto/md5"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path"
	"strings"
	"syscall"
	"time"

	"github.com/tutumcloud/tutum-agent/utils"
)

type DockerMgmtDef struct {
	Version             string `json:"version"`
	Download_url        string `json: "download_url"`
	Checksum_md5_url    string `json: "checksum_md5_url"`
	Checksum_sha256_url string `json: "checksum_sha256_url"`
}

func StartDocker(dockerBinPath, keyFilePath, certFilePath, caFilePath string) {
	var command *exec.Cmd

	command = exec.Command(dockerBinPath, "-d",
		"-H", Conf.DockerHost,
		"-H", DockerDefaultHost,
		"--tlscert", certFilePath,
		"--tlskey", keyFilePath,
		"--tlscacert", caFilePath,
		"--tls")

	go func(cmd *exec.Cmd) {
		//open file to log docker logs
		dockerLog := path.Join(LogDir, DockerLogFileName)
		Logger.Println("Set docker log to", dockerLog)
		f, err := os.OpenFile(dockerLog, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			Logger.Println(err)
			Logger.Println("Cannot set docker log to", dockerLog)
		} else {
			defer f.Close()
			command.Stdout = f
			command.Stderr = f
		}

		Logger.Println("Starting docker daemon:", command.Args)
		if err := cmd.Start(); err != nil {
			Logger.Println("Cannot start docker daemon:", err)
		}
		DockerProcess = cmd.Process
		if err := cmd.Wait(); err != nil {
			Logger.Println("Docker daemon is terminated:", err)
			DockerProcess = nil
		}
	}(command)

}

func StopDocker() {
	if DockerProcess != nil {
		DockerProcess.Signal(syscall.SIGTERM)
		for {
			if DockerProcess != nil {
				time.Sleep(500 * time.Millisecond)
			} else {
				break
			}
		}
	}
}
func DownloadDocker(url, dockerBinPath string) {
	if utils.FileExist(dockerBinPath) {
		Logger.Printf("Found docker locally(%s), skip downloading\n", dockerBinPath)
	} else {
		Logger.Println("No docker binary found locally. Starting to download docker...")

		Logger.Println("Downloading docker definition from", url)
		def := downloadDockerDef(url)
		Logger.Println("Successfully downloaded docker definition")

		Logger.Println("Downloading docker binary from", def.Download_url)
		binary := downloadDockerBin(def)
		Logger.Println("Successfully downloaded docker binary")

		Logger.Println("Writing docker binary to", dockerBinPath)
		writeDockerFile(binary, dockerBinPath)
	}
}

func UpdateDocker(dockerBinPath, dockerNewBinPath, dockerNewBinSigPath, keyFilePath, certFilePath, caFilePath string) {
	if utils.FileExist(dockerNewBinPath) {
		Logger.Printf("New Docker binary(%s) found\n", dockerNewBinPath)
		if verifyDockerSig(dockerNewBinPath, dockerNewBinSigPath) {
			Logger.Println("Stopping docker daemon")
			StopDocker()
			Logger.Println("Removing old docker binary")
			if err := os.RemoveAll(dockerBinPath); err != nil {
				Logger.Println("Cannot remove old docker binary:", err)
			}
			Logger.Println("Renaming new docker binary")
			if err := os.Rename(dockerNewBinPath, dockerBinPath); err != nil {
				Logger.Println("Cannot rename docker binary:", err)
			}
			Logger.Println("Removing the signature file ", dockerNewBinSigPath)
			if err := os.RemoveAll(dockerNewBinSigPath); err != nil {
				Logger.Println(err.Error())
			}
			StartDocker(dockerBinPath, keyFilePath, certFilePath, caFilePath)
		} else {
			Logger.Println("New docker binary signature cannot be verified. Update is rejected!")
			Logger.Println("Removing the invalid docker binary ", dockerNewBinPath)
			if err := os.RemoveAll(dockerNewBinPath); err != nil {
				Logger.Println(err.Error())
			}
			Logger.Println("Removing the invalid signature file ", dockerNewBinSigPath)
			if err := os.RemoveAll(dockerNewBinSigPath); err != nil {
				Logger.Println(err.Error())
			}
		}
	}
}

func downloadDockerDef(url string) *DockerMgmtDef {
	def, err := getDockerDef(url)
	for i := 1; ; i *= 2 {
		if i > MaxWaitingTime {
			i = 1
		}
		if err != nil || def == nil {
			Logger.Printf("Cannot get docker definition: %s. Retry in %d second\n", err.Error(), i)
			time.Sleep(time.Duration(i) * time.Second)
			def, err = getDockerDef(url)

		} else {
			break
		}
	}
	return def
}

func getDockerDef(url string) (*DockerMgmtDef, error) {
	var def DockerMgmtDef
	body, err := getBodyFromURL(url)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &def); err != nil {
		return nil, err
	}
	if def == (DockerMgmtDef{}) {
		return nil, errors.New("Wrong docker defniniton")
	}
	return &def, nil
}

func downloadDockerBin(def *DockerMgmtDef) []byte {
	b, err := getDockerBin(def)
	for i := 1; ; i *= 2 {
		if i > MaxWaitingTime {
			i = 1
		}
		if err != nil {
			Logger.Printf("Cannot get docker binary: %s. Retry in %d second\n", err.Error(), i)
			time.Sleep(time.Duration(i) * time.Second)
			b, err = getDockerBin(def)

		} else {
			break
		}
	}
	return b
}

func getDockerBin(def *DockerMgmtDef) ([]byte, error) {
	b, err := getBodyFromURL(def.Download_url)
	if err != nil {
		return nil, err
	}

	//validate md5 checksum of the docker binary
	md5hasher := md5.New()
	md5hasher.Write(b)
	md5s := hex.EncodeToString(md5hasher.Sum(nil))
	Logger.Println("Checksum of the downloaded docker binary, md5:", md5s)
	md5b, err := getBodyFromURL(def.Checksum_md5_url)
	if err != nil {
		Logger.Println("Failed to get md5 for the docker binary")
		return nil, err
	} else {
		if !strings.Contains(string(md5b), md5s) {
			return nil, errors.New("Failed to pass md5 checksum test")
		}
	}
	Logger.Println("Docker binary passed md5 checksum check")

	//validate sha256 checksum of the docker binary
	sha256hasher := sha256.New()
	sha256hasher.Write(b)
	sha256s := hex.EncodeToString(sha256hasher.Sum(nil))
	Logger.Println("Checksum of the downloaded docker binary, shar256:", sha256s)
	sha256b, err := getBodyFromURL(def.Checksum_sha256_url)
	if err != nil {
		Logger.Println("Failed to get sha256 for the docker binary")
		return nil, err
	} else {
		if !strings.Contains(string(sha256b), sha256s) {
			return nil, errors.New("Failed to pass sha256 checksum test")
		}
	}
	Logger.Println("Docker binary passed sha256 checksum check")

	return b, nil
}

func writeDockerFile(binary []byte, dockerBinPath string) {
	err := ioutil.WriteFile(dockerBinPath, binary, 0755)
	for i := 1; ; i *= 2 {
		if i > MaxWaitingTime {
			i = 1
		}
		if err != nil {
			Logger.Printf("Failed to save docker binary: %s. Retry in %d second\n", err.Error(), i)
			time.Sleep(time.Duration(i) * time.Second)
			err = ioutil.WriteFile(dockerBinPath, binary, 0755)
		} else {
			break
		}
	}
}

func getBodyFromURL(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode >= 400 {
		return nil, errors.New(resp.Status)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return body, nil
}

func verifyDockerSig(dockerNewBinPath, dockerNewBinSigPath string) bool {
	cmd := exec.Command("gpg", "--verify", dockerNewBinSigPath, dockerNewBinPath)
	UnsetHandler()
	err := cmd.Run()
	SetHandler()
	if err != nil {
		fmt.Println(err.Error())
		return false
	}
	return true
}
<commit_msg>fix tlsverify flag<commit_after>package agent

import (
	"crypto/md5"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path"
	"strings"
	"syscall"
	"time"

	"github.com/tutumcloud/tutum-agent/utils"
)

type DockerMgmtDef struct {
	Version             string `json:"version"`
	Download_url        string `json: "download_url"`
	Checksum_md5_url    string `json: "checksum_md5_url"`
	Checksum_sha256_url string `json: "checksum_sha256_url"`
}

func StartDocker(dockerBinPath, keyFilePath, certFilePath, caFilePath string) {
	var command *exec.Cmd

	command = exec.Command(dockerBinPath, "-d",
		"-H", Conf.DockerHost,
		"-H", DockerDefaultHost,
		"--tlscert", certFilePath,
		"--tlskey", keyFilePath,
		"--tlscacert", caFilePath,
		"--tlsverify")

	go func(cmd *exec.Cmd) {
		//open file to log docker logs
		dockerLog := path.Join(LogDir, DockerLogFileName)
		Logger.Println("Set docker log to", dockerLog)
		f, err := os.OpenFile(dockerLog, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			Logger.Println(err)
			Logger.Println("Cannot set docker log to", dockerLog)
		} else {
			defer f.Close()
			command.Stdout = f
			command.Stderr = f
		}

		Logger.Println("Starting docker daemon:", command.Args)
		if err := cmd.Start(); err != nil {
			Logger.Println("Cannot start docker daemon:", err)
		}
		DockerProcess = cmd.Process
		if err := cmd.Wait(); err != nil {
			Logger.Println("Docker daemon is terminated:", err)
			DockerProcess = nil
		}
	}(command)

}

func StopDocker() {
	if DockerProcess != nil {
		DockerProcess.Signal(syscall.SIGTERM)
		for {
			if DockerProcess != nil {
				time.Sleep(500 * time.Millisecond)
			} else {
				break
			}
		}
	}
}
func DownloadDocker(url, dockerBinPath string) {
	if utils.FileExist(dockerBinPath) {
		Logger.Printf("Found docker locally(%s), skip downloading\n", dockerBinPath)
	} else {
		Logger.Println("No docker binary found locally. Starting to download docker...")

		Logger.Println("Downloading docker definition from", url)
		def := downloadDockerDef(url)
		Logger.Println("Successfully downloaded docker definition")

		Logger.Println("Downloading docker binary from", def.Download_url)
		binary := downloadDockerBin(def)
		Logger.Println("Successfully downloaded docker binary")

		Logger.Println("Writing docker binary to", dockerBinPath)
		writeDockerFile(binary, dockerBinPath)
	}
}

func UpdateDocker(dockerBinPath, dockerNewBinPath, dockerNewBinSigPath, keyFilePath, certFilePath, caFilePath string) {
	if utils.FileExist(dockerNewBinPath) {
		Logger.Printf("New Docker binary(%s) found\n", dockerNewBinPath)
		if verifyDockerSig(dockerNewBinPath, dockerNewBinSigPath) {
			Logger.Println("Stopping docker daemon")
			StopDocker()
			Logger.Println("Removing old docker binary")
			if err := os.RemoveAll(dockerBinPath); err != nil {
				Logger.Println("Cannot remove old docker binary:", err)
			}
			Logger.Println("Renaming new docker binary")
			if err := os.Rename(dockerNewBinPath, dockerBinPath); err != nil {
				Logger.Println("Cannot rename docker binary:", err)
			}
			Logger.Println("Removing the signature file ", dockerNewBinSigPath)
			if err := os.RemoveAll(dockerNewBinSigPath); err != nil {
				Logger.Println(err.Error())
			}
			StartDocker(dockerBinPath, keyFilePath, certFilePath, caFilePath)
		} else {
			Logger.Println("New docker binary signature cannot be verified. Update is rejected!")
			Logger.Println("Removing the invalid docker binary ", dockerNewBinPath)
			if err := os.RemoveAll(dockerNewBinPath); err != nil {
				Logger.Println(err.Error())
			}
			Logger.Println("Removing the invalid signature file ", dockerNewBinSigPath)
			if err := os.RemoveAll(dockerNewBinSigPath); err != nil {
				Logger.Println(err.Error())
			}
		}
	}
}

func downloadDockerDef(url string) *DockerMgmtDef {
	def, err := getDockerDef(url)
	for i := 1; ; i *= 2 {
		if i > MaxWaitingTime {
			i = 1
		}
		if err != nil || def == nil {
			Logger.Printf("Cannot get docker definition: %s. Retry in %d second\n", err.Error(), i)
			time.Sleep(time.Duration(i) * time.Second)
			def, err = getDockerDef(url)

		} else {
			break
		}
	}
	return def
}

func getDockerDef(url string) (*DockerMgmtDef, error) {
	var def DockerMgmtDef
	body, err := getBodyFromURL(url)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &def); err != nil {
		return nil, err
	}
	if def == (DockerMgmtDef{}) {
		return nil, errors.New("Wrong docker defniniton")
	}
	return &def, nil
}

func downloadDockerBin(def *DockerMgmtDef) []byte {
	b, err := getDockerBin(def)
	for i := 1; ; i *= 2 {
		if i > MaxWaitingTime {
			i = 1
		}
		if err != nil {
			Logger.Printf("Cannot get docker binary: %s. Retry in %d second\n", err.Error(), i)
			time.Sleep(time.Duration(i) * time.Second)
			b, err = getDockerBin(def)

		} else {
			break
		}
	}
	return b
}

func getDockerBin(def *DockerMgmtDef) ([]byte, error) {
	b, err := getBodyFromURL(def.Download_url)
	if err != nil {
		return nil, err
	}

	//validate md5 checksum of the docker binary
	md5hasher := md5.New()
	md5hasher.Write(b)
	md5s := hex.EncodeToString(md5hasher.Sum(nil))
	Logger.Println("Checksum of the downloaded docker binary, md5:", md5s)
	md5b, err := getBodyFromURL(def.Checksum_md5_url)
	if err != nil {
		Logger.Println("Failed to get md5 for the docker binary")
		return nil, err
	} else {
		if !strings.Contains(string(md5b), md5s) {
			return nil, errors.New("Failed to pass md5 checksum test")
		}
	}
	Logger.Println("Docker binary passed md5 checksum check")

	//validate sha256 checksum of the docker binary
	sha256hasher := sha256.New()
	sha256hasher.Write(b)
	sha256s := hex.EncodeToString(sha256hasher.Sum(nil))
	Logger.Println("Checksum of the downloaded docker binary, shar256:", sha256s)
	sha256b, err := getBodyFromURL(def.Checksum_sha256_url)
	if err != nil {
		Logger.Println("Failed to get sha256 for the docker binary")
		return nil, err
	} else {
		if !strings.Contains(string(sha256b), sha256s) {
			return nil, errors.New("Failed to pass sha256 checksum test")
		}
	}
	Logger.Println("Docker binary passed sha256 checksum check")

	return b, nil
}

func writeDockerFile(binary []byte, dockerBinPath string) {
	err := ioutil.WriteFile(dockerBinPath, binary, 0755)
	for i := 1; ; i *= 2 {
		if i > MaxWaitingTime {
			i = 1
		}
		if err != nil {
			Logger.Printf("Failed to save docker binary: %s. Retry in %d second\n", err.Error(), i)
			time.Sleep(time.Duration(i) * time.Second)
			err = ioutil.WriteFile(dockerBinPath, binary, 0755)
		} else {
			break
		}
	}
}

func getBodyFromURL(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode >= 400 {
		return nil, errors.New(resp.Status)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return body, nil
}

func verifyDockerSig(dockerNewBinPath, dockerNewBinSigPath string) bool {
	cmd := exec.Command("gpg", "--verify", dockerNewBinSigPath, dockerNewBinPath)
	UnsetHandler()
	err := cmd.Run()
	SetHandler()
	if err != nil {
		fmt.Println(err.Error())
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package agent

import (
	"log"
	"net/http"
	"os"

	"github.com/coreos/go-etcd/etcd"
	"github.com/flynn/rpcplus"
	rpc "github.com/flynn/rpcplus/comborpc"
)

const (
	HeartbeatIntervalSecs = 5
	MissedHearbeatTTL     = 5
)

type ServiceUpdate struct {
	Name    string
	Addr    string
	Online  bool
	Attrs   map[string]string
	Created uint
}

type Args struct {
	Name  string
	Addr  string
	Attrs map[string]string
}

type UpdateStream interface {
	Chan() chan *ServiceUpdate
	Close()
}

type DiscoveryBackend interface {
	Subscribe(name string) (UpdateStream, error)
	Register(name string, addr string, attrs map[string]string) error
	Unregister(name string, addr string) error
	Heartbeat(name string, addr string) error
}

type Agent struct {
	Backend DiscoveryBackend
	Address string
}

func NewServer(addr string, etcdAddrs []string) *Agent {
	return &Agent{
		Backend: &EtcdBackend{Client: etcd.NewClient(etcdAddrs)},
		Address: addr,
	}
}

func ListenAndServe(server *Agent) error {
	rpc.HandleHTTP()
	if err := rpc.Register(server); err != nil {
		return err
	}
	return http.ListenAndServe(server.Address, nil)
}

func expandAddr(addr string) string {
	if addr[0] == 58 {
		return os.Getenv("EXTERNAL_IP") + addr
	}
	return addr
}

func (s *Agent) Subscribe(args *Args, stream rpcplus.Stream) error {
	updates, err := s.Backend.Subscribe(args.Name)
	if err != nil {
		log.Println("Subscribe: error:", err)
		stream.Send <- &ServiceUpdate{} // be sure to unblock client
		return err
	}
	log.Println("Subscribe:", args.Name)
	for update := range updates.Chan() {
		select {
		case stream.Send <- update:
		case <-stream.Error:
			updates.Close()
			return nil
		}
	}
	return nil
}

func (s *Agent) Register(args *Args, ret *string) error {
	addr := expandAddr(args.Addr)
	err := s.Backend.Register(args.Name, addr, args.Attrs)
	if err != nil {
		log.Println("Register: error:", err)
		return err
	}
	ret = &addr
	log.Println("Register:", args.Name, addr, args.Attrs)
	return nil
}

func (s *Agent) Unregister(args *Args, ret *struct{}) error {
	addr := expandAddr(args.Addr)
	err := s.Backend.Unregister(args.Name, addr)
	if err != nil {
		log.Println("Unregister: error:", err)
		return err
	}
	log.Println("Unregister:", args.Name, addr)
	return nil
}

func (s *Agent) Heartbeat(args *Args, ret *struct{}) error {
	addr := expandAddr(args.Addr)
	err := s.Backend.Heartbeat(args.Name, addr)
	if err != nil {
		log.Println("Heartbeat: error:", err)
		return err
	}
	log.Println("Heartbeat:", args.Name, addr)
	return nil
}
<commit_msg>more clear<commit_after>package agent

import (
	"log"
	"net/http"
	"os"

	"github.com/coreos/go-etcd/etcd"
	"github.com/flynn/rpcplus"
	rpc "github.com/flynn/rpcplus/comborpc"
)

const (
	HeartbeatIntervalSecs = 5
	MissedHearbeatTTL     = 5
)

type ServiceUpdate struct {
	Name    string
	Addr    string
	Online  bool
	Attrs   map[string]string
	Created uint
}

type Args struct {
	Name  string
	Addr  string
	Attrs map[string]string
}

type UpdateStream interface {
	Chan() chan *ServiceUpdate
	Close()
}

type DiscoveryBackend interface {
	Subscribe(name string) (UpdateStream, error)
	Register(name string, addr string, attrs map[string]string) error
	Unregister(name string, addr string) error
	Heartbeat(name string, addr string) error
}

type Agent struct {
	Backend DiscoveryBackend
	Address string
}

func NewServer(addr string, etcdAddrs []string) *Agent {
	return &Agent{
		Backend: &EtcdBackend{Client: etcd.NewClient(etcdAddrs)},
		Address: addr,
	}
}

func ListenAndServe(server *Agent) error {
	rpc.HandleHTTP()
	if err := rpc.Register(server); err != nil {
		return err
	}
	return http.ListenAndServe(server.Address, nil)
}

func expandAddr(addr string) string {
	if addr[0] == ':' {
		return os.Getenv("EXTERNAL_IP") + addr
	}
	return addr
}

func (s *Agent) Subscribe(args *Args, stream rpcplus.Stream) error {
	updates, err := s.Backend.Subscribe(args.Name)
	if err != nil {
		log.Println("Subscribe: error:", err)
		stream.Send <- &ServiceUpdate{} // be sure to unblock client
		return err
	}
	log.Println("Subscribe:", args.Name)
	for update := range updates.Chan() {
		select {
		case stream.Send <- update:
		case <-stream.Error:
			updates.Close()
			return nil
		}
	}
	return nil
}

func (s *Agent) Register(args *Args, ret *string) error {
	addr := expandAddr(args.Addr)
	err := s.Backend.Register(args.Name, addr, args.Attrs)
	if err != nil {
		log.Println("Register: error:", err)
		return err
	}
	ret = &addr
	log.Println("Register:", args.Name, addr, args.Attrs)
	return nil
}

func (s *Agent) Unregister(args *Args, ret *struct{}) error {
	addr := expandAddr(args.Addr)
	err := s.Backend.Unregister(args.Name, addr)
	if err != nil {
		log.Println("Unregister: error:", err)
		return err
	}
	log.Println("Unregister:", args.Name, addr)
	return nil
}

func (s *Agent) Heartbeat(args *Args, ret *struct{}) error {
	addr := expandAddr(args.Addr)
	err := s.Backend.Heartbeat(args.Name, addr)
	if err != nil {
		log.Println("Heartbeat: error:", err)
		return err
	}
	log.Println("Heartbeat:", args.Name, addr)
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"github.com/keybase/cli"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
)

func getPlatformSpecificCommands(cl *libcmdline.CommandLine, g *libkb.GlobalContext) []cli.Command {
	return []cli.Command{
		NewCmdFuse(cl, g),
		NewCmdInstall(cl, g),
		NewCmdLaunchd(cl, g),
	}
}
<commit_msg>Fix build directive<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

// +build darwin

package client

import (
	"github.com/keybase/cli"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
)

func getPlatformSpecificCommands(cl *libcmdline.CommandLine, g *libkb.GlobalContext) []cli.Command {
	return []cli.Command{
		NewCmdFuse(cl, g),
		NewCmdInstall(cl, g),
		NewCmdLaunchd(cl, g),
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package srvtopo

import (
	"sort"

	"vitess.io/vitess/go/sqltypes"

	"context"

	"google.golang.org/protobuf/proto"

	"vitess.io/vitess/go/vt/key"
	"vitess.io/vitess/go/vt/topo/topoproto"
	"vitess.io/vitess/go/vt/vterrors"
	"vitess.io/vitess/go/vt/vttablet/queryservice"

	querypb "vitess.io/vitess/go/vt/proto/query"
	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
)

// A Gateway is the query processing module for each shard,
// which is used by ScatterConn.
type Gateway interface {
	// the query service that this Gateway wraps around
	queryservice.QueryService

	// QueryServiceByAlias returns a QueryService
	QueryServiceByAlias(alias *topodatapb.TabletAlias, target *querypb.Target) (queryservice.QueryService, error)
}

// A Resolver can resolve keyspace ids and key ranges into ResolvedShard*
// objects. It uses an underlying srvtopo.Server to find the topology,
// and a TargetStats object to find the healthy destinations.
type Resolver struct {
	// topoServ is the srvtopo.Server to use for topo queries.
	topoServ Server

	// gateway
	gateway Gateway

	// localCell is the local cell for the queries.
	localCell string

	// FIXME(alainjobart) also need a list of remote cells.
	// FIXME(alainjobart) and a policy on how to use them.
	// But for now we only use the local cell.
}

// NewResolver creates a new Resolver.
func NewResolver(topoServ Server, gateway Gateway, localCell string) *Resolver {
	return &Resolver{
		topoServ:  topoServ,
		gateway:   gateway,
		localCell: localCell,
	}
}

// ResolvedShard contains everything we need to send a query to a shard.
type ResolvedShard struct {
	// Target describes the target shard.
	Target *querypb.Target

	// Gateway is the way to execute a query on this shard
	Gateway Gateway
}

// ResolvedShardEqual is an equality check on *ResolvedShard.
func ResolvedShardEqual(rs1, rs2 *ResolvedShard) bool {
	return proto.Equal(rs1.Target, rs2.Target)
}

// ResolvedShardsEqual is an equality check on []*ResolvedShard.
func ResolvedShardsEqual(rss1, rss2 []*ResolvedShard) bool {
	if len(rss1) != len(rss2) {
		return false
	}
	for i, rs1 := range rss1 {
		if !ResolvedShardEqual(rs1, rss2[i]) {
			return false
		}
	}
	return true
}

// GetKeyspaceShards return all the shards in a keyspace. It follows
// redirection if ServedFrom is set. It is only valid for the local cell.
// Do not use it to further resolve shards, instead use the Resolve* methods.
func (r *Resolver) GetKeyspaceShards(ctx context.Context, keyspace string, tabletType topodatapb.TabletType) (string, *topodatapb.SrvKeyspace, []*topodatapb.ShardReference, error) {
	srvKeyspace, err := r.topoServ.GetSrvKeyspace(ctx, r.localCell, keyspace)
	if err != nil {
		return "", nil, nil, vterrors.Errorf(vtrpcpb.Code_UNKNOWN, "keyspace %v fetch error: %v", keyspace, err)
	}

	// check if the keyspace has been redirected for this tabletType.
	for _, sf := range srvKeyspace.ServedFrom {
		if sf.TabletType == tabletType {
			keyspace = sf.Keyspace
			srvKeyspace, err = r.topoServ.GetSrvKeyspace(ctx, r.localCell, keyspace)
			if err != nil {
				return "", nil, nil, vterrors.Errorf(vtrpcpb.Code_UNKNOWN, "keyspace %v fetch error: %v", keyspace, err)
			}
		}
	}

	partition := topoproto.SrvKeyspaceGetPartition(srvKeyspace, tabletType)
	if partition == nil {
		return "", nil, nil, vterrors.Errorf(vtrpcpb.Code_UNKNOWN, "No partition found for tabletType %v in keyspace %v", topoproto.TabletTypeLString(tabletType), keyspace)
	}
	return keyspace, srvKeyspace, partition.ShardReferences, nil
}

// GetAllShards returns the list of ResolvedShards associated with all
// the shards in a keyspace.
// FIXME(alainjobart) callers should convert to ResolveDestination(),
// and GetSrvKeyspace.
func (r *Resolver) GetAllShards(ctx context.Context, keyspace string, tabletType topodatapb.TabletType) ([]*ResolvedShard, *topodatapb.SrvKeyspace, error) {
	keyspace, srvKeyspace, allShards, err := r.GetKeyspaceShards(ctx, keyspace, tabletType)
	if err != nil {
		return nil, nil, err
	}

	res := make([]*ResolvedShard, len(allShards))
	for i, shard := range allShards {
		target := &querypb.Target{
			Keyspace:   keyspace,
			Shard:      shard.Name,
			TabletType: tabletType,
			Cell:       r.localCell,
		}
		// Right now we always set the Cell to ""
		// Later we can fallback to another cell if needed.
		// We would then need to read the SrvKeyspace there too.
		target.Cell = ""
		res[i] = &ResolvedShard{
			Target:  target,
			Gateway: r.gateway,
		}
	}
	return res, srvKeyspace, nil
}

// GetAllKeyspaces returns all the known keyspaces in the local cell.
func (r *Resolver) GetAllKeyspaces(ctx context.Context) ([]string, error) {
	keyspaces, err := r.topoServ.GetSrvKeyspaceNames(ctx, r.localCell, true)
	if err != nil {
		return nil, vterrors.Errorf(vtrpcpb.Code_UNKNOWN, "keyspace names fetch error: %v", err)
	}
	// FIXME(alainjobart) this should be unnecessary. The results
	// of ListDir are sorted, and that's the underlying topo code.
	// But the tests depend on this behavior now.
	sort.Strings(keyspaces)
	return keyspaces, nil
}

// ResolveDestinationsMultiCol resolves values and their destinations into their
// respective shards for multi col vindex.
//
// If ids is nil, the returned [][][]sqltypes.Value is also nil.
// Otherwise, len(ids) has to match len(destinations), and then the returned
// [][][]sqltypes.Value is populated with all the values that go in each shard,
// and len([]*ResolvedShard) matches len([][][]sqltypes.Value).
//
// Sample input / output:
// - destinations: dst1, 			dst2, 		dst3
// - ids:          [id1a,id1b],  [id2a,id2b],  [id3a,id3b]
// If dst1 is in shard1, and dst2 and dst3 are in shard2, the output will be:
// - []*ResolvedShard:   shard1, 			shard2
// - [][][]sqltypes.Value: [[id1a,id1b]],  [[id2a,id2b], [id3a,id3b]]
func (r *Resolver) ResolveDestinationsMultiCol(ctx context.Context, keyspace string, tabletType topodatapb.TabletType, ids [][]sqltypes.Value, destinations []key.Destination) ([]*ResolvedShard, [][][]sqltypes.Value, error) {
	keyspace, _, allShards, err := r.GetKeyspaceShards(ctx, keyspace, tabletType)
	if err != nil {
		return nil, nil, err
	}

	var result []*ResolvedShard
	var values [][][]sqltypes.Value
	resolved := make(map[string]int)
	for i, destination := range destinations {
		if err := destination.Resolve(allShards, func(shard string) error {
			s, ok := resolved[shard]
			if !ok {
				target := &querypb.Target{
					Keyspace:   keyspace,
					Shard:      shard,
					TabletType: tabletType,
					Cell:       r.localCell,
				}
				// Right now we always set the Cell to ""
				// Later we can fallback to another cell if needed.
				// We would then need to read the SrvKeyspace there too.
				target.Cell = ""
				s = len(result)
				result = append(result, &ResolvedShard{
					Target:  target,
					Gateway: r.gateway,
				})
				if ids != nil {
					values = append(values, nil)
				}
				resolved[shard] = s
			}
			if ids != nil {
				values[s] = append(values[s], ids[i])
			}
			return nil
		}); err != nil {
			return nil, nil, err
		}
	}
	return result, values, nil
}

// ResolveDestinations resolves values and their destinations into their
// respective shards.
//
// If ids is nil, the returned [][]*querypb.Value is also nil.
// Otherwise, len(ids) has to match len(destinations), and then the returned
// [][]*querypb.Value is populated with all the values that go in each shard,
// and len([]*ResolvedShard) matches len([][]*querypb.Value).
//
// Sample input / output:
// - destinations: dst1, dst2, dst3
// - ids:          id1,  id2,  id3
// If dst1 is in shard1, and dst2 and dst3 are in shard2, the output will be:
// - []*ResolvedShard:   shard1, shard2
// - [][]*querypb.Value: [id1],  [id2, id3]
func (r *Resolver) ResolveDestinations(ctx context.Context, keyspace string, tabletType topodatapb.TabletType, ids []*querypb.Value, destinations []key.Destination) ([]*ResolvedShard, [][]*querypb.Value, error) {
	keyspace, _, allShards, err := r.GetKeyspaceShards(ctx, keyspace, tabletType)
	if err != nil {
		return nil, nil, err
	}

	var result []*ResolvedShard
	var values [][]*querypb.Value
	resolved := make(map[string]int)
	for i, destination := range destinations {
		if err := destination.Resolve(allShards, func(shard string) error {
			s, ok := resolved[shard]
			if !ok {
				target := &querypb.Target{
					Keyspace:   keyspace,
					Shard:      shard,
					TabletType: tabletType,
					Cell:       r.localCell,
				}
				// Right now we always set the Cell to ""
				// Later we can fallback to another cell if needed.
				// We would then need to read the SrvKeyspace there too.
				target.Cell = ""
				s = len(result)
				result = append(result, &ResolvedShard{
					Target:  target,
					Gateway: r.gateway,
				})
				if ids != nil {
					values = append(values, nil)
				}
				resolved[shard] = s
			}
			if ids != nil {
				values[s] = append(values[s], ids[i])
			}
			return nil
		}); err != nil {
			return nil, nil, err
		}
	}
	return result, values, nil
}

// ResolveDestination is a shortcut to ResolveDestinations with only
// one Destination, and no ids.
func (r *Resolver) ResolveDestination(ctx context.Context, keyspace string, tabletType topodatapb.TabletType, destination key.Destination) ([]*ResolvedShard, error) {
	rss, _, err := r.ResolveDestinations(ctx, keyspace, tabletType, nil, []key.Destination{destination})
	return rss, err
}

// ValuesEqual is a helper method to compare arrays of values.
func ValuesEqual(vss1, vss2 [][]*querypb.Value) bool {
	if len(vss1) != len(vss2) {
		return false
	}
	for i, vs1 := range vss1 {
		if len(vs1) != len(vss2[i]) {
			return false
		}
		for j, v1 := range vs1 {
			if !proto.Equal(v1, vss2[i][j]) {
				return false
			}
		}
	}
	return true
}

// GetGateway returns the used gateway
func (r *Resolver) GetGateway() Gateway {
	return r.gateway
}
<commit_msg>refactor: tried to make a method easier to read<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package srvtopo

import (
	"sort"

	"vitess.io/vitess/go/sqltypes"

	"context"

	"google.golang.org/protobuf/proto"

	"vitess.io/vitess/go/vt/key"
	"vitess.io/vitess/go/vt/topo/topoproto"
	"vitess.io/vitess/go/vt/vterrors"
	"vitess.io/vitess/go/vt/vttablet/queryservice"

	querypb "vitess.io/vitess/go/vt/proto/query"
	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
)

// A Gateway is the query processing module for each shard,
// which is used by ScatterConn.
type Gateway interface {
	// the query service that this Gateway wraps around
	queryservice.QueryService

	// QueryServiceByAlias returns a QueryService
	QueryServiceByAlias(alias *topodatapb.TabletAlias, target *querypb.Target) (queryservice.QueryService, error)
}

// A Resolver can resolve keyspace ids and key ranges into ResolvedShard*
// objects. It uses an underlying srvtopo.Server to find the topology,
// and a TargetStats object to find the healthy destinations.
type Resolver struct {
	// topoServ is the srvtopo.Server to use for topo queries.
	topoServ Server

	// gateway
	gateway Gateway

	// localCell is the local cell for the queries.
	localCell string

	// FIXME(alainjobart) also need a list of remote cells.
	// FIXME(alainjobart) and a policy on how to use them.
	// But for now we only use the local cell.
}

// NewResolver creates a new Resolver.
func NewResolver(topoServ Server, gateway Gateway, localCell string) *Resolver {
	return &Resolver{
		topoServ:  topoServ,
		gateway:   gateway,
		localCell: localCell,
	}
}

// ResolvedShard contains everything we need to send a query to a shard.
type ResolvedShard struct {
	// Target describes the target shard.
	Target *querypb.Target

	// Gateway is the way to execute a query on this shard
	Gateway Gateway
}

// ResolvedShardEqual is an equality check on *ResolvedShard.
func ResolvedShardEqual(rs1, rs2 *ResolvedShard) bool {
	return proto.Equal(rs1.Target, rs2.Target)
}

// ResolvedShardsEqual is an equality check on []*ResolvedShard.
func ResolvedShardsEqual(rss1, rss2 []*ResolvedShard) bool {
	if len(rss1) != len(rss2) {
		return false
	}
	for i, rs1 := range rss1 {
		if !ResolvedShardEqual(rs1, rss2[i]) {
			return false
		}
	}
	return true
}

// GetKeyspaceShards return all the shards in a keyspace. It follows
// redirection if ServedFrom is set. It is only valid for the local cell.
// Do not use it to further resolve shards, instead use the Resolve* methods.
func (r *Resolver) GetKeyspaceShards(ctx context.Context, keyspace string, tabletType topodatapb.TabletType) (string, *topodatapb.SrvKeyspace, []*topodatapb.ShardReference, error) {
	srvKeyspace, err := r.topoServ.GetSrvKeyspace(ctx, r.localCell, keyspace)
	if err != nil {
		return "", nil, nil, vterrors.Errorf(vtrpcpb.Code_UNKNOWN, "keyspace %v fetch error: %v", keyspace, err)
	}

	// check if the keyspace has been redirected for this tabletType.
	for _, sf := range srvKeyspace.ServedFrom {
		if sf.TabletType == tabletType {
			keyspace = sf.Keyspace
			srvKeyspace, err = r.topoServ.GetSrvKeyspace(ctx, r.localCell, keyspace)
			if err != nil {
				return "", nil, nil, vterrors.Errorf(vtrpcpb.Code_UNKNOWN, "keyspace %v fetch error: %v", keyspace, err)
			}
		}
	}

	partition := topoproto.SrvKeyspaceGetPartition(srvKeyspace, tabletType)
	if partition == nil {
		return "", nil, nil, vterrors.Errorf(vtrpcpb.Code_UNKNOWN, "No partition found for tabletType %v in keyspace %v", topoproto.TabletTypeLString(tabletType), keyspace)
	}
	return keyspace, srvKeyspace, partition.ShardReferences, nil
}

// GetAllShards returns the list of ResolvedShards associated with all
// the shards in a keyspace.
// FIXME(alainjobart) callers should convert to ResolveDestination(),
// and GetSrvKeyspace.
func (r *Resolver) GetAllShards(ctx context.Context, keyspace string, tabletType topodatapb.TabletType) ([]*ResolvedShard, *topodatapb.SrvKeyspace, error) {
	keyspace, srvKeyspace, allShards, err := r.GetKeyspaceShards(ctx, keyspace, tabletType)
	if err != nil {
		return nil, nil, err
	}

	res := make([]*ResolvedShard, len(allShards))
	for i, shard := range allShards {
		target := &querypb.Target{
			Keyspace:   keyspace,
			Shard:      shard.Name,
			TabletType: tabletType,
			Cell:       r.localCell,
		}
		// Right now we always set the Cell to ""
		// Later we can fallback to another cell if needed.
		// We would then need to read the SrvKeyspace there too.
		target.Cell = ""
		res[i] = &ResolvedShard{
			Target:  target,
			Gateway: r.gateway,
		}
	}
	return res, srvKeyspace, nil
}

// GetAllKeyspaces returns all the known keyspaces in the local cell.
func (r *Resolver) GetAllKeyspaces(ctx context.Context) ([]string, error) {
	keyspaces, err := r.topoServ.GetSrvKeyspaceNames(ctx, r.localCell, true)
	if err != nil {
		return nil, vterrors.Errorf(vtrpcpb.Code_UNKNOWN, "keyspace names fetch error: %v", err)
	}
	// FIXME(alainjobart) this should be unnecessary. The results
	// of ListDir are sorted, and that's the underlying topo code.
	// But the tests depend on this behavior now.
	sort.Strings(keyspaces)
	return keyspaces, nil
}

// ResolveDestinationsMultiCol resolves values and their destinations into their
// respective shards for multi col vindex.
//
// If ids is nil, the returned [][][]sqltypes.Value is also nil.
// Otherwise, len(ids) has to match len(destinations), and then the returned
// [][][]sqltypes.Value is populated with all the values that go in each shard,
// and len([]*ResolvedShard) matches len([][][]sqltypes.Value).
//
// Sample input / output:
// - destinations: dst1, 			dst2, 		dst3
// - ids:          [id1a,id1b],  [id2a,id2b],  [id3a,id3b]
// If dst1 is in shard1, and dst2 and dst3 are in shard2, the output will be:
// - []*ResolvedShard:   shard1, 			shard2
// - [][][]sqltypes.Value: [[id1a,id1b]],  [[id2a,id2b], [id3a,id3b]]
func (r *Resolver) ResolveDestinationsMultiCol(ctx context.Context, keyspace string, tabletType topodatapb.TabletType, ids [][]sqltypes.Value, destinations []key.Destination) ([]*ResolvedShard, [][][]sqltypes.Value, error) {
	keyspace, _, allShards, err := r.GetKeyspaceShards(ctx, keyspace, tabletType)
	if err != nil {
		return nil, nil, err
	}
	accumulator := &resultAcc{
		resolved:   make(map[string]int),
		resolver:   r,
		ids:        ids,
		keyspace:   keyspace,
		tabletType: tabletType,
	}

	for i, destination := range destinations {
		if err := destination.Resolve(allShards, accumulator.resolveShard(i)); err != nil {
			return nil, nil, err
		}
	}
	return accumulator.shards, accumulator.values, nil
}

type resultAcc struct {
	shards     []*ResolvedShard
	values     [][][]sqltypes.Value
	resolved   map[string]int
	resolver   *Resolver
	ids        [][]sqltypes.Value
	keyspace   string
	tabletType topodatapb.TabletType
}

// resolveShard is called once per shard that is resolved. It will keep track of which shards that are
// the destinations resolved, and the values that are bound to each shard
func (acc *resultAcc) resolveShard(idx int) func(shard string) error {
	return func(shard string) error {
		offsetInValues, ok := acc.resolved[shard]
		if !ok {
			target := &querypb.Target{
				Keyspace:   acc.keyspace,
				Shard:      shard,
				TabletType: acc.tabletType,
			}
			// Right now we always set the Cell to ""
			// Later we can fallback to another cell if needed.
			// We would then need to read the SrvKeyspace there too.
			target.Cell = ""
			offsetInValues = len(acc.shards)
			acc.shards = append(acc.shards, &ResolvedShard{
				Target:  target,
				Gateway: acc.resolver.gateway,
			})
			if acc.ids != nil {
				acc.values = append(acc.values, nil)
			}
			acc.resolved[shard] = offsetInValues
		}
		if acc.ids != nil {
			acc.values[offsetInValues] = append(acc.values[offsetInValues], acc.ids[idx])
		}
		return nil
	}
}

// ResolveDestinations resolves values and their destinations into their
// respective shards.
//
// If ids is nil, the returned [][]*querypb.Value is also nil.
// Otherwise, len(ids) has to match len(destinations), and then the returned
// [][]*querypb.Value is populated with all the values that go in each shard,
// and len([]*ResolvedShard) matches len([][]*querypb.Value).
//
// Sample input / output:
// - destinations: dst1, dst2, dst3
// - ids:          id1,  id2,  id3
// If dst1 is in shard1, and dst2 and dst3 are in shard2, the output will be:
// - []*ResolvedShard:   shard1, shard2
// - [][]*querypb.Value: [id1],  [id2, id3]
func (r *Resolver) ResolveDestinations(ctx context.Context, keyspace string, tabletType topodatapb.TabletType, ids []*querypb.Value, destinations []key.Destination) ([]*ResolvedShard, [][]*querypb.Value, error) {
	keyspace, _, allShards, err := r.GetKeyspaceShards(ctx, keyspace, tabletType)
	if err != nil {
		return nil, nil, err
	}

	var result []*ResolvedShard
	var values [][]*querypb.Value
	resolved := make(map[string]int)
	for i, destination := range destinations {
		if err := destination.Resolve(allShards, func(shard string) error {
			s, ok := resolved[shard]
			if !ok {
				target := &querypb.Target{
					Keyspace:   keyspace,
					Shard:      shard,
					TabletType: tabletType,
					Cell:       r.localCell,
				}
				// Right now we always set the Cell to ""
				// Later we can fallback to another cell if needed.
				// We would then need to read the SrvKeyspace there too.
				target.Cell = ""
				s = len(result)
				result = append(result, &ResolvedShard{
					Target:  target,
					Gateway: r.gateway,
				})
				if ids != nil {
					values = append(values, nil)
				}
				resolved[shard] = s
			}
			if ids != nil {
				values[s] = append(values[s], ids[i])
			}
			return nil
		}); err != nil {
			return nil, nil, err
		}
	}
	return result, values, nil
}

// ResolveDestination is a shortcut to ResolveDestinations with only
// one Destination, and no ids.
func (r *Resolver) ResolveDestination(ctx context.Context, keyspace string, tabletType topodatapb.TabletType, destination key.Destination) ([]*ResolvedShard, error) {
	rss, _, err := r.ResolveDestinations(ctx, keyspace, tabletType, nil, []key.Destination{destination})
	return rss, err
}

// ValuesEqual is a helper method to compare arrays of values.
func ValuesEqual(vss1, vss2 [][]*querypb.Value) bool {
	if len(vss1) != len(vss2) {
		return false
	}
	for i, vs1 := range vss1 {
		if len(vs1) != len(vss2[i]) {
			return false
		}
		for j, v1 := range vs1 {
			if !proto.Equal(v1, vss2[i][j]) {
				return false
			}
		}
	}
	return true
}

// GetGateway returns the used gateway
func (r *Resolver) GetGateway() Gateway {
	return r.gateway
}
<|endoftext|>


<commit_before>package goldb

import (
	"errors"
	"fmt"
	"github.com/stretchr/testify/assert"
	"math/rand"
	"os"
	"testing"
)

func TestTransaction_PutVar(t *testing.T) {
	store := NewStorage(fmt.Sprintf("%s/test-goldb-%x.db", os.TempDir(), rand.Int()), nil)
	defer store.Drop()

	// put vars
	err := store.Exec(func(tr *Transaction) {
		tr.PutVar(Key(TabTest, "keyA"), "Alice") // string
		tr.PutVar(Key(TabTest, "keyB"), 0xB0b)   // int
	})

	// get vars
	var v1 string
	var v2 int
	store.GetVar(Key(TabTest, "keyA"), &v1)
	store.GetVar(Key(TabTest, "keyB"), &v2)

	assert.NoError(t, err)
	assert.Equal(t, "Alice", v1)
	assert.Equal(t, 0xB0b, v2)
}

func TestTransaction_GetVar(t *testing.T) {
	store := NewStorage(fmt.Sprintf("%s/test-goldb-%x.db", os.TempDir(), rand.Int()), nil)
	defer store.Drop()

	var a, b, c struct {
		v   string
		ok  bool
		err error
	}
	key := Key(TabTest, "id")
	err := store.Exec(func(tr *Transaction) {
		tr.PutVar(key, "Alice")

		// get data from transaction
		a.ok, a.err = tr.GetVar(key, &a.v)

		// get data from storage (before commit)
		b.ok, b.err = store.GetVar(key, &b.v)
	})
	// get data from storage (after commit)
	c.ok, c.err = store.GetVar(key, &c.v)

	assert.NoError(t, err)
	assert.NoError(t, a.err)
	assert.NoError(t, b.err)
	assert.NoError(t, c.err)
	assert.True(t, a.ok)
	assert.False(t, b.ok)
	assert.True(t, c.ok)
	assert.Equal(t, "Alice", a.v)
	assert.Equal(t, "", b.v)
	assert.Equal(t, "Alice", c.v)
}

func TestTransaction_Discard(t *testing.T) {
	store := NewStorage(fmt.Sprintf("%s/test-goldb-%x.db", os.TempDir(), rand.Int()), nil)
	defer store.Drop()

	var a, b struct {
		v   string
		ok  bool
		err error
	}
	key := Key(TabTest, "id")
	err := store.Exec(func(tr *Transaction) {
		tr.PutVar(key, "Alice")

		a.ok, a.err = tr.GetVar(key, &a.v) // get from transaction

		tr.Fail(errors.New("transaction-fail")) // discard transaction
	})
	b.ok, b.err = store.GetVar(key, &b.v) // get from storage

	assert.Error(t, err)
	assert.NoError(t, b.err)
	assert.True(t, a.ok)
	assert.False(t, b.ok)
	assert.Equal(t, "Alice", a.v)
	assert.Equal(t, "", b.v)
}

func TestTransaction_SequenceNextVal(t *testing.T) {
	store := NewStorage(fmt.Sprintf("%s/test-goldb-%x.db", os.TempDir(), rand.Int()), nil)
	defer store.Drop()

	var a, b, c, d uint64

	store.Exec(func(tr *Transaction) {
		a = tr.SequenceNextVal(TabTest)
		b = tr.SequenceNextVal(TabTest)
	})
	store.Exec(func(tr *Transaction) {
		c = tr.SequenceNextVal(TabTest)
		tr.Fail(errors.New("transaction-fail")) // discard transaction
	})
	store.Exec(func(tr *Transaction) {
		d = tr.SequenceNextVal(TabTest)
	})

	assert.True(t, b == a+1)
	assert.True(t, c == b+1)
	assert.True(t, d == b+1)
}

func TestTransaction_IncNum(t *testing.T) {
	var v1 int64
	store := NewStorage(fmt.Sprintf("%s/test-goldb-%x.db", os.TempDir(), rand.Int()), nil)
	defer store.Drop()
	key := Key(TabTest, "id")
	store.Exec(func(tr *Transaction) {
		tr.PutVar(key, 100)
	})

	err := store.Exec(func(tr *Transaction) {
		// increment value (transaction context)
		v1, _ = tr.IncInt(key, 23)
	})
	// get value (storage context)
	v2, _ := store.GetInt(key)

	assert.NoError(t, err)
	assert.Equal(t, int64(123), v1)
	assert.Equal(t, int64(123), v2)
}
<commit_msg>testing of put any objects<commit_after>package goldb

import (
	"errors"
	"fmt"
	"github.com/stretchr/testify/assert"
	"math/rand"
	"os"
	"testing"
)

type User struct {
	Name string
	Age  int
}

func TestTransaction_PutVar(t *testing.T) {
	store := NewStorage(fmt.Sprintf("%s/test-goldb-%x.db", os.TempDir(), rand.Int()), nil)
	defer store.Drop()

	// put vars
	err := store.Exec(func(tr *Transaction) {
		tr.PutVar(Key(TestTable, "keyA"), "Alice")           // string
		tr.PutVar(Key(TestTable, "keyB"), 0xB0B)             // int
		tr.PutVar(Key(TestTable, "keyC"), User{"Carol", 18}) // any object
	})

	// get vars
	var (
		v1 string
		v2 int
		v3 User
	)
	store.GetVar(Key(TestTable, "keyA"), &v1)
	store.GetVar(Key(TestTable, "keyB"), &v2)
	store.GetVar(Key(TestTable, "keyC"), &v3)

	assert.NoError(t, err)
	assert.Equal(t, "Alice", v1)
	assert.Equal(t, 0xB0B, v2)
	assert.Equal(t, User{"Carol", 18}, v3)
}

func TestTransaction_GetVar(t *testing.T) {
	store := NewStorage(fmt.Sprintf("%s/test-goldb-%x.db", os.TempDir(), rand.Int()), nil)
	defer store.Drop()

	var a, b, c struct {
		v   string
		ok  bool
		err error
	}
	key := Key(TestTable, "id")
	err := store.Exec(func(tr *Transaction) {
		tr.PutVar(key, "Alice")

		// get data from transaction
		a.ok, a.err = tr.GetVar(key, &a.v)

		// get data from storage (before commit)
		b.ok, b.err = store.GetVar(key, &b.v)
	})
	// get data from storage (after commit)
	c.ok, c.err = store.GetVar(key, &c.v)

	assert.NoError(t, err)
	assert.NoError(t, a.err)
	assert.NoError(t, b.err)
	assert.NoError(t, c.err)
	assert.True(t, a.ok)
	assert.False(t, b.ok)
	assert.True(t, c.ok)
	assert.Equal(t, "Alice", a.v)
	assert.Equal(t, "", b.v)
	assert.Equal(t, "Alice", c.v)
}

func TestTransaction_Discard(t *testing.T) {
	store := NewStorage(fmt.Sprintf("%s/test-goldb-%x.db", os.TempDir(), rand.Int()), nil)
	defer store.Drop()

	var a, b struct {
		v   string
		ok  bool
		err error
	}
	key := Key(TestTable, "id")
	err := store.Exec(func(tr *Transaction) {
		tr.PutVar(key, "Alice")

		a.ok, a.err = tr.GetVar(key, &a.v) // get from transaction

		tr.Fail(errors.New("transaction-fail")) // discard transaction
	})
	b.ok, b.err = store.GetVar(key, &b.v) // get from storage

	assert.Error(t, err)
	assert.NoError(t, b.err)
	assert.True(t, a.ok)
	assert.False(t, b.ok)
	assert.Equal(t, "Alice", a.v)
	assert.Equal(t, "", b.v)
}

func TestTransaction_SequenceNextVal(t *testing.T) {
	store := NewStorage(fmt.Sprintf("%s/test-goldb-%x.db", os.TempDir(), rand.Int()), nil)
	defer store.Drop()

	var a, b, c, d uint64

	store.Exec(func(tr *Transaction) {
		a = tr.SequenceNextVal(TestTable)
		b = tr.SequenceNextVal(TestTable)
	})
	store.Exec(func(tr *Transaction) {
		c = tr.SequenceNextVal(TestTable)
		tr.Fail(errors.New("transaction-fail")) // discard transaction
	})
	store.Exec(func(tr *Transaction) {
		d = tr.SequenceNextVal(TestTable)
	})

	assert.True(t, b == a+1)
	assert.True(t, c == b+1)
	assert.True(t, d == b+1)
}

func TestTransaction_IncNum(t *testing.T) {
	var v1 int64
	store := NewStorage(fmt.Sprintf("%s/test-goldb-%x.db", os.TempDir(), rand.Int()), nil)
	defer store.Drop()
	key := Key(TestTable, "id")
	store.Exec(func(tr *Transaction) {
		tr.PutVar(key, 100)
	})

	err := store.Exec(func(tr *Transaction) {
		// increment value (transaction context)
		v1, _ = tr.IncInt(key, 23)
	})
	// get value (storage context)
	v2, _ := store.GetInt(key)

	assert.NoError(t, err)
	assert.Equal(t, int64(123), v1)
	assert.Equal(t, int64(123), v2)
}
<|endoftext|>


<commit_before>package ipdb

import (
	"encoding/binary"
	"errors"
	"fmt"
	"io/ioutil"
	"math"
	"net"
	//"fmt"
)

const METALEN = 20

// Header struct
type Header struct {
	version                                                       uint32
	continentLen, countryLen, areaLen, regionLen, cityLen, ispLen uint16
	netLen                                                        uint32
}

// Record struct
type Record struct {
	ip                                                      uint32 //ip
	mask                                                    uint8  //mask
	continentID, countryID, areaID, regionID, cityID, ispID uint16
}

// Hdata struct
type Hdata struct {
	id   uint16
	name string
}

// DB struct
type DB struct {
	Head                                        Header
	Continent, Country, Area, Region, City, Isp map[uint16]string
	Idx                                         map[int]uint32
	Rstart                                      int
	Data                                        []byte
}

// Result structs
type Result struct {
	Cidr, Continent, Country, Area, Region, City, Isp string
}

func Load(dataFile string) (db *DB, err error) {
	data, err := ioutil.ReadFile(dataFile)
	if err != nil {
		return
	}
	db = new(DB)
	db.Init(data)
	return
}

func IP2long(ipv4 string) (ret uint32, err error) {
	ret = 0
	ip := net.ParseIP(ipv4)
	if ip == nil {
		err = errors.New("Invalid ip format")
		return
	}
	return binary.BigEndian.Uint32(ip.To4()), nil
}

func Long2Ip(a uint32) net.IP {
	return net.IPv4(byte(a>>24), byte(a>>16), byte(a>>8), byte(a))
}

func (db *DB) Init(data []byte) {
	db.Head = Header{
		binary.BigEndian.Uint32(data[:4]),
		binary.BigEndian.Uint16(data[4 : 4+2]),
		binary.BigEndian.Uint16(data[6 : 6+2]),
		binary.BigEndian.Uint16(data[8 : 8+2]),
		binary.BigEndian.Uint16(data[10 : 10+2]),
		binary.BigEndian.Uint16(data[12 : 12+2]),
		binary.BigEndian.Uint16(data[14 : 14+2]),
		binary.BigEndian.Uint32(data[16 : 16+4]),
	}
	//fmt.Printf("head = %+v", db.Head)
	db.Continent = make(map[uint16]string, db.Head.continentLen)
	for i := 0; i < int(db.Head.continentLen); i++ {
		pos := METALEN + i*4
		pack := data[pos : pos+4]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+2]),
		}
		db.Continent[dc.id] = dc.name
		//fmt.Printf("dc = %+v", dc)
	}
	db.Country = make(map[uint16]string, db.Head.countryLen)
	for i := 0; i < int(db.Head.countryLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + i*4
		pack := data[pos : pos+4]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+2]),
		}
		db.Country[dc.id] = dc.name
		//fmt.Printf("dc = %+v", dc)
	}
	db.Area = make(map[uint16]string, db.Head.areaLen)
	for i := 0; i < int(db.Head.areaLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 + i*66
		pack := data[pos : pos+66]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+64]),
		}
		db.Area[dc.id] = dc.name
		//fmt.Printf("dc = %+v", dc)
	}
	db.Region = make(map[uint16]string, db.Head.regionLen)
	for i := 0; i < int(db.Head.regionLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 +
			int(db.Head.areaLen)*66 + i*66
		pack := data[pos : pos+66]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+64]),
		}
		db.Region[dc.id] = dc.name
		//fmt.Printf("dc = %+v", dc)
	}
	db.City = make(map[uint16]string, db.Head.cityLen)
	for i := 0; i < int(db.Head.cityLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 +
			int(db.Head.areaLen)*66 + int(db.Head.regionLen)*66 + i*66
		pack := data[pos : pos+66]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+64]),
		}
		db.City[dc.id] = dc.name
		//fmt.Printf("dc = %+v", dc)
	}
	db.Isp = make(map[uint16]string, db.Head.ispLen)
	for i := 0; i < int(db.Head.ispLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 +
			int(db.Head.areaLen)*66 + int(db.Head.regionLen)*66 + int(db.Head.cityLen)*66 + i*66
		pack := data[pos : pos+66]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+64]),
		}
		db.Isp[dc.id] = dc.name
		//fmt.Printf("dc = %+v", dc)
	}
	db.Rstart = METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 +
		int(db.Head.areaLen)*66 + int(db.Head.regionLen)*66 + int(db.Head.cityLen)*66 +
		int(db.Head.ispLen)*66

	db.Idx = make(map[int]uint32, 256)
	for i := 0; i < 256; i++ {
		off := db.Rstart + int(db.Head.netLen)*17 + i*4
		db.Idx[i] = binary.BigEndian.Uint32(data[off : off+4])
	}
	//fmt.Printf("idx = %+v", db.Idx)
	db.Data = data
	return
}

func (db *DB) Find(ipv4 string) (result *Result, err error) {
	var ip32 uint32
	if ip32, err = IP2long(ipv4); err != nil {
		return nil, err
	}
	return db.FindByUint(ip32)
}

func (db *DB) findStartIdxOffset(ip uint32) (start int) {
	ipFirst := int(ip >> 24)
	start = 0
	for i := ipFirst; i > 0; i-- {
		if db.Idx[i] != 0 {
			start = int(db.Idx[i])
			break
		}
	}
	return
}

func (db *DB) findEndIdxOffset(ip uint32) (end int) {
	ipFirst := int(ip >> 24)
	end = int(db.Head.netLen - 1)
	for i := ipFirst + 1; i < 256; i++ {
		if db.Idx[i] != 0 {
			end = int(db.Idx[i])
			break
		}
	}
	return
}

func (db *DB) FindByUint(ip uint32) (result *Result, err error) {
	f := db.findStartIdxOffset(ip)
	n := 0
	l := db.findEndIdxOffset(ip)
	for f <= l {
		m := int((f + l) / 2)
		n = n + 1
		p := db.Rstart + m*17
		pack := db.Data[p : p+17]
		r := Record{
			binary.BigEndian.Uint32(pack[:4]),
			uint8(pack[4 : 4+1][0]),
			binary.BigEndian.Uint16(pack[5 : 5+2]),
			binary.BigEndian.Uint16(pack[7 : 7+2]),
			binary.BigEndian.Uint16(pack[9 : 9+2]),
			binary.BigEndian.Uint16(pack[11 : 11+2]),
			binary.BigEndian.Uint16(pack[13 : 13+2]),
			binary.BigEndian.Uint16(pack[15 : 15+2]),
		}
		rs := r.ip
		re := rs + uint32(math.Pow(2, float64(32-r.mask))) - 1
		if ip >= rs && ip <= re {
			return &Result{
				Cidr:      fmt.Sprintf("%s/%d", Long2Ip(rs).To4(), r.mask),
				Continent: db.Continent[r.continentID],
				Country:   db.Country[r.countryID],
				Area:      db.Area[r.areaID],
				Region:    db.Region[r.regionID],
				City:      db.City[r.cityID],
				Isp:       db.Isp[r.ispID],
			}, nil
		}
		if ip > re {
			f = m + 1
		}
		if ip < rs {
			l = m - 1
		}
	}
	return nil, errors.New("Not Found")
}
<commit_msg>fix trim<commit_after>package ipdb

import (
	"encoding/binary"
	"errors"
	"fmt"
	"io/ioutil"
	"math"
	"net"
	"strings"
	//"fmt"
)

const METALEN = 20

// Header struct
type Header struct {
	version                                                       uint32
	continentLen, countryLen, areaLen, regionLen, cityLen, ispLen uint16
	netLen                                                        uint32
}

// Record struct
type Record struct {
	ip                                                      uint32 //ip
	mask                                                    uint8  //mask
	continentID, countryID, areaID, regionID, cityID, ispID uint16
}

// Hdata struct
type Hdata struct {
	id   uint16
	name string
}

// DB struct
type DB struct {
	Head                                        Header
	Continent, Country, Area, Region, City, Isp map[uint16]string
	Idx                                         map[int]uint32
	Rstart                                      int
	Data                                        []byte
}

// Result structs
type Result struct {
	Cidr, Continent, Country, Area, Region, City, Isp string
}

func Load(dataFile string) (db *DB, err error) {
	data, err := ioutil.ReadFile(dataFile)
	if err != nil {
		return
	}
	db = new(DB)
	db.Init(data)
	return
}

func IP2long(ipv4 string) (ret uint32, err error) {
	ret = 0
	ip := net.ParseIP(ipv4)
	if ip == nil {
		err = errors.New("Invalid ip format")
		return
	}
	return binary.BigEndian.Uint32(ip.To4()), nil
}

func Long2Ip(a uint32) net.IP {
	return net.IPv4(byte(a>>24), byte(a>>16), byte(a>>8), byte(a))
}

func (db *DB) Init(data []byte) {
	db.Head = Header{
		binary.BigEndian.Uint32(data[:4]),
		binary.BigEndian.Uint16(data[4 : 4+2]),
		binary.BigEndian.Uint16(data[6 : 6+2]),
		binary.BigEndian.Uint16(data[8 : 8+2]),
		binary.BigEndian.Uint16(data[10 : 10+2]),
		binary.BigEndian.Uint16(data[12 : 12+2]),
		binary.BigEndian.Uint16(data[14 : 14+2]),
		binary.BigEndian.Uint32(data[16 : 16+4]),
	}
	//fmt.Printf("head = %+v", db.Head)
	db.Continent = make(map[uint16]string, db.Head.continentLen)
	for i := 0; i < int(db.Head.continentLen); i++ {
		pos := METALEN + i*4
		pack := data[pos : pos+4]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+2]),
		}
		db.Continent[dc.id] = dc.name
		//fmt.Printf("dc = %+v", dc)
	}
	db.Country = make(map[uint16]string, db.Head.countryLen)
	for i := 0; i < int(db.Head.countryLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + i*4
		pack := data[pos : pos+4]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+2]),
		}
		db.Country[dc.id] = dc.name
		//fmt.Printf("dc = %+v", dc)
	}
	db.Area = make(map[uint16]string, db.Head.areaLen)
	for i := 0; i < int(db.Head.areaLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 + i*66
		pack := data[pos : pos+66]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+64]),
		}
		db.Area[dc.id] = dc.name
		//fmt.Printf("dc = %+v", dc)
	}
	db.Region = make(map[uint16]string, db.Head.regionLen)
	for i := 0; i < int(db.Head.regionLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 +
			int(db.Head.areaLen)*66 + i*66
		pack := data[pos : pos+66]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+64]),
		}
		db.Region[dc.id] = dc.name
		//fmt.Printf("dc = %+v", dc)
	}
	db.City = make(map[uint16]string, db.Head.cityLen)
	for i := 0; i < int(db.Head.cityLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 +
			int(db.Head.areaLen)*66 + int(db.Head.regionLen)*66 + i*66
		pack := data[pos : pos+66]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+64]),
		}
		db.City[dc.id] = dc.name
		//fmt.Printf("dc = %+v", dc)
	}
	db.Isp = make(map[uint16]string, db.Head.ispLen)
	for i := 0; i < int(db.Head.ispLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 +
			int(db.Head.areaLen)*66 + int(db.Head.regionLen)*66 + int(db.Head.cityLen)*66 + i*66
		pack := data[pos : pos+66]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+64]),
		}
		db.Isp[dc.id] = dc.name
		//fmt.Printf("dc = %+v", dc)
	}
	db.Rstart = METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 +
		int(db.Head.areaLen)*66 + int(db.Head.regionLen)*66 + int(db.Head.cityLen)*66 +
		int(db.Head.ispLen)*66

	db.Idx = make(map[int]uint32, 256)
	for i := 0; i < 256; i++ {
		off := db.Rstart + int(db.Head.netLen)*17 + i*4
		db.Idx[i] = binary.BigEndian.Uint32(data[off : off+4])
	}
	//fmt.Printf("idx = %+v", db.Idx)
	db.Data = data
	return
}

func (db *DB) Find(ipv4 string) (result *Result, err error) {
	var ip32 uint32
	if ip32, err = IP2long(ipv4); err != nil {
		return nil, err
	}
	return db.FindByUint(ip32)
}

func (db *DB) findStartIdxOffset(ip uint32) (start int) {
	ipFirst := int(ip >> 24)
	start = 0
	for i := ipFirst; i > 0; i-- {
		if db.Idx[i] != 0 {
			start = int(db.Idx[i])
			break
		}
	}
	return
}

func (db *DB) findEndIdxOffset(ip uint32) (end int) {
	ipFirst := int(ip >> 24)
	end = int(db.Head.netLen - 1)
	for i := ipFirst + 1; i < 256; i++ {
		if db.Idx[i] != 0 {
			end = int(db.Idx[i])
			break
		}
	}
	return
}

func (db *DB) FindByUint(ip uint32) (result *Result, err error) {
	f := db.findStartIdxOffset(ip)
	n := 0
	l := db.findEndIdxOffset(ip)
	for f <= l {
		m := int((f + l) / 2)
		n = n + 1
		p := db.Rstart + m*17
		pack := db.Data[p : p+17]
		r := Record{
			binary.BigEndian.Uint32(pack[:4]),
			uint8(pack[4 : 4+1][0]),
			binary.BigEndian.Uint16(pack[5 : 5+2]),
			binary.BigEndian.Uint16(pack[7 : 7+2]),
			binary.BigEndian.Uint16(pack[9 : 9+2]),
			binary.BigEndian.Uint16(pack[11 : 11+2]),
			binary.BigEndian.Uint16(pack[13 : 13+2]),
			binary.BigEndian.Uint16(pack[15 : 15+2]),
		}
		rs := r.ip
		re := rs + uint32(math.Pow(2, float64(32-r.mask))) - 1
		if ip >= rs && ip <= re {
			return &Result{
				Cidr:      fmt.Sprintf("%s/%d", Long2Ip(rs).To4(), r.mask),
				Continent: strings.TrimSpace(db.Continent[r.continentID]),
				Country:   strings.TrimSpace(db.Country[r.countryID]),
				Area:      strings.TrimSpace(db.Area[r.areaID]),
				Region:    strings.TrimSpace(db.Region[r.regionID]),
				City:      strings.TrimSpace(db.City[r.cityID]),
				Isp:       strings.TrimSpace(db.Isp[r.ispID]),
			}, nil
		}
		if ip > re {
			f = m + 1
		}
		if ip < rs {
			l = m - 1
		}
	}
	return nil, errors.New("Not Found")
}
<|endoftext|>


<commit_before>package server

import (
	"../hashtree"
	"../network"
	"fmt"
	"net"
	"testing"
)

func TestServerDataProvider(t *testing.T) {

	set1 := Setting{network.ServerHello{}, ".testServer1", "simple", 30001, "127.0.0.1"}
	set2 := Setting{network.ServerHello{}, ".testServer1", "simple", 30002, "127.0.0.1"}
	s1 := NewServer(set1)
	s2 := NewServer(set2)

	block := hashtree.Bytes(hashtree.FILE_BLOCK_SIZE)

	fileSizes := []hashtree.Bytes{0, 1, block - 1, block, block + 1, block * 2, block * 3, block * 4, block * 5}

	conn := s2.conn
	toAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%v:%v", set1.IP, set1.Port))
	if err != nil {
		panic(err)
	}

	for _, size := range fileSizes {
		id := s1.ImportFromReader(&testFile{length: size})

		p := &network.Packet{}

		p.FillHashRequest(id, 0, 0, hashtree.FileNodesDefault(size))

		conn.Send(p, toAddr)

		//todo, check returned

	}

}
<commit_msg>changed test port, it was in use<commit_after>package server

import (
	"../hashtree"
	"../network"
	"fmt"
	"net"
	"testing"
)

func TestServerDataProvider(t *testing.T) {

	set1 := Setting{network.ServerHello{}, ".testServer1", "simple", 30011, "127.0.0.1"}
	set2 := Setting{network.ServerHello{}, ".testServer1", "simple", 30012, "127.0.0.1"}
	s1 := NewServer(set1)
	s2 := NewServer(set2)

	block := hashtree.Bytes(hashtree.FILE_BLOCK_SIZE)

	fileSizes := []hashtree.Bytes{0, 1, block - 1, block, block + 1, block * 2, block * 3, block * 4, block * 5}

	conn := s2.conn
	toAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%v:%v", set1.IP, set1.Port))
	if err != nil {
		panic(err)
	}

	for _, size := range fileSizes {
		id := s1.ImportFromReader(&testFile{length: size})

		p := &network.Packet{}

		p.FillHashRequest(id, 0, 0, hashtree.FileNodesDefault(size))

		conn.Send(p, toAddr)

		//todo, check returned

	}

}
<|endoftext|>


<commit_before>package composition

import (
	"github.com/tarent/lib-compose/logging"
	"io"
	"net/http"
	"strconv"
	"strings"
)

// A ContentFetcherFactory returns a configured fetch job for a request
// which can return the fetch results.
type ContentFetcherFactory func(r *http.Request) FetchResultSupplier

type CompositionHandler struct {
	contentFetcherFactory ContentFetcherFactory
	contentMergerFactory  func(metaJSON map[string]interface{}) ContentMerger
	cache                 Cache
}

// NewCompositionHandler creates a new Handler with the supplied defualtData,
// which is used for each request.
func NewCompositionHandler(contentFetcherFactory ContentFetcherFactory) *CompositionHandler {
	return &CompositionHandler{
		contentFetcherFactory: contentFetcherFactory,
		contentMergerFactory: func(metaJSON map[string]interface{}) ContentMerger {
			return NewContentMerge(metaJSON)
		},
		cache: nil,
	}
}

func NewCompositionHandlerWithCache(contentFetcherFactory ContentFetcherFactory, cache Cache) *CompositionHandler {
	return &CompositionHandler{
		contentFetcherFactory: contentFetcherFactory,
		contentMergerFactory: func(metaJSON map[string]interface{}) ContentMerger {
			return NewContentMerge(metaJSON)
		},
		cache: cache,
	}
}

func (agg *CompositionHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	fetcher := agg.contentFetcherFactory(r)

	if fetcher.Empty() {
		w.WriteHeader(500)
		w.Write([]byte("Internal server error"))
		logging.Application(r.Header).Error("No fetchers available for composition, throwing error 500")
		return
	}

	// fetch all contents
	results := fetcher.WaitForResults()

	mergeContext := agg.contentMergerFactory(fetcher.MetaJSON())

	for _, res := range results {
		if res.Err == nil && res.Content != nil {

			if res.Content.HttpStatusCode() >= 300 && res.Content.HttpStatusCode() <= 308 {
				copyHeaders(res.Content.HttpHeader(), w.Header(), ForwardResponseHeaders)
				w.WriteHeader(res.Content.HttpStatusCode())
				return
			}

			if res.Content.Reader() != nil {
				copyHeaders(res.Content.HttpHeader(), w.Header(), ForwardResponseHeaders)
				w.WriteHeader(res.Content.HttpStatusCode())
				io.Copy(w, res.Content.Reader())
				res.Content.Reader().Close()
				return
			}

			mergeContext.AddContent(res)

		} else if res.Def.Required {
			// 404 Error already become logged in logger.go
			if res.Content.HttpStatusCode() != 404 {
				logging.Application(r.Header).WithField("fetchResult", res).Errorf("error loading content from: %v", res.Def.URL)
			}
			res.Def.ErrHandler.Handle(res.Err, res.Content.HttpStatusCode(), w, r)
			return
		} else {
			logging.Application(r.Header).WithField("fetchResult", res).Warnf("optional content not loaded: %v", res.Def.URL)
		}
	}

	status := 200
	// Take status code and headers from first fetch definition
	if len(results) > 0 {
		copyHeaders(results[0].Content.HttpHeader(), w.Header(), ForwardResponseHeaders)
		if results[0].Content.HttpStatusCode() != 0 {
			status = results[0].Content.HttpStatusCode()
		}
	}

	// Overwrite Content-Type to ensure, that the encoding is correct
	w.Header().Set("Content-Type", "text/html; charset=utf-8")

	html, err := mergeContext.GetHtml()
	if err != nil {
		if agg.cache != nil {
			agg.cache.PurgeEntries(mergeContext.GetHashes())
		}
		logging.Application(r.Header).Error(err.Error())
		http.Error(w, "Internal Server Error: "+err.Error(), 500)
		return
	}

	w.Header().Set("Content-Length", strconv.Itoa(len(html)))
	w.WriteHeader(status)
	w.Write(html)
}

func MetadataForRequest(r *http.Request) map[string]interface{} {
	return map[string]interface{}{
		"host":     getHostFromRequest(r),
		"base_url": getBaseUrlFromRequest(r),
		"params":   r.URL.Query(),
	}
}

func getBaseUrlFromRequest(r *http.Request) string {
	proto := "http"
	if r.TLS != nil {
		proto = "https"
	}
	if xfph := r.Header.Get("X-Forwarded-Proto"); xfph != "" {
		protoParts := strings.SplitN(xfph, ",", 2)
		proto = protoParts[0]
	}

	return proto + "://" + getHostFromRequest(r)
}

func getHostFromRequest(r *http.Request) string {
	host := r.Host
	if xffh := r.Header.Get("X-Forwarded-For"); xffh != "" {
		hostParts := strings.SplitN(xffh, ",", 2)
		host = hostParts[0]
	}
	return host
}
<commit_msg>Adds status code 502 to condition, so errors are not displayed multiple times<commit_after>package composition

import (
	"github.com/tarent/lib-compose/logging"
	"io"
	"net/http"
	"strconv"
	"strings"
)

// A ContentFetcherFactory returns a configured fetch job for a request
// which can return the fetch results.
type ContentFetcherFactory func(r *http.Request) FetchResultSupplier

type CompositionHandler struct {
	contentFetcherFactory ContentFetcherFactory
	contentMergerFactory  func(metaJSON map[string]interface{}) ContentMerger
	cache                 Cache
}

// NewCompositionHandler creates a new Handler with the supplied defualtData,
// which is used for each request.
func NewCompositionHandler(contentFetcherFactory ContentFetcherFactory) *CompositionHandler {
	return &CompositionHandler{
		contentFetcherFactory: contentFetcherFactory,
		contentMergerFactory: func(metaJSON map[string]interface{}) ContentMerger {
			return NewContentMerge(metaJSON)
		},
		cache: nil,
	}
}

func NewCompositionHandlerWithCache(contentFetcherFactory ContentFetcherFactory, cache Cache) *CompositionHandler {
	return &CompositionHandler{
		contentFetcherFactory: contentFetcherFactory,
		contentMergerFactory: func(metaJSON map[string]interface{}) ContentMerger {
			return NewContentMerge(metaJSON)
		},
		cache: cache,
	}
}

func (agg *CompositionHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	fetcher := agg.contentFetcherFactory(r)

	if fetcher.Empty() {
		w.WriteHeader(500)
		w.Write([]byte("Internal server error"))
		logging.Application(r.Header).Error("No fetchers available for composition, throwing error 500")
		return
	}

	// fetch all contents
	results := fetcher.WaitForResults()

	mergeContext := agg.contentMergerFactory(fetcher.MetaJSON())

	for _, res := range results {
		if res.Err == nil && res.Content != nil {

			if res.Content.HttpStatusCode() >= 300 && res.Content.HttpStatusCode() <= 308 {
				copyHeaders(res.Content.HttpHeader(), w.Header(), ForwardResponseHeaders)
				w.WriteHeader(res.Content.HttpStatusCode())
				return
			}

			if res.Content.Reader() != nil {
				copyHeaders(res.Content.HttpHeader(), w.Header(), ForwardResponseHeaders)
				w.WriteHeader(res.Content.HttpStatusCode())
				io.Copy(w, res.Content.Reader())
				res.Content.Reader().Close()
				return
			}

			mergeContext.AddContent(res)

		} else if res.Def.Required {
			// 404 and 502 Error already become logged in logger.go
			if res.Content.HttpStatusCode() != 404 && res.Content.HttpStatusCode() != 502 {
				logging.Application(r.Header).WithField("fetchResult", res).Errorf("error loading content from: %v", res.Def.URL)
			}
			res.Def.ErrHandler.Handle(res.Err, res.Content.HttpStatusCode(), w, r)
			return
		} else {
			logging.Application(r.Header).WithField("fetchResult", res).Warnf("optional content not loaded: %v", res.Def.URL)
		}
	}

	status := 200
	// Take status code and headers from first fetch definition
	if len(results) > 0 {
		copyHeaders(results[0].Content.HttpHeader(), w.Header(), ForwardResponseHeaders)
		if results[0].Content.HttpStatusCode() != 0 {
			status = results[0].Content.HttpStatusCode()
		}
	}

	// Overwrite Content-Type to ensure, that the encoding is correct
	w.Header().Set("Content-Type", "text/html; charset=utf-8")

	html, err := mergeContext.GetHtml()
	if err != nil {
		if agg.cache != nil {
			agg.cache.PurgeEntries(mergeContext.GetHashes())
		}
		logging.Application(r.Header).Error(err.Error())
		http.Error(w, "Internal Server Error: "+err.Error(), 500)
		return
	}

	w.Header().Set("Content-Length", strconv.Itoa(len(html)))
	w.WriteHeader(status)
	w.Write(html)
}

func MetadataForRequest(r *http.Request) map[string]interface{} {
	return map[string]interface{}{
		"host":     getHostFromRequest(r),
		"base_url": getBaseUrlFromRequest(r),
		"params":   r.URL.Query(),
	}
}

func getBaseUrlFromRequest(r *http.Request) string {
	proto := "http"
	if r.TLS != nil {
		proto = "https"
	}
	if xfph := r.Header.Get("X-Forwarded-Proto"); xfph != "" {
		protoParts := strings.SplitN(xfph, ",", 2)
		proto = protoParts[0]
	}

	return proto + "://" + getHostFromRequest(r)
}

func getHostFromRequest(r *http.Request) string {
	host := r.Host
	if xffh := r.Header.Get("X-Forwarded-For"); xffh != "" {
		hostParts := strings.SplitN(xffh, ",", 2)
		host = hostParts[0]
	}
	return host
}
<|endoftext|>


<commit_before>package transpiler

import (
	"strings"

	goast "go/ast"
	"go/token"

	"fmt"
	"github.com/elliotchance/c2go/util"
)

func transpileUnion(name string, size int, fields []*goast.Field) []goast.Decl {
	res := []goast.Decl{
		// Type declaration (array: [x]byte with x the size of union)
		&goast.GenDecl{
			Tok: token.TYPE,
			Specs: []goast.Spec{
				&goast.TypeSpec{
					Name: util.NewIdent(name),
					Type: &goast.ArrayType{
						Elt: util.NewIdent("byte"),
						Len: util.NewIntLit(size), // Size of the union
					},
				},
			},
		},

		// cast() method
		&goast.FuncDecl{
			Name: util.NewIdent("cast"),
			Recv: &goast.FieldList{
				List: []*goast.Field{
					{
						Names: []*goast.Ident{util.NewIdent("self")},
						Type: &goast.StarExpr{
							X: util.NewIdent(name),
						},
					},
				},
			},
			Type: &goast.FuncType{
				Params: &goast.FieldList{
					List: []*goast.Field{
						{
							Names: []*goast.Ident{util.NewIdent("t")},
							Type: &goast.SelectorExpr{
								X:   util.NewIdent("reflect"),
								Sel: util.NewIdent("Type"),
							},
						},
					},
				},
				Results: &goast.FieldList{
					List: []*goast.Field{
						{
							Type: &goast.SelectorExpr{
								X:   util.NewIdent("reflect"),
								Sel: util.NewIdent("Value"),
							},
						},
					},
				},
			},
			Body: &goast.BlockStmt{
				List: []goast.Stmt{
					&goast.ReturnStmt{
						Results: []goast.Expr{
							util.NewCallExpr("reflect.NewAt",
								util.NewIdent("t"),
								&goast.CallExpr{
									Fun: &goast.SelectorExpr{
										X:   util.NewIdent("unsafe"),
										Sel: util.NewIdent("Pointer"),
									},
									Args: []goast.Expr{
										&goast.UnaryExpr{
											Op: token.AND,
											X: &goast.IndexExpr{
												X:     util.NewIdent("self"),
												Index: util.NewIntLit(0),
											},
										},
									},
								},
							),
						},
					},
				},
			},
		},

		// assign() method
		&goast.FuncDecl{
			Name: util.NewIdent("assign"),
			Recv: &goast.FieldList{
				List: []*goast.Field{
					{
						Names: []*goast.Ident{util.NewIdent("self")},
						Type: &goast.StarExpr{
							X: util.NewIdent(name),
						},
					},
				},
			},
			Type: &goast.FuncType{
				Params: &goast.FieldList{
					List: []*goast.Field{
						{
							Names: []*goast.Ident{util.NewIdent("v")},
							Type: &goast.InterfaceType{
								Methods: new(goast.FieldList),
							},
						},
					},
				},
			},
			Body: &goast.BlockStmt{
				List: []goast.Stmt{
					&goast.AssignStmt{
						Lhs: []goast.Expr{
							util.NewIdent("value"),
						},
						Tok: token.DEFINE,
						Rhs: []goast.Expr{
							&goast.CallExpr{
								Fun: &goast.SelectorExpr{
									X:   util.NewCallExpr("reflect.ValueOf", util.NewIdent("v")),
									Sel: util.NewIdent("Elem"),
								},
							},
						},
					},
					util.NewExprStmt(
						util.NewCallExpr("value.Set",
							&goast.CallExpr{
								Fun: &goast.SelectorExpr{
									X:   util.NewCallExpr("self.cast", util.NewCallExpr("value.Type")),
									Sel: util.NewIdent("Elem"),
								},
							},
						),
					),
				},
			},
		},

		// UntypedSet() method
		&goast.FuncDecl{
			Name: util.NewIdent("UntypedSet"),
			Recv: &goast.FieldList{
				List: []*goast.Field{
					{
						Names: []*goast.Ident{util.NewIdent("self")},
						Type: &goast.StarExpr{
							X: util.NewIdent(name),
						},
					},
				},
			},
			Type: &goast.FuncType{
				Params: &goast.FieldList{
					List: []*goast.Field{
						{
							Names: []*goast.Ident{util.NewIdent("v")},
							Type: &goast.InterfaceType{
								Methods: new(goast.FieldList),
							},
						},
					},
				},
			},
			Body: &goast.BlockStmt{
				List: []goast.Stmt{
					&goast.AssignStmt{
						Lhs: []goast.Expr{
							util.NewIdent("value"),
						},
						Tok: token.DEFINE,
						Rhs: []goast.Expr{
							util.NewCallExpr("reflect.ValueOf", util.NewIdent("v")),
						},
					},
					util.NewExprStmt(
						&goast.CallExpr{
							Fun: &goast.SelectorExpr{
								X: &goast.CallExpr{
									Fun: &goast.SelectorExpr{
										X:   util.NewCallExpr("self.cast", util.NewCallExpr("value.Type")),
										Sel: util.NewIdent("Elem"),
									},
								},
								Sel: util.NewIdent("Set"),
							},
							Args: []goast.Expr{
								util.NewIdent("value"),
							},
						},
					),
				},
			},
		},
	}

	// Methods for each union field
	for _, f := range fields {
		fieldID := strings.Title(f.Names[0].Name)

		res = append(res,
			// Setter method (SetXX)
			&goast.FuncDecl{
				Name: util.NewIdent("Set" + fieldID),
				Recv: &goast.FieldList{
					List: []*goast.Field{
						{
							Names: []*goast.Ident{util.NewIdent("self")},
							Type: &goast.StarExpr{
								X: util.NewIdent(name),
							},
						},
					},
				},
				Type: &goast.FuncType{
					Params: &goast.FieldList{
						List: []*goast.Field{
							{
								Names: []*goast.Ident{util.NewIdent("v")},
								Type:  f.Type,
							},
						},
					},
					Results: &goast.FieldList{
						List: []*goast.Field{
							{
								Type: f.Type,
							},
						},
					},
				},
				Body: &goast.BlockStmt{
					List: []goast.Stmt{
						util.NewExprStmt(
							util.NewCallExpr("self.UntypedSet", util.NewIdent("v")),
						),
						&goast.ReturnStmt{
							Results: []goast.Expr{
								util.NewIdent("v"),
							},
						},
					},
				},
			},

			// Getter method (GetXX)
			&goast.FuncDecl{
				Name: util.NewIdent("Get" + fieldID),
				Recv: &goast.FieldList{
					List: []*goast.Field{
						{
							Names: []*goast.Ident{util.NewIdent("self")},
							Type: &goast.StarExpr{
								X: util.NewIdent(name),
							},
						},
					},
				},
				Type: &goast.FuncType{
					Results: &goast.FieldList{
						List: []*goast.Field{
							{
								Names: []*goast.Ident{util.NewIdent("res")},
								Type:  f.Type,
							},
						},
					},
				},
				Body: &goast.BlockStmt{
					List: []goast.Stmt{
						util.NewExprStmt(
							util.NewCallExpr("self.assign", util.NewUnaryExpr(token.AND, util.NewIdent("res"))),
						),
						new(goast.ReturnStmt),
					},
				},
			},
		)
	}

	return res
}

func getFunctionNameForUnionGetter(variableName, variableType, attributeName string) string {
	if strings.HasPrefix(variableType, "[]") {
		return fmt.Sprintf("%s[0].Get%s", variableName, strings.Title(attributeName))
	}

	return fmt.Sprintf("%s.Get%s", variableName, strings.Title(attributeName))
}

func getFunctionNameForUnionSetter(variableName, variableType, attributeName string) string {
	if strings.HasPrefix(variableType, "[]") {
		return fmt.Sprintf("%s[0].Set%s", variableName, strings.Title(attributeName))
	}

	return fmt.Sprintf("%s.Set%s", variableName, strings.Title(attributeName))
}
<commit_msg>Fix duplicate code<commit_after>package transpiler

import (
	"strings"

	goast "go/ast"
	"go/token"

	"fmt"
	"github.com/elliotchance/c2go/util"
)

func transpileUnion(name string, size int, fields []*goast.Field) []goast.Decl {
	res := []goast.Decl{
		// Type declaration (array: [x]byte with x the size of union)
		&goast.GenDecl{
			Tok: token.TYPE,
			Specs: []goast.Spec{
				&goast.TypeSpec{
					Name: util.NewIdent(name),
					Type: &goast.ArrayType{
						Elt: util.NewIdent("byte"),
						Len: util.NewIntLit(size), // Size of the union
					},
				},
			},
		},

		// cast() method
		&goast.FuncDecl{
			Name: util.NewIdent("cast"),
			Recv: &goast.FieldList{
				List: []*goast.Field{
					{
						Names: []*goast.Ident{util.NewIdent("self")},
						Type: &goast.StarExpr{
							X: util.NewIdent(name),
						},
					},
				},
			},
			Type: &goast.FuncType{
				Params: &goast.FieldList{
					List: []*goast.Field{
						{
							Names: []*goast.Ident{util.NewIdent("t")},
							Type: &goast.SelectorExpr{
								X:   util.NewIdent("reflect"),
								Sel: util.NewIdent("Type"),
							},
						},
					},
				},
				Results: &goast.FieldList{
					List: []*goast.Field{
						{
							Type: &goast.SelectorExpr{
								X:   util.NewIdent("reflect"),
								Sel: util.NewIdent("Value"),
							},
						},
					},
				},
			},
			Body: &goast.BlockStmt{
				List: []goast.Stmt{
					&goast.ReturnStmt{
						Results: []goast.Expr{
							util.NewCallExpr("reflect.NewAt",
								util.NewIdent("t"),
								&goast.CallExpr{
									Fun: &goast.SelectorExpr{
										X:   util.NewIdent("unsafe"),
										Sel: util.NewIdent("Pointer"),
									},
									Args: []goast.Expr{
										&goast.UnaryExpr{
											Op: token.AND,
											X: &goast.IndexExpr{
												X:     util.NewIdent("self"),
												Index: util.NewIntLit(0),
											},
										},
									},
								},
							),
						},
					},
				},
			},
		},

		// assign() method
		&goast.FuncDecl{
			Name: util.NewIdent("assign"),
			Recv: &goast.FieldList{
				List: []*goast.Field{
					{
						Names: []*goast.Ident{util.NewIdent("self")},
						Type: &goast.StarExpr{
							X: util.NewIdent(name),
						},
					},
				},
			},
			Type: &goast.FuncType{
				Params: &goast.FieldList{
					List: []*goast.Field{
						{
							Names: []*goast.Ident{util.NewIdent("v")},
							Type: &goast.InterfaceType{
								Methods: new(goast.FieldList),
							},
						},
					},
				},
			},
			Body: &goast.BlockStmt{
				List: []goast.Stmt{
					&goast.AssignStmt{
						Lhs: []goast.Expr{
							util.NewIdent("value"),
						},
						Tok: token.DEFINE,
						Rhs: []goast.Expr{
							&goast.CallExpr{
								Fun: &goast.SelectorExpr{
									X:   util.NewCallExpr("reflect.ValueOf", util.NewIdent("v")),
									Sel: util.NewIdent("Elem"),
								},
							},
						},
					},
					util.NewExprStmt(
						util.NewCallExpr("value.Set",
							&goast.CallExpr{
								Fun: &goast.SelectorExpr{
									X:   util.NewCallExpr("self.cast", util.NewCallExpr("value.Type")),
									Sel: util.NewIdent("Elem"),
								},
							},
						),
					),
				},
			},
		},

		// UntypedSet() method
		&goast.FuncDecl{
			Name: util.NewIdent("UntypedSet"),
			Recv: &goast.FieldList{
				List: []*goast.Field{
					{
						Names: []*goast.Ident{util.NewIdent("self")},
						Type: &goast.StarExpr{
							X: util.NewIdent(name),
						},
					},
				},
			},
			Type: &goast.FuncType{
				Params: &goast.FieldList{
					List: []*goast.Field{
						{
							Names: []*goast.Ident{util.NewIdent("v")},
							Type: &goast.InterfaceType{
								Methods: new(goast.FieldList),
							},
						},
					},
				},
			},
			Body: &goast.BlockStmt{
				List: []goast.Stmt{
					&goast.AssignStmt{
						Lhs: []goast.Expr{
							util.NewIdent("value"),
						},
						Tok: token.DEFINE,
						Rhs: []goast.Expr{
							util.NewCallExpr("reflect.ValueOf", util.NewIdent("v")),
						},
					},
					util.NewExprStmt(
						&goast.CallExpr{
							Fun: &goast.SelectorExpr{
								X: &goast.CallExpr{
									Fun: &goast.SelectorExpr{
										X:   util.NewCallExpr("self.cast", util.NewCallExpr("value.Type")),
										Sel: util.NewIdent("Elem"),
									},
								},
								Sel: util.NewIdent("Set"),
							},
							Args: []goast.Expr{
								util.NewIdent("value"),
							},
						},
					),
				},
			},
		},
	}

	// Methods for each union field
	for _, f := range fields {
		fieldID := strings.Title(f.Names[0].Name)

		res = append(res,
			// Setter method (SetXX)
			&goast.FuncDecl{
				Name: util.NewIdent("Set" + fieldID),
				Recv: &goast.FieldList{
					List: []*goast.Field{
						{
							Names: []*goast.Ident{util.NewIdent("self")},
							Type: &goast.StarExpr{
								X: util.NewIdent(name),
							},
						},
					},
				},
				Type: &goast.FuncType{
					Params: &goast.FieldList{
						List: []*goast.Field{
							{
								Names: []*goast.Ident{util.NewIdent("v")},
								Type:  f.Type,
							},
						},
					},
					Results: &goast.FieldList{
						List: []*goast.Field{
							{
								Type: f.Type,
							},
						},
					},
				},
				Body: &goast.BlockStmt{
					List: []goast.Stmt{
						util.NewExprStmt(
							util.NewCallExpr("self.UntypedSet", util.NewIdent("v")),
						),
						&goast.ReturnStmt{
							Results: []goast.Expr{
								util.NewIdent("v"),
							},
						},
					},
				},
			},

			// Getter method (GetXX)
			&goast.FuncDecl{
				Name: util.NewIdent("Get" + fieldID),
				Recv: &goast.FieldList{
					List: []*goast.Field{
						{
							Names: []*goast.Ident{util.NewIdent("self")},
							Type: &goast.StarExpr{
								X: util.NewIdent(name),
							},
						},
					},
				},
				Type: &goast.FuncType{
					Results: &goast.FieldList{
						List: []*goast.Field{
							{
								Names: []*goast.Ident{util.NewIdent("res")},
								Type:  f.Type,
							},
						},
					},
				},
				Body: &goast.BlockStmt{
					List: []goast.Stmt{
						util.NewExprStmt(
							util.NewCallExpr("self.assign", util.NewUnaryExpr(token.AND, util.NewIdent("res"))),
						),
						new(goast.ReturnStmt),
					},
				},
			},
		)
	}

	return res
}

func getFunctionNameForUnion(verb, variableName, variableType, attributeName string) string {
	if strings.HasPrefix(variableType, "[]") {
		return fmt.Sprintf("%s[0].%s%s", variableName, verb, strings.Title(attributeName))
	}

	return fmt.Sprintf("%s.%s%s", variableName, verb, strings.Title(attributeName))
}

func getFunctionNameForUnionGetter(variableName, variableType, attributeName string) string {
	return getFunctionNameForUnion("Get", variableName, variableType, attributeName)
}

func getFunctionNameForUnionSetter(variableName, variableType, attributeName string) string {
	return getFunctionNameForUnion("Set", variableName, variableType, attributeName)
}
<|endoftext|>