<commit_before>package main

import (
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"strings"
)

type downloadResult struct {
	Page  []byte
	Error error
}

type releaseView struct {
	DownloadUrl string
	Version     string
	Md5         string
	Path        string
}

var rePackageUrl = regexp.MustCompile(`(?i)<a href=\"(?P<url>.+?)#md5=.+?\">(?P<filename>.+?)</a>`)
var reDownloadUrl = regexp.MustCompile(`(?i)<a href=\"(?P<url>.+?)\"\s+rel=\".+\">(?P<version>.+?) download_url</a>`)

func simpleIndexHandler(w http.ResponseWriter, r *http.Request) {
	renderTemplate(w, "index", nil)
}

func getPage(url string) downloadResult {
	log.Println("Downloading page: " + url)
	res, err := http.Get(url)
	if err != nil {
		return downloadResult{Page: []byte(""), Error: err}
	}
	page, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return downloadResult{Page: []byte(""), Error: err}
	}
	return downloadResult{Page: page, Error: err}
}

func updateProxyCache(w http.ResponseWriter, pkg Package) error {
	log.Println("Updating proxy cache for: " + pkg.Name)
	url := Config.PypiMirror + "/simple/" + pkg.Name + "/"

	result := getPage(url)

	if result.Error != nil {
		// We'll try next time
		return result.Error
	}

	finalizeCache(w, pkg, result.Page)
	return nil
}

func finalizeCache(w http.ResponseWriter, pkg Package, data []byte) {
	log.Println("Finalizing cache for: " + pkg.Name)
	returnData := string(data)
	// Replace the local package links with links to a local proxy
	// so we can cache that result as well.
	packageUris := rePackageUrl.FindAllSubmatch(data, -1)
	for _, line := range packageUris {
		uri := line[1]
		filename := line[2]
		// TODO: This most certainly has edge cases that aren't addressed here.
		// Specifically: ".tar.gz" can't be the only filetype uploaded...
		versionSplit := strings.Split(string(uri), "-")
		almostVersion := versionSplit[len(versionSplit)-1]
		version := strings.Replace(almostVersion, ".tar.gz", "", -1)
		quoteduri := url.QueryEscape(Config.PypiMirror + "/a/b/" + string(uri))
		replaceuri := "/fondu/cached-file/" + string(filename) + "?package=" + pkg.Name + "&release=" + version + "&original=" + quoteduri + "&name=" + url.QueryEscape(string(filename))
		returnData = strings.Replace(returnData, string(uri), replaceuri, -1)
	}

	// Replace the download links with links to a local proxy so that
	// we can cache the downloads as well.
	downloadUrls := reDownloadUrl.FindAllSubmatch(data, -1)
	for _, line := range downloadUrls {
		uri := line[1]
		version := line[2]
		filename := pkg.Name + "-" + string(version) + ".tar.gz"
		quotedUri := url.QueryEscape(string(uri))
		replaceUri := "/fondu/cached-file/" + filename + "?package=" + pkg.Name + "&release=" + string(version) + "&original=" + quotedUri + "&name=" + url.QueryEscape(filename)
		returnData = strings.Replace(returnData, string(uri), replaceUri, -1)
	}

	pkg.SetProxy([]byte(returnData))
}

func renderProxy(w http.ResponseWriter, pkg Package) {
	log.Println("Rendering proxy for: " + pkg.Name)
	io.WriteString(w, string(pkg.ProxyData()))
}

func buildReleaseMap(pkg Package) []releaseView {
	releaseMap := []releaseView{}
	for _, rel := range pkg.Releases() {
		metadata, err := rel.Metadata()
		if err != nil {
			break
		}
		md5Json, err := metadata.Get("md5_digest").Array()
		if err != nil {
			continue
		}
		md5 := md5Json[0]

		releaseMap = append(releaseMap, releaseView{
			DownloadUrl: rel.DownloadUrl(),
			Version:     rel.Version,
			Md5:         md5.(string),
			Path:        rel.Path(),
		})
	}
	return releaseMap
}

func simpleHandler(w http.ResponseWriter, r *http.Request) {
	paths := strings.Split(r.URL.Path, "/")
	name := paths[len(paths)-2]
	pkg := Package{Name: name, DataDir: Config.DataDir}

	// The package is ours, so we serve it ourselves.
	if pkg.Exists() && !pkg.Proxied() {
		log.Print("Private package: " + name + ". Serving it.")
		releaseMap := buildReleaseMap(pkg)
		renderTemplate(w, "single", &releaseMap)
		return
	}

	// Public package, so just render the proxy
	if pkg.Proxied() {
		log.Print("Proxied package: " + name + ". Sending cached data.")
		go updateProxyCache(w, pkg)
		renderProxy(w, pkg)
		return
	}

	if err := updateProxyCache(w, pkg); err != nil {
		http.Error(w, "Unable to update proxy", http.StatusInternalServerError)
		return
	}

	renderProxy(w, pkg)
}
<commit_msg>Not using the ResponseWriter when updating cache<commit_after>package main

import (
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"strings"
)

type downloadResult struct {
	Page  []byte
	Error error
}

type releaseView struct {
	DownloadUrl string
	Version     string
	Md5         string
	Path        string
}

var rePackageUrl = regexp.MustCompile(`(?i)<a href=\"(?P<url>.+?)#md5=.+?\">(?P<filename>.+?)</a>`)
var reDownloadUrl = regexp.MustCompile(`(?i)<a href=\"(?P<url>.+?)\"\s+rel=\".+\">(?P<version>.+?) download_url</a>`)

func simpleIndexHandler(w http.ResponseWriter, r *http.Request) {
	renderTemplate(w, "index", nil)
}

func getPage(url string) downloadResult {
	log.Println("Downloading page: " + url)
	res, err := http.Get(url)
	if err != nil {
		return downloadResult{Page: []byte(""), Error: err}
	}
	page, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return downloadResult{Page: []byte(""), Error: err}
	}
	return downloadResult{Page: page, Error: err}
}

func updateProxyCache(pkg Package) error {
	log.Println("Updating proxy cache for: " + pkg.Name)
	url := Config.PypiMirror + "/simple/" + pkg.Name + "/"

	result := getPage(url)

	if result.Error != nil {
		// We'll try next time
		return result.Error
	}

	finalizeCache(pkg, result.Page)
	return nil
}

func finalizeCache(pkg Package, data []byte) {
	log.Println("Finalizing cache for: " + pkg.Name)
	returnData := string(data)
	// Replace the local package links with links to a local proxy
	// so we can cache that result as well.
	packageUris := rePackageUrl.FindAllSubmatch(data, -1)
	for _, line := range packageUris {
		uri := line[1]
		filename := line[2]
		// TODO: This most certainly has edge cases that aren't addressed here.
		// Specifically: ".tar.gz" can't be the only filetype uploaded...
		versionSplit := strings.Split(string(uri), "-")
		almostVersion := versionSplit[len(versionSplit)-1]
		version := strings.Replace(almostVersion, ".tar.gz", "", -1)
		quoteduri := url.QueryEscape(Config.PypiMirror + "/a/b/" + string(uri))
		replaceuri := "/fondu/cached-file/" + string(filename) + "?package=" + pkg.Name + "&release=" + version + "&original=" + quoteduri + "&name=" + url.QueryEscape(string(filename))
		returnData = strings.Replace(returnData, string(uri), replaceuri, -1)
	}

	// Replace the download links with links to a local proxy so that
	// we can cache the downloads as well.
	downloadUrls := reDownloadUrl.FindAllSubmatch(data, -1)
	for _, line := range downloadUrls {
		uri := line[1]
		version := line[2]
		filename := pkg.Name + "-" + string(version) + ".tar.gz"
		quotedUri := url.QueryEscape(string(uri))
		replaceUri := "/fondu/cached-file/" + filename + "?package=" + pkg.Name + "&release=" + string(version) + "&original=" + quotedUri + "&name=" + url.QueryEscape(filename)
		returnData = strings.Replace(returnData, string(uri), replaceUri, -1)
	}

	pkg.SetProxy([]byte(returnData))
}

func renderProxy(w http.ResponseWriter, pkg Package) {
	log.Println("Rendering proxy for: " + pkg.Name)
	io.WriteString(w, string(pkg.ProxyData()))
}

func buildReleaseMap(pkg Package) []releaseView {
	releaseMap := []releaseView{}
	for _, rel := range pkg.Releases() {
		metadata, err := rel.Metadata()
		if err != nil {
			break
		}
		md5Json, err := metadata.Get("md5_digest").Array()
		if err != nil {
			continue
		}
		md5 := md5Json[0]

		releaseMap = append(releaseMap, releaseView{
			DownloadUrl: rel.DownloadUrl(),
			Version:     rel.Version,
			Md5:         md5.(string),
			Path:        rel.Path(),
		})
	}
	return releaseMap
}

func simpleHandler(w http.ResponseWriter, r *http.Request) {
	paths := strings.Split(r.URL.Path, "/")
	name := paths[len(paths)-2]
	pkg := Package{Name: name, DataDir: Config.DataDir}

	// The package is ours, so we serve it ourselves.
	if pkg.Exists() && !pkg.Proxied() {
		log.Print("Private package: " + name + ". Serving it.")
		releaseMap := buildReleaseMap(pkg)
		renderTemplate(w, "single", &releaseMap)
		return
	}

	// Public package, so just render the proxy
	if pkg.Proxied() {
		log.Print("Proxied package: " + name + ". Sending cached data.")
		go updateProxyCache(pkg)
		renderProxy(w, pkg)
		return
	}

	if err := updateProxyCache(pkg); err != nil {
		http.Error(w, "Unable to update proxy", http.StatusInternalServerError)
		return
	}

	renderProxy(w, pkg)
}
<|endoftext|>


<commit_before>package nginx

import (
	"testing"

	"github.com/topicai/candy"
	"gopkg.in/yaml.v2"

	"github.com/k8sp/auto-install/config"
	"github.com/stretchr/testify/assert"
)

const (
	nginxConf = `
events {

}

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    access_log  /var/log/nginx/access.log;

    sendfile        on;

    server {
        listen       80;
        server_name  localhost;
        autoindex on;
        autoindex_exact_size off;
        autoindex_localtime on;

        location / {
            root   /usr/share/nginx/html;
            index  index.html index.htm;
        }

        location /cloud-config/ {
            proxy_pass   http://172.17.0.2:8080;
        }

    }
}
`
)

func TestNginxConf(t *testing.T) {
	c := &config.Cluster{}
	candy.Must(yaml.Unmarshal([]byte(config.ExampleYAML), c))
	assert.Equal(t, nginxConf, Conf("", c))
}
<commit_msg>Correct gold result in nginx_conf_test.go<commit_after>package nginx

import (
	"testing"

	"github.com/topicai/candy"
	"gopkg.in/yaml.v2"

	"github.com/k8sp/auto-install/config"
	"github.com/stretchr/testify/assert"
)

const (
	nginxConf = `
events {

}

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    access_log  /var/log/nginx/access.log;

    sendfile        on;

    server {
        listen       80;
        server_name  localhost;
        autoindex on;
        autoindex_exact_size off;
        autoindex_localtime on;

        location / {
            root   /usr/share/nginx/html;
            index  index.html index.htm;
        }

        location /cloud-config/ {
            proxy_pass   http://10.0.2.15:8080;
        }

    }
}
`
)

func TestNginxConf(t *testing.T) {
	c := &config.Cluster{}
	candy.Must(yaml.Unmarshal([]byte(config.ExampleYAML), c))
	assert.Equal(t, nginxConf, Conf("", c))
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

// UnusedProperties are properties previously set but currently unused.
type UnusedProperties struct {
	// Infra was used to store the proto's infra field value in order to reduce
	// the size of the proto being unmarshalled in projection queries.
	// Since moved to its own entity (see details.go) and backfilled for all builds.
	Infra []byte `gae:"infra_bytes,noindex"`
	// InputProperties was used to store the proto's input.properties field value
	// in order to reduce the size of the proto being unmarshalled in projection queries.
	// Since moved to its own entity (see details.go) and backfilled for all builds.
	InputProperties []byte `gae:"input_properties_bytes,noindex"`
	// SwarmingTaskKey was used before Swarming task creation was made idempotent
	// in order to ensure only one task could call UpdateBuild.
	SwarmingTaskKey string `gae:"swarming_task_key,noindex"`
	// PubSubCallback is normally a struct (see build.go), which translates into datastore
	// fields pubsub_callback.auth_token, pubsub_callback.topic, pubsub_callback.user_data
	// with no actual field called pubsub_callback. However, nil values in the datastore
	// may exist for pubsub_callback (should instead be represented by having all three
	// pubsub_callback.* fields nil, but isn't). Capture such nil values here.
	// TODO(crbug/1042991): Support this case properly in gae datastore package.
	PubSubCallback []byte `gae:"pubsub_callback,noindex"`
}
<commit_msg>[buildbucket] Ignore unused canary_preference<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

// UnusedProperties are properties previously set but currently unused.
type UnusedProperties struct {
	// CanaryPreference was used to indicate if the build may use canary infrastructure.
	// Since replaced by the proto's canary field.
	CanaryPreference int32 `gae:"canary_preference,noindex"`
	// Infra was used to store the proto's infra field value in order to reduce
	// the size of the proto being unmarshalled in projection queries.
	// Since moved to its own entity (see details.go) and backfilled for all builds.
	Infra []byte `gae:"infra_bytes,noindex"`
	// InputProperties was used to store the proto's input.properties field value
	// in order to reduce the size of the proto being unmarshalled in projection queries.
	// Since moved to its own entity (see details.go) and backfilled for all builds.
	InputProperties []byte `gae:"input_properties_bytes,noindex"`
	// SwarmingTaskKey was used before Swarming task creation was made idempotent
	// in order to ensure only one task could call UpdateBuild.
	SwarmingTaskKey string `gae:"swarming_task_key,noindex"`
	// PubSubCallback is normally a struct (see build.go), which translates into datastore
	// fields pubsub_callback.auth_token, pubsub_callback.topic, pubsub_callback.user_data
	// with no actual field called pubsub_callback. However, nil values in the datastore
	// may exist for pubsub_callback (should instead be represented by having all three
	// pubsub_callback.* fields nil, but isn't). Capture such nil values here.
	// TODO(crbug/1042991): Support this case properly in gae datastore package.
	PubSubCallback []byte `gae:"pubsub_callback,noindex"`
}
<|endoftext|>


<commit_before>package chroot

import (
	"github.com/mitchellh/packer/packer"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"
)

// Communicator is a special communicator that works by executing
// commands locally but within a chroot.
type Communicator struct {
	Chroot        string
	ChrootCommand string
	CopyCommand   string
}

func (c *Communicator) Start(cmd *packer.RemoteCmd) error {

	chrootCommand := fmt.Sprintf("%s %s %s", c.ChrootCommand, c.Chroot, cmd.Command)
	localcmd := exec.Command("/bin/sh", "-c", chrootCommand)
	localCmd.Stdin = cmd.Stdin
	localCmd.Stdout = cmd.Stdout
	localCmd.Stderr = cmd.Stderr
	log.Printf("Executing: %s %#v", localCmd.Path, localCmd.Args)
	if err := localCmd.Start(); err != nil {
		return err
	}

	go func() {
		exitStatus := 0
		if err := localCmd.Wait(); err != nil {
			if exitErr, ok := err.(*exec.ExitError); ok {
				exitStatus = 1

				// There is no process-independent way to get the REAL
				// exit status so we just try to go deeper.
				if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
					exitStatus = status.ExitStatus()
				}
			}
		}

		log.Printf(
			"Chroot executation ended with '%d': '%s'",
			exitStatus, cmd.Command)
		cmd.SetExited(exitStatus)
	}()

	return nil
}

func (c *Communicator) UploadDir(dst string, src string, exclude []string) error {
	walkFn := func(fullPath string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		path, err := filepath.Rel(src, fullPath)
		if err != nil {
			return err
		}

		for _, e := range exclude {
			if e == path {
				log.Printf("Skipping excluded file: %s", path)
				return nil
			}
		}

		dstPath := filepath.Join(dst, path)
		dst := filepath.Join(c.Chroot, dst)
		log.Printf("Uploading to chroot dir: %s", dst)
		return copySingle(dst, fullPath, c.CopyCommand)
	}

	log.Printf("Uploading directory '%s' to '%s'", src, dst)
	return filepath.Walk(src, walkFn)
}

func (c *Communicator) Upload(dst string, r io.Reader) error {
	dst = filepath.Join(c.Chroot, dst)
	log.Printf("Uploading to chroot dir: %s", dst)
	f, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer f.Close()

	if _, err := io.Copy(f, r); err != nil {
		return err
	}

	return nil
}

func (c *Communicator) Download(src string, w io.Writer) error {
	src = filepath.Join(c.Chroot, src)
	log.Printf("Downloading from chroot dir: %s", src)
	f, err := os.Open(src)
	if err != nil {
		return err
	}
	defer f.Close()

	if _, err := io.Copy(w, f); err != nil {
		return err
	}

	return nil
}
<commit_msg>Revert "notes/reorg."<commit_after>package chroot

import (
	"github.com/mitchellh/packer/packer"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"
)

// Communicator is a special communicator that works by executing
// commands locally but within a chroot.
type Communicator struct {
	Chroot        string
	ChrootCommand string
	CopyCommand   string
}

func (c *Communicator) Start(cmd *packer.RemoteCmd) error {

	chrootCommand := fmt.Sprintf("%s %s %s", c.ChrootCommand, c.Chroot, cmd.Command)
	localcmd := exec.Command("/bin/sh", "-c", chrootCommand)
	localCmd.Stdin = cmd.Stdin
	localCmd.Stdout = cmd.Stdout
	localCmd.Stderr = cmd.Stderr
	log.Printf("Executing: %s %#v", localCmd.Path, localCmd.Args)
	if err := localCmd.Start(); err != nil {
		return err
	}

	go func() {
		exitStatus := 0
		if err := localCmd.Wait(); err != nil {
			if exitErr, ok := err.(*exec.ExitError); ok {
				exitStatus = 1

				// There is no process-independent way to get the REAL
				// exit status so we just try to go deeper.
				if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
					exitStatus = status.ExitStatus()
				}
			}
		}

		log.Printf(
			"Chroot executation ended with '%d': '%s'",
			exitStatus, cmd.Command)
		cmd.SetExited(exitStatus)
	}()

	return nil
}

func (c *Communicator) Upload(dst string, r io.Reader) error {
	dst = filepath.Join(c.Chroot, dst)
	log.Printf("Uploading to chroot dir: %s", dst)
	f, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer f.Close()

	if _, err := io.Copy(f, r); err != nil {
		return err
	}

	return nil
}

func (c *Communicator) UploadDir(dst string, src string, exclude []string) error {
	walkFn := func(fullPath string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		path, err := filepath.Rel(src, fullPath)
		if err != nil {
			return err
		}

		for _, e := range exclude {
			if e == path {
				log.Printf("Skipping excluded file: %s", path)
				return nil
			}
		}

		dstPath := filepath.Join(dst, path)
		dst := filepath.Join(c.Chroot, dst)
		log.Printf("Uploading to chroot dir: %s", dst)
		return copySingle(dst, fullPath, c.CopyCommand)
	}

	log.Printf("Uploading directory '%s' to '%s'", src, dst)
	return filepath.Walk(src, walkFn)
}

func (c *Communicator) Download(src string, w io.Writer) error {
	src = filepath.Join(c.Chroot, src)
	log.Printf("Downloading from chroot dir: %s", src)
	f, err := os.Open(src)
	if err != nil {
		return err
	}
	defer f.Close()

	if _, err := io.Copy(w, f); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// +build !windows

package buildkit

import (
	"os"
	"path/filepath"
	"strconv"
	"sync"

	"github.com/docker/docker/daemon/config"
	"github.com/docker/docker/pkg/idtools"
	"github.com/docker/docker/pkg/stringid"
	"github.com/docker/libnetwork"
	"github.com/moby/buildkit/executor"
	"github.com/moby/buildkit/executor/oci"
	"github.com/moby/buildkit/executor/runcexecutor"
	"github.com/moby/buildkit/identity"
	"github.com/moby/buildkit/solver/pb"
	"github.com/moby/buildkit/util/network"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/sirupsen/logrus"
)

const networkName = "bridge"

func newExecutor(root, cgroupParent string, net libnetwork.NetworkController, dnsConfig *oci.DNSConfig, rootless bool, idmap *idtools.IdentityMapping) (executor.Executor, error) {
	networkProviders := map[pb.NetMode]network.Provider{
		pb.NetMode_UNSET: &bridgeProvider{NetworkController: net, Root: filepath.Join(root, "net")},
		pb.NetMode_HOST:  network.NewHostProvider(),
		pb.NetMode_NONE:  network.NewNoneProvider(),
	}
	return runcexecutor.New(runcexecutor.Opt{
		Root:                filepath.Join(root, "executor"),
		CommandCandidates:   []string{"runc"},
		DefaultCgroupParent: cgroupParent,
		Rootless:            rootless,
		NoPivot:             os.Getenv("DOCKER_RAMDISK") != "",
		IdentityMapping:     idmap,
		DNS:                 dnsConfig,
	}, networkProviders)
}

type bridgeProvider struct {
	libnetwork.NetworkController
	Root string
}

func (p *bridgeProvider) New() (network.Namespace, error) {
	n, err := p.NetworkByName(networkName)
	if err != nil {
		return nil, err
	}

	iface := &lnInterface{ready: make(chan struct{}), provider: p}
	iface.Once.Do(func() {
		go iface.init(p.NetworkController, n)
	})

	return iface, nil
}

type lnInterface struct {
	ep  libnetwork.Endpoint
	sbx libnetwork.Sandbox
	sync.Once
	err      error
	ready    chan struct{}
	provider *bridgeProvider
}

func (iface *lnInterface) init(c libnetwork.NetworkController, n libnetwork.Network) {
	defer close(iface.ready)
	id := identity.NewID()

	ep, err := n.CreateEndpoint(id, libnetwork.CreateOptionDisableResolution())
	if err != nil {
		iface.err = err
		return
	}

	sbx, err := c.NewSandbox(id, libnetwork.OptionUseExternalKey(), libnetwork.OptionHostsPath(filepath.Join(iface.provider.Root, id, "hosts")),
		libnetwork.OptionResolvConfPath(filepath.Join(iface.provider.Root, id, "resolv.conf")))
	if err != nil {
		iface.err = err
		return
	}

	if err := ep.Join(sbx); err != nil {
		iface.err = err
		return
	}

	iface.sbx = sbx
	iface.ep = ep
}

func (iface *lnInterface) Set(s *specs.Spec) error {
	<-iface.ready
	if iface.err != nil {
		logrus.WithError(iface.err).Error("failed to set networking spec")
		return iface.err
	}
	shortNetCtlrID := stringid.TruncateID(iface.provider.NetworkController.ID())
	// attach netns to bridge within the container namespace, using reexec in a prestart hook
	s.Hooks = &specs.Hooks{
		Prestart: []specs.Hook{{
			Path: filepath.Join("/proc", strconv.Itoa(os.Getpid()), "exe"),
			Args: []string{"libnetwork-setkey", "-exec-root=" + iface.provider.Config().Daemon.ExecRoot, iface.sbx.ContainerID(), shortNetCtlrID},
		}},
	}
	return nil
}

func (iface *lnInterface) Close() error {
	<-iface.ready
	if iface.sbx != nil {
		go func() {
			if err := iface.sbx.Delete(); err != nil {
				logrus.Errorf("failed to delete builder network sandbox: %v", err)
			}
		}()
	}
	return iface.err
}

func getDNSConfig(cfg config.DNSConfig) *oci.DNSConfig {
	if cfg.DNS != nil || cfg.DNSSearch != nil || cfg.DNSOptions != nil {
		return &oci.DNSConfig{
			Nameservers:   cfg.DNS,
			SearchDomains: cfg.DNSSearch,
			Options:       cfg.DNSOptions,
		}
	}
	return nil
}
<commit_msg>builder: ensure libnetwork state file do not leak<commit_after>// +build !windows

package buildkit

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"sync"

	"github.com/docker/docker/daemon/config"
	"github.com/docker/docker/pkg/idtools"
	"github.com/docker/docker/pkg/stringid"
	"github.com/docker/libnetwork"
	"github.com/moby/buildkit/executor"
	"github.com/moby/buildkit/executor/oci"
	"github.com/moby/buildkit/executor/runcexecutor"
	"github.com/moby/buildkit/identity"
	"github.com/moby/buildkit/solver/pb"
	"github.com/moby/buildkit/util/network"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/sirupsen/logrus"
)

const networkName = "bridge"

func newExecutor(root, cgroupParent string, net libnetwork.NetworkController, dnsConfig *oci.DNSConfig, rootless bool, idmap *idtools.IdentityMapping) (executor.Executor, error) {
	netRoot := filepath.Join(root, "net")
	networkProviders := map[pb.NetMode]network.Provider{
		pb.NetMode_UNSET: &bridgeProvider{NetworkController: net, Root: netRoot},
		pb.NetMode_HOST:  network.NewHostProvider(),
		pb.NetMode_NONE:  network.NewNoneProvider(),
	}

	// make sure net state directory is cleared from previous state
	fis, err := ioutil.ReadDir(netRoot)
	if err == nil {
		for _, fi := range fis {
			fp := filepath.Join(netRoot, fi.Name())
			if err := os.RemoveAll(fp); err != nil {
				logrus.WithError(err).Errorf("failed to delete old network state: %v", fp)
			}
		}
	}

	return runcexecutor.New(runcexecutor.Opt{
		Root:                filepath.Join(root, "executor"),
		CommandCandidates:   []string{"runc"},
		DefaultCgroupParent: cgroupParent,
		Rootless:            rootless,
		NoPivot:             os.Getenv("DOCKER_RAMDISK") != "",
		IdentityMapping:     idmap,
		DNS:                 dnsConfig,
	}, networkProviders)
}

type bridgeProvider struct {
	libnetwork.NetworkController
	Root string
}

func (p *bridgeProvider) New() (network.Namespace, error) {
	n, err := p.NetworkByName(networkName)
	if err != nil {
		return nil, err
	}

	iface := &lnInterface{ready: make(chan struct{}), provider: p}
	iface.Once.Do(func() {
		go iface.init(p.NetworkController, n)
	})

	return iface, nil
}

type lnInterface struct {
	ep  libnetwork.Endpoint
	sbx libnetwork.Sandbox
	sync.Once
	err      error
	ready    chan struct{}
	provider *bridgeProvider
}

func (iface *lnInterface) init(c libnetwork.NetworkController, n libnetwork.Network) {
	defer close(iface.ready)
	id := identity.NewID()

	ep, err := n.CreateEndpoint(id, libnetwork.CreateOptionDisableResolution())
	if err != nil {
		iface.err = err
		return
	}

	sbx, err := c.NewSandbox(id, libnetwork.OptionUseExternalKey(), libnetwork.OptionHostsPath(filepath.Join(iface.provider.Root, id, "hosts")),
		libnetwork.OptionResolvConfPath(filepath.Join(iface.provider.Root, id, "resolv.conf")))
	if err != nil {
		iface.err = err
		return
	}

	if err := ep.Join(sbx); err != nil {
		iface.err = err
		return
	}

	iface.sbx = sbx
	iface.ep = ep
}

func (iface *lnInterface) Set(s *specs.Spec) error {
	<-iface.ready
	if iface.err != nil {
		logrus.WithError(iface.err).Error("failed to set networking spec")
		return iface.err
	}
	shortNetCtlrID := stringid.TruncateID(iface.provider.NetworkController.ID())
	// attach netns to bridge within the container namespace, using reexec in a prestart hook
	s.Hooks = &specs.Hooks{
		Prestart: []specs.Hook{{
			Path: filepath.Join("/proc", strconv.Itoa(os.Getpid()), "exe"),
			Args: []string{"libnetwork-setkey", "-exec-root=" + iface.provider.Config().Daemon.ExecRoot, iface.sbx.ContainerID(), shortNetCtlrID},
		}},
	}
	return nil
}

func (iface *lnInterface) Close() error {
	<-iface.ready
	if iface.sbx != nil {
		go func() {
			if err := iface.sbx.Delete(); err != nil {
				logrus.WithError(err).Errorf("failed to delete builder network sandbox")
			}
			if err := os.RemoveAll(filepath.Join(iface.provider.Root, iface.sbx.ContainerID())); err != nil {
				logrus.WithError(err).Errorf("failed to delete builder sandbox directory")
			}
		}()
	}
	return iface.err
}

func getDNSConfig(cfg config.DNSConfig) *oci.DNSConfig {
	if cfg.DNS != nil || cfg.DNSSearch != nil || cfg.DNSOptions != nil {
		return &oci.DNSConfig{
			Nameservers:   cfg.DNS,
			SearchDomains: cfg.DNSSearch,
			Options:       cfg.DNSOptions,
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012-present Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.

package elastic

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strings"

	"github.com/olivere/elastic/v7/uritemplates"
)

// BulkService allows for batching bulk requests and sending them to
// Elasticsearch in one roundtrip. Use the Add method with BulkIndexRequest,
// BulkUpdateRequest, and BulkDeleteRequest to add bulk requests to a batch,
// then use Do to send them to Elasticsearch.
//
// BulkService will be reset after each Do call. In other words, you can
// reuse BulkService to send many batches. You do not have to create a new
// BulkService for each batch.
//
// See https://www.elastic.co/guide/en/elasticsearch/reference/7.0/docs-bulk.html
// for more details.
type BulkService struct {
	client  *Client
	retrier Retrier

	pretty     *bool       // pretty format the returned JSON response
	human      *bool       // return human readable values for statistics
	errorTrace *bool       // include the stack trace of returned errors
	filterPath []string    // list of filters used to reduce the response
	headers    http.Header // custom request-level HTTP headers

	index               string
	typ                 string
	requests            []BulkableRequest
	pipeline            string
	timeout             string
	refresh             string
	routing             string
	waitForActiveShards string

	// estimated bulk size in bytes, up to the request index sizeInBytesCursor
	sizeInBytes       int64
	sizeInBytesCursor int
}

// NewBulkService initializes a new BulkService.
func NewBulkService(client *Client) *BulkService {
	builder := &BulkService{
		client: client,
	}
	return builder
}

// Pretty tells Elasticsearch whether to return a formatted JSON response.
func (s *BulkService) Pretty(pretty bool) *BulkService {
	s.pretty = &pretty
	return s
}

// Human specifies whether human readable values should be returned in
// the JSON response, e.g. "7.5mb".
func (s *BulkService) Human(human bool) *BulkService {
	s.human = &human
	return s
}

// ErrorTrace specifies whether to include the stack trace of returned errors.
func (s *BulkService) ErrorTrace(errorTrace bool) *BulkService {
	s.errorTrace = &errorTrace
	return s
}

// FilterPath specifies a list of filters used to reduce the response.
func (s *BulkService) FilterPath(filterPath ...string) *BulkService {
	s.filterPath = filterPath
	return s
}

// Header adds a header to the request.
func (s *BulkService) Header(name string, value string) *BulkService {
	if s.headers == nil {
		s.headers = http.Header{}
	}
	s.headers.Add(name, value)
	return s
}

// Headers specifies the headers of the request.
func (s *BulkService) Headers(headers http.Header) *BulkService {
	s.headers = headers
	return s
}

// Reset cleans up the request queue
func (s *BulkService) Reset() {
	s.requests = make([]BulkableRequest, 0)
	s.sizeInBytes = 0
	s.sizeInBytesCursor = 0
}

// Retrier allows to set specific retry logic for this BulkService.
// If not specified, it will use the client's default retrier.
func (s *BulkService) Retrier(retrier Retrier) *BulkService {
	s.retrier = retrier
	return s
}

// Index specifies the index to use for all batches. You may also leave
// this blank and specify the index in the individual bulk requests.
func (s *BulkService) Index(index string) *BulkService {
	s.index = index
	return s
}

// Type specifies the type to use for all batches. You may also leave
// this blank and specify the type in the individual bulk requests.
func (s *BulkService) Type(typ string) *BulkService {
	s.typ = typ
	return s
}

// Timeout is a global timeout for processing bulk requests. This is a
// server-side timeout, i.e. it tells Elasticsearch the time after which
// it should stop processing.
func (s *BulkService) Timeout(timeout string) *BulkService {
	s.timeout = timeout
	return s
}

// Refresh controls when changes made by this request are made visible
// to search. The allowed values are: "true" (refresh the relevant
// primary and replica shards immediately), "wait_for" (wait for the
// changes to be made visible by a refresh before reying), or "false"
// (no refresh related actions). The default value is "false".
//
// See https://www.elastic.co/guide/en/elasticsearch/reference/7.0/docs-refresh.html
// for details.
func (s *BulkService) Refresh(refresh string) *BulkService {
	s.refresh = refresh
	return s
}

// Routing specifies the routing value.
func (s *BulkService) Routing(routing string) *BulkService {
	s.routing = routing
	return s
}

// Pipeline specifies the pipeline id to preprocess incoming documents with.
func (s *BulkService) Pipeline(pipeline string) *BulkService {
	s.pipeline = pipeline
	return s
}

// WaitForActiveShards sets the number of shard copies that must be active
// before proceeding with the bulk operation. Defaults to 1, meaning the
// primary shard only. Set to `all` for all shard copies, otherwise set to
// any non-negative value less than or equal to the total number of copies
// for the shard (number of replicas + 1).
func (s *BulkService) WaitForActiveShards(waitForActiveShards string) *BulkService {
	s.waitForActiveShards = waitForActiveShards
	return s
}

// Add adds bulkable requests, i.e. BulkIndexRequest, BulkUpdateRequest,
// and/or BulkDeleteRequest.
func (s *BulkService) Add(requests ...BulkableRequest) *BulkService {
	s.requests = append(s.requests, requests...)
	return s
}

// EstimatedSizeInBytes returns the estimated size of all bulkable
// requests added via Add.
func (s *BulkService) EstimatedSizeInBytes() int64 {
	if s.sizeInBytesCursor == len(s.requests) {
		return s.sizeInBytes
	}
	for _, r := range s.requests[s.sizeInBytesCursor:] {
		s.sizeInBytes += s.estimateSizeInBytes(r)
		s.sizeInBytesCursor++
	}
	return s.sizeInBytes
}

// estimateSizeInBytes returns the estimates size of the given
// bulkable request, i.e. BulkIndexRequest, BulkUpdateRequest, and
// BulkDeleteRequest.
func (s *BulkService) estimateSizeInBytes(r BulkableRequest) int64 {
	lines, _ := r.Source()
	size := 0
	for _, line := range lines {
		// +1 for the \n
		size += len(line) + 1
	}
	return int64(size)
}

// NumberOfActions returns the number of bulkable requests that need to
// be sent to Elasticsearch on the next batch.
func (s *BulkService) NumberOfActions() int {
	return len(s.requests)
}

func (s *BulkService) bodyAsString() (string, error) {
	// Pre-allocate to reduce allocs
	var buf strings.Builder
	buf.Grow(int(s.EstimatedSizeInBytes()))

	for _, req := range s.requests {
		source, err := req.Source()
		if err != nil {
			return "", err
		}
		for _, line := range source {
			buf.WriteString(line)
			buf.WriteByte('\n')
		}
	}

	return buf.String(), nil
}

// Do sends the batched requests to Elasticsearch. Note that, when successful,
// you can reuse the BulkService for the next batch as the list of bulk
// requests is cleared on success.
func (s *BulkService) Do(ctx context.Context) (*BulkResponse, error) {
	// No actions?
	if s.NumberOfActions() == 0 {
		return nil, errors.New("elastic: No bulk actions to commit")
	}

	// Get body
	body, err := s.bodyAsString()
	if err != nil {
		return nil, err
	}

	// Build url
	path := "/"
	if len(s.index) > 0 {
		index, err := uritemplates.Expand("{index}", map[string]string{
			"index": s.index,
		})
		if err != nil {
			return nil, err
		}
		path += index + "/"
	}
	if len(s.typ) > 0 {
		typ, err := uritemplates.Expand("{type}", map[string]string{
			"type": s.typ,
		})
		if err != nil {
			return nil, err
		}
		path += typ + "/"
	}
	path += "_bulk"

	// Parameters
	params := url.Values{}
	if v := s.pretty; v != nil {
		params.Set("pretty", fmt.Sprint(*v))
	}
	if v := s.human; v != nil {
		params.Set("human", fmt.Sprint(*v))
	}
	if v := s.errorTrace; v != nil {
		params.Set("error_trace", fmt.Sprint(*v))
	}
	if len(s.filterPath) > 0 {
		params.Set("filter_path", strings.Join(s.filterPath, ","))
	}
	if s.pipeline != "" {
		params.Set("pipeline", s.pipeline)
	}
	if s.refresh != "" {
		params.Set("refresh", s.refresh)
	}
	if s.routing != "" {
		params.Set("routing", s.routing)
	}
	if s.timeout != "" {
		params.Set("timeout", s.timeout)
	}
	if s.waitForActiveShards != "" {
		params.Set("wait_for_active_shards", s.waitForActiveShards)
	}

	// Get response
	res, err := s.client.PerformRequest(ctx, PerformRequestOptions{
		Method:      "POST",
		Path:        path,
		Params:      params,
		Body:        body,
		ContentType: "application/x-ndjson",
		Retrier:     s.retrier,
		Headers:     s.headers,
	})
	if err != nil {
		return nil, err
	}

	// Return results
	ret := new(BulkResponse)
	if err := s.client.decoder.Decode(res.Body, ret); err != nil {
		return nil, err
	}

	// Reset so the request can be reused
	s.Reset()

	return ret, nil
}

// BulkResponse is a response to a bulk execution.
//
// Example:
// {
//   "took":3,
//   "errors":false,
//   "items":[{
//     "index":{
//       "_index":"index1",
//       "_type":"tweet",
//       "_id":"1",
//       "_version":3,
//       "status":201
//     }
//   },{
//     "index":{
//       "_index":"index2",
//       "_type":"tweet",
//       "_id":"2",
//       "_version":3,
//       "status":200
//     }
//   },{
//     "delete":{
//       "_index":"index1",
//       "_type":"tweet",
//       "_id":"1",
//       "_version":4,
//       "status":200,
//       "found":true
//     }
//   },{
//     "update":{
//       "_index":"index2",
//       "_type":"tweet",
//       "_id":"2",
//       "_version":4,
//       "status":200
//     }
//   }]
// }
type BulkResponse struct {
	Took   int                            `json:"took,omitempty"`
	Errors bool                           `json:"errors,omitempty"`
	Items  []map[string]*BulkResponseItem `json:"items,omitempty"`
}

// BulkResponseItem is the result of a single bulk request.
type BulkResponseItem struct {
	Index         string        `json:"_index,omitempty"`
	Type          string        `json:"_type,omitempty"`
	Id            string        `json:"_id,omitempty"`
	Version       int64         `json:"_version,omitempty"`
	Result        string        `json:"result,omitempty"`
	Shards        *ShardsInfo   `json:"_shards,omitempty"`
	SeqNo         int64         `json:"_seq_no,omitempty"`
	PrimaryTerm   int64         `json:"_primary_term,omitempty"`
	Status        int           `json:"status,omitempty"`
	ForcedRefresh bool          `json:"forced_refresh,omitempty"`
	Error         *ErrorDetails `json:"error,omitempty"`
	GetResult     *GetResult    `json:"get,omitempty"`
}

// Indexed returns all bulk request results of "index" actions.
func (r *BulkResponse) Indexed() []*BulkResponseItem {
	return r.ByAction("index")
}

// Created returns all bulk request results of "create" actions.
func (r *BulkResponse) Created() []*BulkResponseItem {
	return r.ByAction("create")
}

// Updated returns all bulk request results of "update" actions.
func (r *BulkResponse) Updated() []*BulkResponseItem {
	return r.ByAction("update")
}

// Deleted returns all bulk request results of "delete" actions.
func (r *BulkResponse) Deleted() []*BulkResponseItem {
	return r.ByAction("delete")
}

// ByAction returns all bulk request results of a certain action,
// e.g. "index" or "delete".
func (r *BulkResponse) ByAction(action string) []*BulkResponseItem {
	if r.Items == nil {
		return nil
	}
	var items []*BulkResponseItem
	for _, item := range r.Items {
		if result, found := item[action]; found {
			items = append(items, result)
		}
	}
	return items
}

// ById returns all bulk request results of a given document id,
// regardless of the action ("index", "delete" etc.).
func (r *BulkResponse) ById(id string) []*BulkResponseItem {
	if r.Items == nil {
		return nil
	}
	var items []*BulkResponseItem
	for _, item := range r.Items {
		for _, result := range item {
			if result.Id == id {
				items = append(items, result)
			}
		}
	}
	return items
}

// Failed returns those items of a bulk response that have errors,
// i.e. those that don't have a status code between 200 and 299.
func (r *BulkResponse) Failed() []*BulkResponseItem {
	if r.Items == nil {
		return nil
	}
	var errors []*BulkResponseItem
	for _, item := range r.Items {
		for _, result := range item {
			if !(result.Status >= 200 && result.Status <= 299) {
				errors = append(errors, result)
			}
		}
	}
	return errors
}

// Succeeded returns those items of a bulk response that have no errors,
// i.e. those have a status code between 200 and 299.
func (r *BulkResponse) Succeeded() []*BulkResponseItem {
	if r.Items == nil {
		return nil
	}
	var succeeded []*BulkResponseItem
	for _, item := range r.Items {
		for _, result := range item {
			if result.Status >= 200 && result.Status <= 299 {
				succeeded = append(succeeded, result)
			}
		}
	}
	return succeeded
}
<commit_msg>Fix typo<commit_after>// Copyright 2012-present Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.

package elastic

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strings"

	"github.com/olivere/elastic/v7/uritemplates"
)

// BulkService allows for batching bulk requests and sending them to
// Elasticsearch in one roundtrip. Use the Add method with BulkIndexRequest,
// BulkUpdateRequest, and BulkDeleteRequest to add bulk requests to a batch,
// then use Do to send them to Elasticsearch.
//
// BulkService will be reset after each Do call. In other words, you can
// reuse BulkService to send many batches. You do not have to create a new
// BulkService for each batch.
//
// See https://www.elastic.co/guide/en/elasticsearch/reference/7.0/docs-bulk.html
// for more details.
type BulkService struct {
	client  *Client
	retrier Retrier

	pretty     *bool       // pretty format the returned JSON response
	human      *bool       // return human readable values for statistics
	errorTrace *bool       // include the stack trace of returned errors
	filterPath []string    // list of filters used to reduce the response
	headers    http.Header // custom request-level HTTP headers

	index               string
	typ                 string
	requests            []BulkableRequest
	pipeline            string
	timeout             string
	refresh             string
	routing             string
	waitForActiveShards string

	// estimated bulk size in bytes, up to the request index sizeInBytesCursor
	sizeInBytes       int64
	sizeInBytesCursor int
}

// NewBulkService initializes a new BulkService.
func NewBulkService(client *Client) *BulkService {
	builder := &BulkService{
		client: client,
	}
	return builder
}

// Pretty tells Elasticsearch whether to return a formatted JSON response.
func (s *BulkService) Pretty(pretty bool) *BulkService {
	s.pretty = &pretty
	return s
}

// Human specifies whether human readable values should be returned in
// the JSON response, e.g. "7.5mb".
func (s *BulkService) Human(human bool) *BulkService {
	s.human = &human
	return s
}

// ErrorTrace specifies whether to include the stack trace of returned errors.
func (s *BulkService) ErrorTrace(errorTrace bool) *BulkService {
	s.errorTrace = &errorTrace
	return s
}

// FilterPath specifies a list of filters used to reduce the response.
func (s *BulkService) FilterPath(filterPath ...string) *BulkService {
	s.filterPath = filterPath
	return s
}

// Header adds a header to the request.
func (s *BulkService) Header(name string, value string) *BulkService {
	if s.headers == nil {
		s.headers = http.Header{}
	}
	s.headers.Add(name, value)
	return s
}

// Headers specifies the headers of the request.
func (s *BulkService) Headers(headers http.Header) *BulkService {
	s.headers = headers
	return s
}

// Reset cleans up the request queue
func (s *BulkService) Reset() {
	s.requests = make([]BulkableRequest, 0)
	s.sizeInBytes = 0
	s.sizeInBytesCursor = 0
}

// Retrier allows to set specific retry logic for this BulkService.
// If not specified, it will use the client's default retrier.
func (s *BulkService) Retrier(retrier Retrier) *BulkService {
	s.retrier = retrier
	return s
}

// Index specifies the index to use for all batches. You may also leave
// this blank and specify the index in the individual bulk requests.
func (s *BulkService) Index(index string) *BulkService {
	s.index = index
	return s
}

// Type specifies the type to use for all batches. You may also leave
// this blank and specify the type in the individual bulk requests.
func (s *BulkService) Type(typ string) *BulkService {
	s.typ = typ
	return s
}

// Timeout is a global timeout for processing bulk requests. This is a
// server-side timeout, i.e. it tells Elasticsearch the time after which
// it should stop processing.
func (s *BulkService) Timeout(timeout string) *BulkService {
	s.timeout = timeout
	return s
}

// Refresh controls when changes made by this request are made visible
// to search. The allowed values are: "true" (refresh the relevant
// primary and replica shards immediately), "wait_for" (wait for the
// changes to be made visible by a refresh before replying), or "false"
// (no refresh related actions). The default value is "false".
//
// See https://www.elastic.co/guide/en/elasticsearch/reference/7.0/docs-refresh.html
// for details.
func (s *BulkService) Refresh(refresh string) *BulkService {
	s.refresh = refresh
	return s
}

// Routing specifies the routing value.
func (s *BulkService) Routing(routing string) *BulkService {
	s.routing = routing
	return s
}

// Pipeline specifies the pipeline id to preprocess incoming documents with.
func (s *BulkService) Pipeline(pipeline string) *BulkService {
	s.pipeline = pipeline
	return s
}

// WaitForActiveShards sets the number of shard copies that must be active
// before proceeding with the bulk operation. Defaults to 1, meaning the
// primary shard only. Set to `all` for all shard copies, otherwise set to
// any non-negative value less than or equal to the total number of copies
// for the shard (number of replicas + 1).
func (s *BulkService) WaitForActiveShards(waitForActiveShards string) *BulkService {
	s.waitForActiveShards = waitForActiveShards
	return s
}

// Add adds bulkable requests, i.e. BulkIndexRequest, BulkUpdateRequest,
// and/or BulkDeleteRequest.
func (s *BulkService) Add(requests ...BulkableRequest) *BulkService {
	s.requests = append(s.requests, requests...)
	return s
}

// EstimatedSizeInBytes returns the estimated size of all bulkable
// requests added via Add.
func (s *BulkService) EstimatedSizeInBytes() int64 {
	if s.sizeInBytesCursor == len(s.requests) {
		return s.sizeInBytes
	}
	for _, r := range s.requests[s.sizeInBytesCursor:] {
		s.sizeInBytes += s.estimateSizeInBytes(r)
		s.sizeInBytesCursor++
	}
	return s.sizeInBytes
}

// estimateSizeInBytes returns the estimates size of the given
// bulkable request, i.e. BulkIndexRequest, BulkUpdateRequest, and
// BulkDeleteRequest.
func (s *BulkService) estimateSizeInBytes(r BulkableRequest) int64 {
	lines, _ := r.Source()
	size := 0
	for _, line := range lines {
		// +1 for the \n
		size += len(line) + 1
	}
	return int64(size)
}

// NumberOfActions returns the number of bulkable requests that need to
// be sent to Elasticsearch on the next batch.
func (s *BulkService) NumberOfActions() int {
	return len(s.requests)
}

func (s *BulkService) bodyAsString() (string, error) {
	// Pre-allocate to reduce allocs
	var buf strings.Builder
	buf.Grow(int(s.EstimatedSizeInBytes()))

	for _, req := range s.requests {
		source, err := req.Source()
		if err != nil {
			return "", err
		}
		for _, line := range source {
			buf.WriteString(line)
			buf.WriteByte('\n')
		}
	}

	return buf.String(), nil
}

// Do sends the batched requests to Elasticsearch. Note that, when successful,
// you can reuse the BulkService for the next batch as the list of bulk
// requests is cleared on success.
func (s *BulkService) Do(ctx context.Context) (*BulkResponse, error) {
	// No actions?
	if s.NumberOfActions() == 0 {
		return nil, errors.New("elastic: No bulk actions to commit")
	}

	// Get body
	body, err := s.bodyAsString()
	if err != nil {
		return nil, err
	}

	// Build url
	path := "/"
	if len(s.index) > 0 {
		index, err := uritemplates.Expand("{index}", map[string]string{
			"index": s.index,
		})
		if err != nil {
			return nil, err
		}
		path += index + "/"
	}
	if len(s.typ) > 0 {
		typ, err := uritemplates.Expand("{type}", map[string]string{
			"type": s.typ,
		})
		if err != nil {
			return nil, err
		}
		path += typ + "/"
	}
	path += "_bulk"

	// Parameters
	params := url.Values{}
	if v := s.pretty; v != nil {
		params.Set("pretty", fmt.Sprint(*v))
	}
	if v := s.human; v != nil {
		params.Set("human", fmt.Sprint(*v))
	}
	if v := s.errorTrace; v != nil {
		params.Set("error_trace", fmt.Sprint(*v))
	}
	if len(s.filterPath) > 0 {
		params.Set("filter_path", strings.Join(s.filterPath, ","))
	}
	if s.pipeline != "" {
		params.Set("pipeline", s.pipeline)
	}
	if s.refresh != "" {
		params.Set("refresh", s.refresh)
	}
	if s.routing != "" {
		params.Set("routing", s.routing)
	}
	if s.timeout != "" {
		params.Set("timeout", s.timeout)
	}
	if s.waitForActiveShards != "" {
		params.Set("wait_for_active_shards", s.waitForActiveShards)
	}

	// Get response
	res, err := s.client.PerformRequest(ctx, PerformRequestOptions{
		Method:      "POST",
		Path:        path,
		Params:      params,
		Body:        body,
		ContentType: "application/x-ndjson",
		Retrier:     s.retrier,
		Headers:     s.headers,
	})
	if err != nil {
		return nil, err
	}

	// Return results
	ret := new(BulkResponse)
	if err := s.client.decoder.Decode(res.Body, ret); err != nil {
		return nil, err
	}

	// Reset so the request can be reused
	s.Reset()

	return ret, nil
}

// BulkResponse is a response to a bulk execution.
//
// Example:
// {
//   "took":3,
//   "errors":false,
//   "items":[{
//     "index":{
//       "_index":"index1",
//       "_type":"tweet",
//       "_id":"1",
//       "_version":3,
//       "status":201
//     }
//   },{
//     "index":{
//       "_index":"index2",
//       "_type":"tweet",
//       "_id":"2",
//       "_version":3,
//       "status":200
//     }
//   },{
//     "delete":{
//       "_index":"index1",
//       "_type":"tweet",
//       "_id":"1",
//       "_version":4,
//       "status":200,
//       "found":true
//     }
//   },{
//     "update":{
//       "_index":"index2",
//       "_type":"tweet",
//       "_id":"2",
//       "_version":4,
//       "status":200
//     }
//   }]
// }
type BulkResponse struct {
	Took   int                            `json:"took,omitempty"`
	Errors bool                           `json:"errors,omitempty"`
	Items  []map[string]*BulkResponseItem `json:"items,omitempty"`
}

// BulkResponseItem is the result of a single bulk request.
type BulkResponseItem struct {
	Index         string        `json:"_index,omitempty"`
	Type          string        `json:"_type,omitempty"`
	Id            string        `json:"_id,omitempty"`
	Version       int64         `json:"_version,omitempty"`
	Result        string        `json:"result,omitempty"`
	Shards        *ShardsInfo   `json:"_shards,omitempty"`
	SeqNo         int64         `json:"_seq_no,omitempty"`
	PrimaryTerm   int64         `json:"_primary_term,omitempty"`
	Status        int           `json:"status,omitempty"`
	ForcedRefresh bool          `json:"forced_refresh,omitempty"`
	Error         *ErrorDetails `json:"error,omitempty"`
	GetResult     *GetResult    `json:"get,omitempty"`
}

// Indexed returns all bulk request results of "index" actions.
func (r *BulkResponse) Indexed() []*BulkResponseItem {
	return r.ByAction("index")
}

// Created returns all bulk request results of "create" actions.
func (r *BulkResponse) Created() []*BulkResponseItem {
	return r.ByAction("create")
}

// Updated returns all bulk request results of "update" actions.
func (r *BulkResponse) Updated() []*BulkResponseItem {
	return r.ByAction("update")
}

// Deleted returns all bulk request results of "delete" actions.
func (r *BulkResponse) Deleted() []*BulkResponseItem {
	return r.ByAction("delete")
}

// ByAction returns all bulk request results of a certain action,
// e.g. "index" or "delete".
func (r *BulkResponse) ByAction(action string) []*BulkResponseItem {
	if r.Items == nil {
		return nil
	}
	var items []*BulkResponseItem
	for _, item := range r.Items {
		if result, found := item[action]; found {
			items = append(items, result)
		}
	}
	return items
}

// ById returns all bulk request results of a given document id,
// regardless of the action ("index", "delete" etc.).
func (r *BulkResponse) ById(id string) []*BulkResponseItem {
	if r.Items == nil {
		return nil
	}
	var items []*BulkResponseItem
	for _, item := range r.Items {
		for _, result := range item {
			if result.Id == id {
				items = append(items, result)
			}
		}
	}
	return items
}

// Failed returns those items of a bulk response that have errors,
// i.e. those that don't have a status code between 200 and 299.
func (r *BulkResponse) Failed() []*BulkResponseItem {
	if r.Items == nil {
		return nil
	}
	var errors []*BulkResponseItem
	for _, item := range r.Items {
		for _, result := range item {
			if !(result.Status >= 200 && result.Status <= 299) {
				errors = append(errors, result)
			}
		}
	}
	return errors
}

// Succeeded returns those items of a bulk response that have no errors,
// i.e. those have a status code between 200 and 299.
func (r *BulkResponse) Succeeded() []*BulkResponseItem {
	if r.Items == nil {
		return nil
	}
	var succeeded []*BulkResponseItem
	for _, item := range r.Items {
		for _, result := range item {
			if result.Status >= 200 && result.Status <= 299 {
				succeeded = append(succeeded, result)
			}
		}
	}
	return succeeded
}
<|endoftext|>


<commit_before>// Package clog implements an alternative logger to the one found in the standard
// library with support for more logging levels and a different output format.
// This is not exhaustive or feature-rich.
//
// Author: Clint Caywood (www.clintcaywood.com)
package clog

import (
	"fmt"
	"io"
	"sync"
	"time"
)

type Level uint8

const (
	LevelFatal Level = iota + 1
	LevelError
	LevelWarning
	LevelInfo
	LevelTrace
)

var LevelStrings = map[Level]string{
	LevelFatal:   "Fatal",
	LevelError:   "Error",
	LevelWarning: "Warning",
	LevelInfo:    "Info",
	LevelTrace:   "Trace",
}

type Output struct {
	writer io.Writer
	level  Level
}

type Clog struct {
	mtx     sync.Mutex
	outputs []Output
}

func NewClog() *Clog {
	return &Clog{sync.Mutex{}, make([]Output, 0)}
}

func (this *Clog) AddOutput(writer io.Writer, level Level) {
	this.outputs = append(this.outputs, Output{writer, level})
}

func (this *Clog) Trace(format string, v ...interface{}) {
	this.Log(LevelTrace, format, v...)
}

func (this *Clog) Info(format string, v ...interface{}) {
	this.Log(LevelInfo, format, v...)
}

func (this *Clog) Warning(format string, v ...interface{}) {
	this.Log(LevelWarning, format, v...)
}

func (this *Clog) Error(format string, v ...interface{}) {
	this.Log(LevelError, format, v...)
}

// Will not terminate the program
func (this *Clog) Fatal(format string, v ...interface{}) {
	this.Log(LevelFatal, format, v...)
}

// Logs a message
func (this *Clog) Log(level Level, format string, v ...interface{}) {
	message := fmt.Sprintf(format+"\n", v...)
	strTimestamp := getTimestamp()
	strFinal := fmt.Sprintf("%s [%-7s] %s", strTimestamp, LevelStrings[level], message)
	bytes := []byte(strFinal)
	this.mtx.Lock()
	defer this.mtx.Unlock()
	for _, output := range this.outputs {
		if output.level >= level {
			output.writer.Write(bytes)
		}
	}
}

// Gets the timestamp string
func getTimestamp() string {
	now := time.Now()
	return fmt.Sprintf("%v-%02d-%02d %02d:%02d:%02d.%03d", now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second(), now.Nanosecond()/1000000)
}
<commit_msg>Beef up the doc<commit_after>// Package clog implements an alternative logger to the one found in the standard
// library with support for more logging levels and a different output format.
// It also has support for splitting log files on daily boundaries.
//
// Author: Clint Caywood
//
// https://github.com/cratonica/clog
package clog

import (
	"fmt"
	"io"
	"sync"
	"time"
)

// Represents how critical the logged
// message is.
type Level uint8

const (
	LevelFatal Level = iota + 1
	LevelError
	LevelWarning
	LevelInfo
	LevelTrace
)

var levelStrings = map[Level]string{
	LevelFatal:   "Fatal",
	LevelError:   "Error",
	LevelWarning: "Warning",
	LevelInfo:    "Info",
	LevelTrace:   "Trace",
}

type output struct {
	writer io.Writer
	level  Level
}

// The Logger
type Clog struct {
	mtx     sync.Mutex
	outputs []output
}

// Instantiate a new Clog
func NewClog() *Clog {
	return &Clog{sync.Mutex{}, make([]output, 0)}
}

// Adds an ouput, specifying the maximum log Level
// you want to be written to this output. For instance,
// if you pass Warning for level, all logs of type
// Warning, Error, and Fatal would be logged to this output.
func (this *Clog) AddOutput(writer io.Writer, level Level) {
	this.outputs = append(this.outputs, output{writer, level})
}

// Convenience function
func (this *Clog) Trace(format string, v ...interface{}) {
	this.Log(LevelTrace, format, v...)
}

// Convenience function
func (this *Clog) Info(format string, v ...interface{}) {
	this.Log(LevelInfo, format, v...)
}

// Convenience function
func (this *Clog) Warning(format string, v ...interface{}) {
	this.Log(LevelWarning, format, v...)
}

// Convenience function
func (this *Clog) Error(format string, v ...interface{}) {
	this.Log(LevelError, format, v...)
}

// Convenience function, will not terminate the program
func (this *Clog) Fatal(format string, v ...interface{}) {
	this.Log(LevelFatal, format, v...)
}

// Logs a message for the given level. Most callers will likely
// prefer to use one of the provided convenience functions.
func (this *Clog) Log(level Level, format string, v ...interface{}) {
	message := fmt.Sprintf(format+"\n", v...)
	strTimestamp := getTimestamp()
	strFinal := fmt.Sprintf("%s [%-7s] %s", strTimestamp, levelStrings[level], message)
	bytes := []byte(strFinal)
	this.mtx.Lock()
	defer this.mtx.Unlock()
	for _, output := range this.outputs {
		if output.level >= level {
			output.writer.Write(bytes)
		}
	}
}

// Gets the timestamp string
func getTimestamp() string {
	now := time.Now()
	return fmt.Sprintf("%v-%02d-%02d %02d:%02d:%02d.%03d", now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second(), now.Nanosecond()/1000000)
}
<|endoftext|>


<commit_before>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package siesta

import (
	"net"
	"time"
	"sync"
)

type ConnectionPool struct {
	connectStr         string
	size               int
	conns              int
	keepAlive          bool
	keepAlivePeriod    time.Duration
	connections        []net.Conn
	lock               sync.Mutex
	connReleasedCond   *sync.Cond
}

func NewConnectionPool(connectStr string, size int, keepAlive bool, keepAlivePeriod time.Duration) *ConnectionPool {
	pool := &ConnectionPool{
		connectStr: connectStr,
		size: size,
		conns: 0,
		keepAlive: keepAlive,
		keepAlivePeriod: keepAlivePeriod,
		connections: make([]*net.Conn, 0),
	}

	pool.connReleasedCond = sync.NewCond(pool.lock)

	return pool
}

func (this *ConnectionPool) Borrow() (conn *net.TCPConn, err error) {
	inLock(&this.lock, func() {
		for this.conns >= this.size && len(this.connections) == 0 {
			this.connReleasedCond.Wait()
		}

		if len(this.connections) > 0 {
			conn = this.connections[0]
			this.connections = this.connections[1:]
		} else {
			conn, err = this.connect()
			if err != nil {
				return nil, err
			}
			this.conns++
		}
	})
	return conn, err
}

func (this *ConnectionPool) Return(conn *net.TCPConn) {
	inLock(&this.lock, func() {
		this.connections = append(this.connections, conn)
		this.connReleasedCond.Broadcast()
	})
}

func (this *ConnectionPool) connect() (*net.TCPConn, error) {
	addr, err := net.ResolveTCPAddr("tcp", this.connectStr)
	if err != nil {
		return nil, err
	}
	conn, err := net.DialTCP("tcp", nil, addr)
	if err != nil {
		return nil, err
	}

	if this.keepAlive {
		conn.SetKeepAlive(this.keepAlive)
		conn.SetKeepAlivePeriod(this.keepAlivePeriod)
	}

	return conn, nil
}
<commit_msg>Added connection pool, sync utils. build fix<commit_after>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package siesta

import (
	"net"
	"time"
	"sync"
)

type ConnectionPool struct {
	connectStr         string
	size               int
	conns              int
	keepAlive          bool
	keepAlivePeriod    time.Duration
	connections        []*net.TCPConn
	lock               sync.Mutex
	connReleasedCond   *sync.Cond
}

func NewConnectionPool(connectStr string, size int, keepAlive bool, keepAlivePeriod time.Duration) *ConnectionPool {
	pool := &ConnectionPool{
		connectStr: connectStr,
		size: size,
		conns: 0,
		keepAlive: keepAlive,
		keepAlivePeriod: keepAlivePeriod,
		connections: make([]*net.TCPConn, 0),
	}

	pool.connReleasedCond = sync.NewCond(&pool.lock)

	return pool
}

func (this *ConnectionPool) Borrow() (conn *net.TCPConn, err error) {
	inLock(&this.lock, func() {
		for this.conns >= this.size && len(this.connections) == 0 {
			this.connReleasedCond.Wait()
		}

		if len(this.connections) > 0 {
			conn = this.connections[0]
			this.connections = this.connections[1:]
		} else {
			conn, err = this.connect()
			if err != nil {
				return
			}
			this.conns++
		}
	})
	return conn, err
}

func (this *ConnectionPool) Return(conn *net.TCPConn) {
	inLock(&this.lock, func() {
		this.connections = append(this.connections, conn)
		this.connReleasedCond.Broadcast()
	})
}

func (this *ConnectionPool) connect() (*net.TCPConn, error) {
	addr, err := net.ResolveTCPAddr("tcp", this.connectStr)
	if err != nil {
		return nil, err
	}
	conn, err := net.DialTCP("tcp", nil, addr)
	if err != nil {
		return nil, err
	}

	if this.keepAlive {
		conn.SetKeepAlive(this.keepAlive)
		conn.SetKeepAlivePeriod(this.keepAlivePeriod)
	}

	return conn, nil
}
<|endoftext|>


<commit_before>package mint

import (
	"encoding/hex"
	"fmt"
)

type Conn struct {
	in, out *recordLayer
}

const verifyDataLen = 20 // XXX

func (c *Conn) ClientHandshake() {
	hIn := newHandshakeLayer(c.in)
	hOut := newHandshakeLayer(c.out)

	// Generate, construct, and marshal key share
	config_keyShareGroups := []namedGroup{namedGroupP256, namedGroupP384, namedGroupP521}
	privateKeys := map[namedGroup][]byte{}
	ks := &keyShareExtension{
		roleIsServer: false,
		shares:       make([]keyShare, len(config_keyShareGroups)),
	}
	for i, group := range config_keyShareGroups {
		pub, priv, err := newKeyShare(group)
		if err != nil {
			panic(err) // XXX
		}

		ks.shares[i].group = group
		ks.shares[i].keyExchange = pub
		privateKeys[group] = priv
	}
	keyShareBytes, err := ks.Marshal()
	if err != nil {
		panic(err)
	}

	// Construct and write ClientHello
	ch := &clientHelloBody{
		cipherSuites: []cipherSuite{0x0000}, // XXX
		extensions: []extension{
			extension{
				extensionType: extensionTypeKeyShare,
				extensionData: keyShareBytes,
			},
		},
	}
	err = hOut.WriteMessageBody(ch)
	if err != nil {
		panic(err) // XXX Do something better
	}

	// Read ServerHello
	sh := new(serverHelloBody)
	err = hIn.ReadMessageBody(sh)
	if err != nil {
		panic(err) // XXX Do something better
	}

	// Read the key_share extension and do key agreement
	foundKeyShare := false
	serverKeyShares := keyShareExtension{roleIsServer: true}
	for _, ext := range sh.extensions {
		if ext.extensionType == extensionTypeKeyShare {
			_, err := serverKeyShares.Unmarshal(ext.extensionData)
			foundKeyShare = (err == nil)
			if foundKeyShare {
				break
			}
		}
	}
	if !foundKeyShare {
		panic("No client key share")
	}
	if len(serverKeyShares.shares) != 1 {
		panic("Server provided empty key_shares") // XXX should check in Unmarshal
	}
	sks := serverKeyShares.shares[0]
	priv, ok := privateKeys[sks.group]
	if !ok {
		panic("Server sent a private key for a group we didn't send")
	}
	ES, err := keyAgreement(sks.group, sks.keyExchange, priv)
	if err != nil {
		panic(err)
	}

	fmt.Println("ES_client:", hex.EncodeToString(ES))

	// Read Finished
	serverFin := new(finishedBody)
	serverFin.verifyDataLen = verifyDataLen // XXX
	err = hIn.ReadMessageBody(serverFin)
	if err != nil {
		panic(err) // XXX Do something better
	}

	// Write Finished
	clientFin := &finishedBody{
		verifyDataLen: verifyDataLen,
		verifyData:    make([]byte, verifyDataLen),
	}
	err = hOut.WriteMessageBody(clientFin)
	if err != nil {
		panic(err)
	}

	fmt.Println("Client done")
}

func (c *Conn) ServerHandshake() {
	hIn := newHandshakeLayer(c.in)
	hOut := newHandshakeLayer(c.out)

	// Read ClientHello
	ch := new(clientHelloBody)
	err := hIn.ReadMessageBody(ch)
	if err != nil {
		panic(err) // XXX Do something better
	}

	// Find key_share extension and do key agreement
	config_supportedGroup := map[namedGroup]bool{
		namedGroupP384: true,
		namedGroupP521: true,
	}
	foundKeyShare := false
	clientKeyShares := keyShareExtension{roleIsServer: false}
	for _, ext := range ch.extensions {
		if ext.extensionType == extensionTypeKeyShare {
			_, err := clientKeyShares.Unmarshal(ext.extensionData)
			foundKeyShare = (err == nil)
			if foundKeyShare {
				break
			}
		}
	}
	if !foundKeyShare {
		panic("No client key share")
	}
	var keyShareBytes []byte
	var ES []byte
	for _, share := range clientKeyShares.shares {
		if config_supportedGroup[share.group] {
			pub, priv, err := newKeyShare(share.group)
			if err != nil {
				panic(err) // XXX
			}

			ES, err = keyAgreement(share.group, share.keyExchange, priv)
			ks := keyShareExtension{
				roleIsServer: true,
				shares:       []keyShare{keyShare{group: share.group, keyExchange: pub}},
			}
			keyShareBytes, err = ks.Marshal()
			if err != nil {
				panic(err)
			}
			break
		}
	}
	if len(keyShareBytes) == 0 || len(ES) == 0 {
		panic("key agreement failed")
	}

	fmt.Println("ES_server:", hex.EncodeToString(ES))

	// Create and write ServerHello
	sh := &serverHelloBody{
		cipherSuite: 0x0000,
		extensions: []extension{
			extension{
				extensionType: extensionTypeKeyShare,
				extensionData: keyShareBytes,
			},
		},
	}
	err = hOut.WriteMessageBody(sh)
	if err != nil {
		panic(err) // XXX Do something better
	}

	// Create and write Finished
	serverFin := &finishedBody{
		verifyDataLen: verifyDataLen,
		verifyData:    make([]byte, verifyDataLen),
	}
	err = hOut.WriteMessageBody(serverFin)
	if err != nil {
		panic(err)
	}

	// Read Finished
	clientFin := new(finishedBody)
	clientFin.verifyDataLen = verifyDataLen // XXX
	err = hIn.ReadMessageBody(clientFin)
	if err != nil {
		panic(err) // XXX Do something better
	}

	fmt.Println("Server done")
}
<commit_msg>Take advantage of improved extension handling<commit_after>package mint

import (
	"encoding/hex"
	"fmt"
)

type Conn struct {
	in, out *recordLayer
}

const verifyDataLen = 20 // XXX

func (c *Conn) ClientHandshake() {
	hIn := newHandshakeLayer(c.in)
	hOut := newHandshakeLayer(c.out)

	// Generate, construct, and marshal key share
	config_keyShareGroups := []namedGroup{namedGroupP256, namedGroupP384, namedGroupP521}
	privateKeys := map[namedGroup][]byte{}
	ks := &keyShareExtension{
		roleIsServer: false,
		shares:       make([]keyShare, len(config_keyShareGroups)),
	}
	for i, group := range config_keyShareGroups {
		pub, priv, err := newKeyShare(group)
		if err != nil {
			panic(err) // XXX
		}

		ks.shares[i].group = group
		ks.shares[i].keyExchange = pub
		privateKeys[group] = priv
	}

	// Construct and write ClientHello
	ch := &clientHelloBody{
		cipherSuites: []cipherSuite{0x0000}, // XXX
	}
	ch.extensions.Add(extensionTypeKeyShare, ks)
	err := hOut.WriteMessageBody(ch)
	if err != nil {
		panic(err) // XXX Do something better
	}

	// Read ServerHello
	sh := new(serverHelloBody)
	err = hIn.ReadMessageBody(sh)
	if err != nil {
		panic(err) // XXX Do something better
	}

	// Read the key_share extension and do key agreement
	foundKeyShare := false
	serverKeyShares := keyShareExtension{roleIsServer: true}
	for _, ext := range sh.extensions {
		if ext.extensionType == extensionTypeKeyShare {
			_, err := serverKeyShares.Unmarshal(ext.extensionData)
			foundKeyShare = (err == nil)
			if foundKeyShare {
				break
			}
		}
	}
	if !foundKeyShare {
		panic("No client key share")
	}
	if len(serverKeyShares.shares) != 1 {
		panic("Server provided empty key_shares") // XXX should check in Unmarshal
	}
	sks := serverKeyShares.shares[0]
	priv, ok := privateKeys[sks.group]
	if !ok {
		panic("Server sent a private key for a group we didn't send")
	}
	ES, err := keyAgreement(sks.group, sks.keyExchange, priv)
	if err != nil {
		panic(err)
	}

	fmt.Println("ES_client:", hex.EncodeToString(ES))

	// Read Finished
	serverFin := new(finishedBody)
	serverFin.verifyDataLen = verifyDataLen // XXX
	err = hIn.ReadMessageBody(serverFin)
	if err != nil {
		panic(err) // XXX Do something better
	}

	// Write Finished
	clientFin := &finishedBody{
		verifyDataLen: verifyDataLen,
		verifyData:    make([]byte, verifyDataLen),
	}
	err = hOut.WriteMessageBody(clientFin)
	if err != nil {
		panic(err)
	}

	fmt.Println("Client done")
}

func (c *Conn) ServerHandshake() {
	hIn := newHandshakeLayer(c.in)
	hOut := newHandshakeLayer(c.out)

	// Read ClientHello
	ch := new(clientHelloBody)
	err := hIn.ReadMessageBody(ch)
	if err != nil {
		panic(err) // XXX Do something better
	}

	// Find key_share extension and do key agreement
	config_supportedGroup := map[namedGroup]bool{
		namedGroupP384: true,
		namedGroupP521: true,
	}
	clientKeyShares := &keyShareExtension{roleIsServer: false}
	found := ch.extensions.Find(extensionTypeKeyShare, clientKeyShares)
	if !found {
		panic("No client key shares")
	}
	var serverKeyShare *keyShareExtension
	var ES []byte
	for _, share := range clientKeyShares.shares {
		if config_supportedGroup[share.group] {
			pub, priv, err := newKeyShare(share.group)
			if err != nil {
				panic(err) // XXX
			}

			ES, err = keyAgreement(share.group, share.keyExchange, priv)
			serverKeyShare = &keyShareExtension{
				roleIsServer: true,
				shares:       []keyShare{keyShare{group: share.group, keyExchange: pub}},
			}
			break
		}
	}
	if serverKeyShare == nil || len(ES) == 0 {
		panic("key agreement failed")
	}

	fmt.Println("ES_server:", hex.EncodeToString(ES))

	// Create and write ServerHello
	sh := &serverHelloBody{
		cipherSuite: 0x0000,
	}
	sh.extensions.Add(extensionTypeKeyShare, serverKeyShare)
	err = hOut.WriteMessageBody(sh)
	if err != nil {
		panic(err) // XXX Do something better
	}

	// Create and write Finished
	serverFin := &finishedBody{
		verifyDataLen: verifyDataLen,
		verifyData:    make([]byte, verifyDataLen),
	}
	err = hOut.WriteMessageBody(serverFin)
	if err != nil {
		panic(err)
	}

	// Read Finished
	clientFin := new(finishedBody)
	clientFin.verifyDataLen = verifyDataLen // XXX
	err = hIn.ReadMessageBody(clientFin)
	if err != nil {
		panic(err) // XXX Do something better
	}

	fmt.Println("Server done")
}
<|endoftext|>


<commit_before>package proxyprotocol

import (
	"bufio"
	"bytes"
	parser "github.com/racker/go-proxy-protocol"
	"io"
	"net"
	"time"
)

type conn struct {
	raw   net.Conn
	proxy *parser.ProxyLine
	err   error
}

func (c *conn) init() {
	if c.proxy != nil || c.err != nil {
		return
	}

	// prepare for the longest (TCP6) line
	buf := make([]byte, 107)

	// read at least the shortest (UNKNOWN) line
	n, err := c.raw.Read(buf[:15])
	if err != nil && !(err == io.EOF && n == 15) {
		c.err = err
		return
	}
	if n < 15 {
		c.err = io.EOF
		return
	}

	for i := n; i < len(buf); i++ {
		if buf[i-2] == byte('\r') {
			if buf[i-1] == byte('\n') {
				buf = buf[:i]
				break
			} else {
				c.err = parser.InvalidProxyLine
				return
			}
		}

		n, c.err = c.raw.Read(buf[i : i+1])
		if c.err != nil {
			return
		}
		if n != 1 {
			c.err = io.EOF
			return
		}
	}

	if bytes.HasPrefix(buf, []byte("PROXY UNKNOWN")) {
		c.proxy = new(parser.ProxyLine)
		return
	}

	c.proxy, c.err = parser.ConsumeProxyLine(bufio.NewReader(bytes.NewReader(buf)))
	return
}

func (c *conn) Read(b []byte) (n int, err error) {
	c.init()
	if c.err != nil {
		err = c.err
	} else {
		n, err = c.raw.Read(b)
	}
	return
}

func (c *conn) Write(b []byte) (n int, err error) {
	n, err = c.raw.Write(b)
	return
}

func (c *conn) Close() error {
	return c.raw.Close()
}

func (c *conn) LocalAddr() net.Addr {
	c.init()
	if c.err == nil && c.proxy.DstAddr != nil {
		return c.proxy.DstAddr
	}
	return c.raw.LocalAddr()
}

func (c *conn) RemoteAddr() net.Addr {
	c.init()
	if c.err == nil && c.proxy.SrcAddr != nil {
		return c.proxy.SrcAddr
	}
	return c.raw.RemoteAddr()
}

func (c *conn) SetDeadline(t time.Time) error {
	return c.raw.SetDeadline(t)
}

func (c *conn) SetReadDeadline(t time.Time) error {
	return c.raw.SetReadDeadline(t)
}

func (c *conn) SetWriteDeadline(t time.Time) error {
	return c.raw.SetWriteDeadline(t)
}
<commit_msg>handle non-PROXY input<commit_after>package proxyprotocol

import (
	"bufio"
	"bytes"
	parser "github.com/racker/go-proxy-protocol"
	"io"
	"net"
	"time"
)

type conn struct {
	raw   net.Conn
	proxy *parser.ProxyLine
	err   error
}

func (c *conn) init() {
	if c.proxy != nil || c.err != nil {
		return
	}

	// prepare for the longest (TCP6) line
	buf := make([]byte, 107)

	// read at least the shortest (UNKNOWN) line
	n, err := c.raw.Read(buf[:15])
	if err != nil && !(err == io.EOF && n == 15) {
		c.err = err
		return
	}
	if n < 15 {
		c.err = io.EOF
		return
	}

	for i := n; i < len(buf); i++ {
		if buf[i-2] == byte('\r') {
			if buf[i-1] == byte('\n') {
				buf = buf[:i]
				break
			} else {
				c.err = parser.InvalidProxyLine
				return
			}
		}

		n, c.err = c.raw.Read(buf[i : i+1])
		if c.err != nil {
			return
		}
		if n != 1 {
			c.err = io.EOF
			return
		}
	}

	if bytes.HasPrefix(buf, []byte("PROXY UNKNOWN")) {
		c.proxy = new(parser.ProxyLine)
		return
	}

	c.proxy, c.err = parser.ConsumeProxyLine(bufio.NewReader(bytes.NewReader(buf)))
	if c.proxy == nil && c.err == nil {
		c.err = parser.InvalidProxyLine
	}
	return
}

func (c *conn) Read(b []byte) (n int, err error) {
	c.init()
	if c.err != nil {
		err = c.err
	} else {
		n, err = c.raw.Read(b)
	}
	return
}

func (c *conn) Write(b []byte) (n int, err error) {
	n, err = c.raw.Write(b)
	return
}

func (c *conn) Close() error {
	return c.raw.Close()
}

func (c *conn) LocalAddr() net.Addr {
	c.init()
	if c.err == nil && c.proxy.DstAddr != nil {
		return c.proxy.DstAddr
	}
	return c.raw.LocalAddr()
}

func (c *conn) RemoteAddr() net.Addr {
	c.init()
	if c.err == nil && c.proxy.SrcAddr != nil {
		return c.proxy.SrcAddr
	}
	return c.raw.RemoteAddr()
}

func (c *conn) SetDeadline(t time.Time) error {
	return c.raw.SetDeadline(t)
}

func (c *conn) SetReadDeadline(t time.Time) error {
	return c.raw.SetReadDeadline(t)
}

func (c *conn) SetWriteDeadline(t time.Time) error {
	return c.raw.SetWriteDeadline(t)
}
<|endoftext|>


<commit_before>package sol

import (
	"database/sql"
	"log"
	"reflect"

	"github.com/aodin/sol/dialect"
)

// executer is a common interface that database/sql *DB and *Tx can share
type executer interface {
	Exec(query string, args ...interface{}) (sql.Result, error)
	Query(query string, args ...interface{}) (*sql.Rows, error)
}

func compile(d dialect.Dialect, stmt Executable) (string, *Parameters, error) {
	// Initialize a list of empty parameters
	params := Params()

	// Compile with the database connection's current dialect
	compiled, err := stmt.Compile(d, params)
	return compiled, params, err
}

func execute(exec executer, d dialect.Dialect, stmt Executable) (sql.Result, error) {
	compiled, params, err := compile(d, stmt)
	if err != nil {
		return nil, err
	}
	return exec.Exec(compiled, *params...)
}

func perform(exec executer, d dialect.Dialect, stmt Executable, dest ...interface{}) error {
	if len(dest) == 0 {
		_, err := execute(exec, d, stmt)
		return err
	}

	if len(dest) > 1 {
		return queryAll(exec, d, stmt, dest)
	}

	t := reflect.Indirect(reflect.ValueOf(dest[0]))
	if t.Kind() == reflect.Slice {
		return queryAll(exec, d, stmt, dest[0])
	}
	return queryOne(exec, d, stmt, dest[0])
}

func query(exec executer, d dialect.Dialect, stmt Executable) (*Result, error) {
	compiled, params, err := compile(d, stmt)
	if err != nil {
		return nil, err
	}

	rows, err := exec.Query(compiled, *params...)
	if err != nil {
		return nil, err
	}
	// Wrap the sql rows in a result
	return &Result{Scanner: rows, stmt: compiled}, nil
}

// QueryAll will query the statement and populate the given destination
// interface with all results.
func queryAll(exec executer, d dialect.Dialect, stmt Executable, dest interface{}) error {
	result, err := query(exec, d, stmt)
	if err != nil {
		return err
	}
	return result.All(dest)
}

// QueryOne will query the statement and populate the given destination
// interface with a single result.
func queryOne(exec executer, d dialect.Dialect, stmt Executable, dest interface{}) error {
	result, err := query(exec, d, stmt)
	if err != nil {
		return err
	}
	// Close the result rows or sqlite3 will open another connection
	defer result.Close()
	return result.One(dest)
}

// Connection is an alias for Conn
type Connection interface {
	Conn
}

// Transaction is an alias for TX
type Transaction interface {
	TX
}

type Conn interface {
	Begin() (TX, error)
	Close() error
	Query(stmt Executable, dest ...interface{}) error
	String(stmt Executable) string
}

type TX interface {
	Conn
	Commit() error
	IsSuccessful()
	Rollback() error
}

type conn struct {
	db      *sql.DB
	dialect dialect.Dialect
}

// Begin will start a new transaction on the current connection pool
func (c *conn) Begin() (TX, error) {
	tx, err := c.db.Begin()
	return &transaction{Tx: tx, dialect: c.dialect}, err
}

// Close will make the current connection pool unusable
func (c *conn) Close() error {
	return c.db.Close()
}

// Dialect returns the current connection pool's dialect, e.g. sqlite3
func (c *conn) Dialect() dialect.Dialect {
	return c.dialect
}

// Query executes an Executable statement.
func (c *conn) Query(stmt Executable, dest ...interface{}) error {
	return perform(c.db, c.dialect, stmt, dest...)
}

// String returns parameter-less SQL. If an error occurred during compilation,
// then the string output of the error will be returned.
// TODO Common string function
func (c *conn) String(stmt Executable) string {
	compiled, err := stmt.Compile(c.dialect, Params())
	if err != nil {
		return err.Error()
	}
	return compiled
}

// PanicOnError returns a panicConn, which implements the Connection
// interface but will also panic on any error
func (c *conn) PanicOnError() panicConn {
	return panicConn{c}
}

// Must is an alias for PanicOnError
func (c *conn) Must() panicConn {
	return c.PanicOnError()
}

// Open connects to the database using the given driver and credentials.
// It returns a database connection pool and an error if one occurred.
func Open(driver, credentials string) (*conn, error) {
	db, err := sql.Open(driver, credentials)
	if err != nil {
		return nil, err
	}

	// Get the dialect
	d, err := dialect.Get(driver)
	if err != nil {
		return nil, err
	}
	return &conn{db: db, dialect: d}, nil
}

// panicConn is a connection that panics on error
type panicConn struct {
	*conn
}

// Begin calls the internal conn Begin(), but will panic on any error
// and returns a panicTx instead of a transaction
func (c panicConn) Begin() (TX, error) {
	tx, err := c.conn.Begin()
	if err != nil {
		log.Panic(err)
	}
	return panicTx{tx.(*transaction)}, err
}

// Close calls the internal conn Close(), but will panic on any error
func (c panicConn) Close() error {
	err := c.conn.Close()
	if err != nil {
		log.Panic(err)
	}
	return err
}

// Query calls the internal conn Query(), but will panic on any error
func (c panicConn) Query(stmt Executable, dest ...interface{}) error {
	err := c.conn.Query(stmt, dest...)
	if err != nil {
		log.Panic(err)
	}
	return err
}

type transaction struct {
	*sql.Tx
	dialect    dialect.Dialect
	successful bool
}

// Begin simply returns the transaction itself
// TODO Are nested transactions possible? Or should this error?
func (tx *transaction) Begin() (TX, error) {
	return tx, nil
}

// Close will commit the transaction unless it has failed
func (tx *transaction) Close() error {
	if tx.successful {
		return tx.Commit()
	}
	return tx.Rollback()
}

func (tx *transaction) IsSuccessful() {
	tx.successful = true
}

// Query executes an Executable statement
func (tx *transaction) Query(stmt Executable, dest ...interface{}) error {
	return perform(tx.Tx, tx.dialect, stmt, dest...)
}

func (tx *transaction) String(stmt Executable) string {
	compiled, err := stmt.Compile(tx.dialect, Params())
	if err != nil {
		return err.Error()
	}
	return compiled
}

type panicTx struct {
	*transaction
}

// Begin returns the panicTx itself
// TODO Are nested transactions possible? Or should this error?
func (tx panicTx) Begin() (TX, error) {
	return tx, nil
}

// Close will call the internal transaction Close(). It will panic on error.
func (tx panicTx) Close() error {
	err := tx.transaction.Close()
	if err != nil {
		log.Panic(err)
	}
	return err
}

// Query will call the internal transaction Query(). It will panic on error.
func (tx panicTx) Query(stmt Executable, dest ...interface{}) error {
	err := tx.transaction.Query(stmt, dest...)
	if err != nil {
		log.Panic(err)
	}
	return err
}
<commit_msg>Replaced panic connection and transaction types with a panicky field on each parent type<commit_after>package sol

import (
	"database/sql"
	"log"
	"reflect"

	"github.com/aodin/sol/dialect"
)

// executer is a common interface that database/sql *DB and *Tx can share
type executer interface {
	Exec(query string, args ...interface{}) (sql.Result, error)
	Query(query string, args ...interface{}) (*sql.Rows, error)
}

func compile(d dialect.Dialect, stmt Executable) (string, *Parameters, error) {
	// Initialize a list of empty parameters
	params := Params()

	// Compile with the database connection's current dialect
	compiled, err := stmt.Compile(d, params)
	return compiled, params, err
}

func execute(exec executer, d dialect.Dialect, stmt Executable) (sql.Result, error) {
	compiled, params, err := compile(d, stmt)
	if err != nil {
		return nil, err
	}
	return exec.Exec(compiled, *params...)
}

func perform(exec executer, d dialect.Dialect, stmt Executable, dest ...interface{}) error {
	if len(dest) == 0 {
		_, err := execute(exec, d, stmt)
		return err
	}

	if len(dest) > 1 {
		return queryAll(exec, d, stmt, dest)
	}

	t := reflect.Indirect(reflect.ValueOf(dest[0]))
	if t.Kind() == reflect.Slice {
		return queryAll(exec, d, stmt, dest[0])
	}
	return queryOne(exec, d, stmt, dest[0])
}

func query(exec executer, d dialect.Dialect, stmt Executable) (*Result, error) {
	compiled, params, err := compile(d, stmt)
	if err != nil {
		return nil, err
	}

	rows, err := exec.Query(compiled, *params...)
	if err != nil {
		return nil, err
	}
	// Wrap the sql rows in a result
	return &Result{Scanner: rows, stmt: compiled}, nil
}

// QueryAll will query the statement and populate the given destination
// interface with all results.
func queryAll(exec executer, d dialect.Dialect, stmt Executable, dest interface{}) error {
	result, err := query(exec, d, stmt)
	if err != nil {
		return err
	}
	return result.All(dest)
}

// QueryOne will query the statement and populate the given destination
// interface with a single result.
func queryOne(exec executer, d dialect.Dialect, stmt Executable, dest interface{}) error {
	result, err := query(exec, d, stmt)
	if err != nil {
		return err
	}
	// Close the result rows or sqlite3 will open another connection
	defer result.Close()
	return result.One(dest)
}

// Connection is an alias for Conn
type Connection interface {
	Conn
}

// Transaction is an alias for TX
type Transaction interface {
	TX
}

type Conn interface {
	Begin() (TX, error)
	Close() error
	Query(stmt Executable, dest ...interface{}) error
	String(stmt Executable) string
}

type TX interface {
	Conn
	Commit() error
	IsSuccessful()
	Rollback() error
}

type conn struct {
	db      *sql.DB
	dialect dialect.Dialect
	panicky bool
}

// Begin will start a new transaction on the current connection pool
func (c *conn) Begin() (TX, error) {
	tx, err := c.db.Begin()
	if c.panicky && err != nil {
		log.Panic(err)
	}
	return &transaction{Tx: tx, dialect: c.dialect, panicky: c.panicky}, err
}

// Close will make the current connection pool unusable
func (c *conn) Close() error {
	err := c.db.Close()
	if c.panicky && err != nil {
		log.Panic(err)
	}
	return err
}

// Dialect returns the current connection pool's dialect, e.g. sqlite3
func (c *conn) Dialect() dialect.Dialect {
	return c.dialect
}

// Query executes an Executable statement.
func (c *conn) Query(stmt Executable, dest ...interface{}) error {
	err := perform(c.db, c.dialect, stmt, dest...)
	if c.panicky && err != nil {
		log.Panic(err)
	}
	return err
}

// String returns parameter-less SQL. If an error occurred during compilation,
// then the string output of the error will be returned.
// TODO Common string function
func (c *conn) String(stmt Executable) string {
	compiled, err := stmt.Compile(c.dialect, Params())
	if err != nil {
		return err.Error()
	}
	return compiled
}

// PanicOnError will set the connection to panic on any error
func (c *conn) PanicOnError() *conn {
	c.panicky = true
	return c
}

// Must is an alias for PanicOnError
func (c *conn) Must() *conn {
	return c.PanicOnError()
}

// Open connects to the database using the given driver and credentials.
// It returns a database connection pool and an error if one occurred.
func Open(driver, credentials string) (*conn, error) {
	db, err := sql.Open(driver, credentials)
	if err != nil {
		return nil, err
	}

	// Get the dialect
	d, err := dialect.Get(driver)
	if err != nil {
		return nil, err
	}
	return &conn{db: db, dialect: d}, nil
}

type transaction struct {
	*sql.Tx
	dialect    dialect.Dialect
	successful bool
	panicky    bool
}

// Begin simply returns the transaction itself
// TODO Are nested transactions possible? Or should this error?
func (tx *transaction) Begin() (TX, error) {
	return tx, nil
}

// Close will commit the transaction unless it has failed
func (tx *transaction) Close() (err error) {
	if tx.successful {
		err = tx.Commit()
	} else {
		err = tx.Rollback()
	}
	if tx.panicky && err != nil {
		log.Panic(err)
	}
	return
}

func (tx *transaction) IsSuccessful() {
	tx.successful = true
}

// Query executes an Executable statement
func (tx *transaction) Query(stmt Executable, dest ...interface{}) error {
	err := perform(tx.Tx, tx.dialect, stmt, dest...)
	if tx.panicky && err != nil {
		log.Panic(err)
	}
	return err
}

func (tx *transaction) String(stmt Executable) string {
	compiled, err := stmt.Compile(tx.dialect, Params())
	if err != nil {
		return err.Error()
	}
	return compiled
}
<|endoftext|>


<commit_before>package ginDoi

import (
	"fmt"
	"os"
	"crypto/rsa"
)

type MockDataSource struct {
	calls []string
}

func (ds *MockDataSource) ValidDoiFile(URI string, user OauthIdentity) (bool, *CBerry) {
	return true, &CBerry{}
}
func (ds *MockDataSource) Get(URI string, To string, key *rsa.PrivateKey) (string, error) {
	os.Mkdir(To, os.ModePerm)
	ds.calls = append(ds.calls, fmt.Sprintf("%s, %s", URI, To))
	return "", nil
}

func (ds *MockDataSource) MakeUUID(URI string, user OauthIdentity) (string, error) {
	return "123", nil
}

type MockDoiProvider struct {
}

func (dp MockDoiProvider) MakeDoi(doiInfo *CBerry) string {
	return "133"
}
func (dp MockDoiProvider) GetXml(doiInfo *CBerry) (string, error) {
	return "xml", nil
}
func (dp MockDoiProvider) RegDoi(doiInfo CBerry) (string, error) {
	return "", nil
}
<commit_msg>Oauth provider mock<commit_after>package ginDoi

import (
	"fmt"
	"os"
	"crypto/rsa"
	"github.com/G-Node/gin-core/gin"
)

type MockDataSource struct {
	calls        []string
	validDoiFile bool
	Berry        CBerry
}

func (ds *MockDataSource) ValidDoiFile(URI string, user OauthIdentity) (bool, *CBerry) {
	return ds.validDoiFile, &ds.Berry
}
func (ds *MockDataSource) Get(URI string, To string, key *rsa.PrivateKey) (string, error) {
	os.Mkdir(To, os.ModePerm)
	ds.calls = append(ds.calls, fmt.Sprintf("%s, %s", URI, To))
	return "", nil
}

func (ds *MockDataSource) MakeUUID(URI string, user OauthIdentity) (string, error) {
	return "123", nil
}

type MockDoiProvider struct {
}

func (dp MockDoiProvider) MakeDoi(doiInfo *CBerry) string {
	return "133"
}
func (dp MockDoiProvider) GetXml(doiInfo *CBerry) (string, error) {
	return "xml", nil
}
func (dp MockDoiProvider) RegDoi(doiInfo CBerry) (string, error) {
	return "", nil
}

type MockOauthProvider struct {
	ValidToken bool
	User       OauthIdentity
}

func (op MockOauthProvider) ValidateToken(userName string, token string) (bool, error) {
	return op.ValidToken, nil
}

func (op MockOauthProvider) getUser(userName string, token string) (OauthIdentity, error) {
	return op.User, nil
}

func (op MockOauthProvider) AuthorizePull(user OauthIdentity) (*rsa.PrivateKey, error) {
	return &rsa.PrivateKey{}, nil
}

func (op MockOauthProvider) DeAuthorizePull(user OauthIdentity, key gin.SSHKey) (error) {
	return nil
}<|endoftext|>


<commit_before>package dnews

import (
	"regexp"
	"time"
)

// User represents an author of an article
type User struct {
	ID      int
	Created time.Time
	LName   string
	FName   string
	Email   string
	Pubkey  string
	User    string
	Authed  bool
}

var userLineRE = regexp.MustCompile(`^(.*)\s(.*)\s<(.*)>$`)

// Parse takes a 'First Last <user@email.com>' style string and creates a User
func (u *User) Parse(s string) {
	u.FName = userLineRE.ReplaceAllString(s, "$1")
	u.LName = userLineRE.ReplaceAllString(s, "$2")
	u.Email = userLineRE.ReplaceAllString(s, "$3")
}

// Users are a collection of User
type Users *[]User
<commit_msg>add Combine to give us our purty strings<commit_after>package dnews

import (
	"fmt"
	"regexp"
	"time"
)

// User represents an author of an article
type User struct {
	ID      int
	Created time.Time
	LName   string
	FName   string
	Email   string
	Pubkey  string
	User    string
	Pass    string
	Hash    string
	Authed  bool
}

var userLineRE = regexp.MustCompile(`^(.*)\s(.*)\s<(.*)>$`)

// Parse takes a 'First Last <user@email.com>' style string and creates a User
func (u *User) Parse(s string) {
	u.FName = userLineRE.ReplaceAllString(s, "$1")
	u.LName = userLineRE.ReplaceAllString(s, "$2")
	u.Email = userLineRE.ReplaceAllString(s, "$3")
}

// Combine concatenates FName, LName and Email into one line
func (u *User) Combine() string {
	return fmt.Sprintf("%s %s <%s>", u.FName, u.LName, u.Email)
}

// Users are a collection of User
type Users *[]User
<|endoftext|>


<commit_before>package srv

import (
	"log"
	"net"
)

const Version = "0.7.4"

var sing struct {
	Dao			*dao
	Rao			*rao
	UssnMgr		*ussnMgr
	BookMgr		*bookMgr
	TssnMgr		*tssnMgr
}

func Serve(port string) {
	sing.Dao = newDao()
	sing.Rao = newRao()
	sing.UssnMgr = newUssnMgr()
	sing.BookMgr = newBookMgr()
	sing.TssnMgr = newTssnMgr()
	defer sing.Dao.Close()
	go sing.UssnMgr.Loop()
	go sing.BookMgr.Loop()
	go sing.TssnMgr.Loop()

	ln, err := net.Listen("tcp", ":" + port)
	if err != nil {
		log.Fatalln(err)
	} else {
		log.Println("mjpancake-server", Version, "listen", port)
	}

	for {
		conn, err := ln.Accept()

		if err != nil {
			log.Println("E accept", err)
		} else {
			go loopUssn(conn)
		}
	}
}

<commit_msg>bump to 0.7.5<commit_after>package srv

import (
	"log"
	"net"
)

const Version = "0.7.5"

var sing struct {
	Dao			*dao
	Rao			*rao
	UssnMgr		*ussnMgr
	BookMgr		*bookMgr
	TssnMgr		*tssnMgr
}

func Serve(port string) {
	sing.Dao = newDao()
	sing.Rao = newRao()
	sing.UssnMgr = newUssnMgr()
	sing.BookMgr = newBookMgr()
	sing.TssnMgr = newTssnMgr()
	defer sing.Dao.Close()
	go sing.UssnMgr.Loop()
	go sing.BookMgr.Loop()
	go sing.TssnMgr.Loop()

	ln, err := net.Listen("tcp", ":" + port)
	if err != nil {
		log.Fatalln(err)
	} else {
		log.Println("mjpancake-server", Version, "listen", port)
	}

	for {
		conn, err := ln.Accept()

		if err != nil {
			log.Println("E accept", err)
		} else {
			go loopUssn(conn)
		}
	}
}

<|endoftext|>