<commit_before>package v1alpha1

import (
	"fmt"
	"strings"

	"github.com/appscode/kube-mon/api"
	crdutils "github.com/appscode/kutil/apiextensions/v1beta1"
	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
	crd_api "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
)

const (
	ESSearchguardDisabled = ElasticsearchKey + "/searchguard-disabled"
)

func (e Elasticsearch) OffshootName() string {
	return e.Name
}

func (e Elasticsearch) OffshootLabels() map[string]string {
	return map[string]string{
		LabelDatabaseKind: ResourceKindElasticsearch,
		LabelDatabaseName: e.Name,
	}
}

func (e Elasticsearch) StatefulSetLabels() map[string]string {
	labels := e.OffshootLabels()
	for key, val := range e.Labels {
		if !strings.HasPrefix(key, GenericKey+"/") && !strings.HasPrefix(key, ElasticsearchKey+"/") {
			labels[key] = val
		}
	}
	return labels
}

func (e Elasticsearch) StatefulSetAnnotations() map[string]string {
	annotations := make(map[string]string)
	for key, val := range e.Annotations {
		if !strings.HasPrefix(key, GenericKey+"/") && !strings.HasPrefix(key, ElasticsearchKey+"/") {
			annotations[key] = val
		}
	}
	return annotations
}

var _ ResourceInfo = &Elasticsearch{}

func (e Elasticsearch) ResourceShortCode() string {
	return ResourceCodeElasticsearch
}

func (e Elasticsearch) ResourceKind() string {
	return ResourceKindElasticsearch
}

func (e Elasticsearch) ResourceSingular() string {
	return ResourceSingularElasticsearch
}

func (e Elasticsearch) ResourcePlural() string {
	return ResourcePluralElasticsearch
}

func (r Elasticsearch) ServiceName() string {
	return r.OffshootName()
}

func (r *Elasticsearch) MasterServiceName() string {
	return fmt.Sprintf("%v-master", r.ServiceName())
}

func (r Elasticsearch) ServiceMonitorName() string {
	return fmt.Sprintf("kubedb-%s-%s", r.Namespace, r.Name)
}

func (r Elasticsearch) Path() string {
	return fmt.Sprintf("/kubedb.com/v1alpha1/namespaces/%s/%s/%s/metrics", r.Namespace, r.ResourcePlural(), r.Name)
}

func (r Elasticsearch) Scheme() string {
	return ""
}

func (r *Elasticsearch) StatsAccessor() api.StatsAccessor {
	return r
}

func (e *Elasticsearch) GetMonitoringVendor() string {
	if e.Spec.Monitor != nil {
		return e.Spec.Monitor.Agent.Vendor()
	}
	return ""
}

func (r Elasticsearch) CustomResourceDefinition() *crd_api.CustomResourceDefinition {
	return crdutils.NewCustomResourceDefinition(crdutils.Config{
		Group:         SchemeGroupVersion.Group,
		Version:       SchemeGroupVersion.Version,
		Plural:        ResourcePluralElasticsearch,
		Singular:      ResourceSingularElasticsearch,
		Kind:          ResourceKindElasticsearch,
		ShortNames:    []string{ResourceCodeElasticsearch},
		ResourceScope: string(apiextensions.NamespaceScoped),
		Labels: crdutils.Labels{
			LabelsMap: map[string]string{"app": "kubedb"},
		},
		SpecDefinitionName:    "github.com/kubedb/apimachinery/apis/kubedb/v1alpha1.Elasticsearch",
		EnableValidation:      true,
		GetOpenAPIDefinitions: GetOpenAPIDefinitions,
	}, setNameSchema)
}
<commit_msg>Add SearchGuardDisabled() to ES (#229)<commit_after>package v1alpha1

import (
	"fmt"
	"strings"

	"github.com/appscode/kube-mon/api"
	crdutils "github.com/appscode/kutil/apiextensions/v1beta1"
	"github.com/appscode/kutil/meta"
	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
	crd_api "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
)

func (e Elasticsearch) OffshootName() string {
	return e.Name
}

func (e Elasticsearch) OffshootLabels() map[string]string {
	return map[string]string{
		LabelDatabaseKind: ResourceKindElasticsearch,
		LabelDatabaseName: e.Name,
	}
}

func (e Elasticsearch) StatefulSetLabels() map[string]string {
	labels := e.OffshootLabels()
	for key, val := range e.Labels {
		if !strings.HasPrefix(key, GenericKey+"/") && !strings.HasPrefix(key, ElasticsearchKey+"/") {
			labels[key] = val
		}
	}
	return labels
}

func (e Elasticsearch) StatefulSetAnnotations() map[string]string {
	annotations := make(map[string]string)
	for key, val := range e.Annotations {
		if !strings.HasPrefix(key, GenericKey+"/") && !strings.HasPrefix(key, ElasticsearchKey+"/") {
			annotations[key] = val
		}
	}
	return annotations
}

var _ ResourceInfo = &Elasticsearch{}

func (e Elasticsearch) ResourceShortCode() string {
	return ResourceCodeElasticsearch
}

func (e Elasticsearch) ResourceKind() string {
	return ResourceKindElasticsearch
}

func (e Elasticsearch) ResourceSingular() string {
	return ResourceSingularElasticsearch
}

func (e Elasticsearch) ResourcePlural() string {
	return ResourcePluralElasticsearch
}

func (r Elasticsearch) ServiceName() string {
	return r.OffshootName()
}

func (r *Elasticsearch) MasterServiceName() string {
	return fmt.Sprintf("%v-master", r.ServiceName())
}

func (r Elasticsearch) ServiceMonitorName() string {
	return fmt.Sprintf("kubedb-%s-%s", r.Namespace, r.Name)
}

func (r Elasticsearch) Path() string {
	return fmt.Sprintf("/kubedb.com/v1alpha1/namespaces/%s/%s/%s/metrics", r.Namespace, r.ResourcePlural(), r.Name)
}

func (r Elasticsearch) Scheme() string {
	return ""
}

func (r *Elasticsearch) StatsAccessor() api.StatsAccessor {
	return r
}

func (e *Elasticsearch) GetMonitoringVendor() string {
	if e.Spec.Monitor != nil {
		return e.Spec.Monitor.Agent.Vendor()
	}
	return ""
}

func (r Elasticsearch) CustomResourceDefinition() *crd_api.CustomResourceDefinition {
	return crdutils.NewCustomResourceDefinition(crdutils.Config{
		Group:         SchemeGroupVersion.Group,
		Version:       SchemeGroupVersion.Version,
		Plural:        ResourcePluralElasticsearch,
		Singular:      ResourceSingularElasticsearch,
		Kind:          ResourceKindElasticsearch,
		ShortNames:    []string{ResourceCodeElasticsearch},
		ResourceScope: string(apiextensions.NamespaceScoped),
		Labels: crdutils.Labels{
			LabelsMap: map[string]string{"app": "kubedb"},
		},
		SpecDefinitionName:    "github.com/kubedb/apimachinery/apis/kubedb/v1alpha1.Elasticsearch",
		EnableValidation:      true,
		GetOpenAPIDefinitions: GetOpenAPIDefinitions,
	}, setNameSchema)
}

const (
	ESSearchGuardDisabled = ElasticsearchKey + "/searchguard-disabled"
)

func (r Elasticsearch) SearchGuardDisabled() bool {
	v, _ := meta.GetBoolValue(r.Annotations, ESSearchGuardDisabled)
	return v
}
<|endoftext|>


<commit_before>package maildir

import (

  "github.com/golang/glog"

  "fmt"
  "io"
  "os"
  "path/filepath"
  "time"
)

// maildir mailbox protocol

type MailDir string

func (d MailDir) String() (str string) {
  str = string(d)
  return
}

func (d MailDir) Create() (err error) {
  err = os.Mkdir(d.String(), 0700)
  err = os.Mkdir(filepath.Join(d.String(), "new"), 0700)
  err = os.Mkdir(filepath.Join(d.String(), "cur"), 0700)
  err = os.Mkdir(filepath.Join(d.String(), "tmp"), 0700)
  return
}

// get a string of the current filename to use
func (d MailDir) File() (fname string) {
  hostname, err := os.Hostname()
  if err == nil {
    fname = fmt.Sprintf("%d.%d.%s", time.Now().Unix(), os.Getpid(), hostname)
  } else {
    glog.Fatal("hostname() call failed", err)
  }
  return
}

func (d MailDir) TempFile() (fname string) {
  fname = d.Temp(d.File())
  return
}

func (d MailDir) Temp(fname string) (f string) {
  f = filepath.Join(d.String(), "tmp", fname)
  return
}

func (d MailDir) NewFile() (fname string) {
  fname = d.New(d.File())
  return
}

func (d MailDir) New(fname string) (f string) {
  f = filepath.Join(d.String(), "new", fname)
  return
}

// deliver mail to this maildir
func (d MailDir) Deliver(body io.Reader) (err error) {
  var oldwd string
  oldwd, err = os.Getwd()
  if err == nil {
    // no error getting working directory, let's begin

    // when done chdir to previous directory
    defer func(){
      err := os.Chdir(oldwd)
      if err != nil {
        glog.Fatal("chdir failed", err)
      }
    }()
    // chdir to maildir
    err = os.Chdir(d.String())
    if err == nil {
      fname := d.File()
      for {
        _, err = os.Stat(d.Temp(fname))
        if os.IsNotExist(err) {
          break
        }
        time.Sleep(time.Second * 2)
        fname = d.TempFile()
      }
      // set err to nil
      err = nil
      var f *os.File
      // create tmp file
      f, err = os.Create(d.Temp(fname))
      if err == nil {
        // success creation
        err = f.Close()
      }
      // try writing file
      if err == nil {
        f, err = os.OpenFile(d.Temp(fname), os.O_CREATE | os.O_WRONLY, 0600)
        if err == nil {
          // write body
          _, err = io.CopyBuffer(f, body, nil)
          f.Close()
          if err == nil {
            // now symlink
            err = os.Symlink(filepath.Join("tmp", fname), filepath.Join("new", fname))
            // if err is nil it's delivered
          }
        }
      }
    }
  }
  return
}

<commit_msg>use MailDir.File not MailDir.TempFile<commit_after>package maildir

import (

  "github.com/golang/glog"

  "fmt"
  "io"
  "os"
  "path/filepath"
  "time"
)

// maildir mailbox protocol

type MailDir string

func (d MailDir) String() (str string) {
  str = string(d)
  return
}

func (d MailDir) Create() (err error) {
  err = os.Mkdir(d.String(), 0700)
  err = os.Mkdir(filepath.Join(d.String(), "new"), 0700)
  err = os.Mkdir(filepath.Join(d.String(), "cur"), 0700)
  err = os.Mkdir(filepath.Join(d.String(), "tmp"), 0700)
  return
}

// get a string of the current filename to use
func (d MailDir) File() (fname string) {
  hostname, err := os.Hostname()
  if err == nil {
    fname = fmt.Sprintf("%d.%d.%s", time.Now().Unix(), os.Getpid(), hostname)
  } else {
    glog.Fatal("hostname() call failed", err)
  }
  return
}

func (d MailDir) TempFile() (fname string) {
  fname = d.Temp(d.File())
  return
}

func (d MailDir) Temp(fname string) (f string) {
  f = filepath.Join(d.String(), "tmp", fname)
  return
}

func (d MailDir) NewFile() (fname string) {
  fname = d.New(d.File())
  return
}

func (d MailDir) New(fname string) (f string) {
  f = filepath.Join(d.String(), "new", fname)
  return
}

// deliver mail to this maildir
func (d MailDir) Deliver(body io.Reader) (err error) {
  var oldwd string
  oldwd, err = os.Getwd()
  if err == nil {
    // no error getting working directory, let's begin

    // when done chdir to previous directory
    defer func(){
      err := os.Chdir(oldwd)
      if err != nil {
        glog.Fatal("chdir failed", err)
      }
    }()
    // chdir to maildir
    err = os.Chdir(d.String())
    if err == nil {
      fname := d.File()
      for {
        _, err = os.Stat(d.Temp(fname))
        if os.IsNotExist(err) {
          break
        }
        time.Sleep(time.Second * 2)
        fname = d.File()
      }
      // set err to nil
      err = nil
      var f *os.File
      // create tmp file
      f, err = os.Create(d.Temp(fname))
      if err == nil {
        // success creation
        err = f.Close()
      }
      // try writing file
      if err == nil {
        f, err = os.OpenFile(d.Temp(fname), os.O_CREATE | os.O_WRONLY, 0600)
        if err == nil {
          // write body
          _, err = io.CopyBuffer(f, body, nil)
          f.Close()
          if err == nil {
            // now symlink
            err = os.Symlink(filepath.Join("tmp", fname), filepath.Join("new", fname))
            // if err is nil it's delivered
          }
        }
      }
    }
  }
  return
}

<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package run

import (
	gocontext "context"
	"errors"
	"strings"

	"github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options"
	"github.com/containerd/console"
	"github.com/containerd/containerd"
	"github.com/containerd/containerd/cmd/ctr/commands"
	"github.com/containerd/containerd/oci"
	"github.com/containerd/containerd/pkg/netns"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/sirupsen/logrus"
	"github.com/urfave/cli"
)

var platformRunFlags = []cli.Flag{
	cli.BoolFlag{
		Name:  "isolated",
		Usage: "run the container with vm isolation",
	},
	cli.StringFlag{
		Name:  "user",
		Usage: "run the container as the specified user",
	},
}

// NewContainer creates a new container
func NewContainer(ctx gocontext.Context, client *containerd.Client, context *cli.Context) (containerd.Container, error) {
	var (
		id    string
		opts  []oci.SpecOpts
		cOpts []containerd.NewContainerOpts
		spec  containerd.NewContainerOpts

		config = context.IsSet("config")
	)

	if config {
		id = context.Args().First()
		opts = append(opts, oci.WithSpecFromFile(context.String("config")))
		cOpts = append(cOpts, containerd.WithContainerLabels(commands.LabelArgs(context.StringSlice("label"))))
	} else {
		var (
			ref  = context.Args().First()
			args = context.Args()[2:]
		)

		id = context.Args().Get(1)
		snapshotter := context.String("snapshotter")
		if snapshotter == "windows-lcow" {
			opts = append(opts, oci.WithDefaultSpecForPlatform("linux/amd64"))
			// Clear the rootfs section.
			opts = append(opts, oci.WithRootFSPath(""))
		} else {
			opts = append(opts, oci.WithDefaultSpec())
			opts = append(opts, oci.WithWindowNetworksAllowUnqualifiedDNSQuery())
			opts = append(opts, oci.WithWindowsIgnoreFlushesDuringBoot())
		}
		if ef := context.String("env-file"); ef != "" {
			opts = append(opts, oci.WithEnvFile(ef))
		}
		opts = append(opts, oci.WithEnv(context.StringSlice("env")))
		opts = append(opts, withMounts(context))

		image, err := client.GetImage(ctx, ref)
		if err != nil {
			return nil, err
		}
		unpacked, err := image.IsUnpacked(ctx, snapshotter)
		if err != nil {
			return nil, err
		}
		if !unpacked {
			if err := image.Unpack(ctx, snapshotter); err != nil {
				return nil, err
			}
		}
		opts = append(opts, oci.WithImageConfig(image))
		labels := buildLabels(commands.LabelArgs(context.StringSlice("label")), image.Labels())
		cOpts = append(cOpts,
			containerd.WithImage(image),
			containerd.WithImageConfigLabels(image),
			containerd.WithSnapshotter(snapshotter),
			containerd.WithNewSnapshot(id, image),
			containerd.WithAdditionalContainerLabels(labels))

		if len(args) > 0 {
			opts = append(opts, oci.WithProcessArgs(args...))
		}
		if cwd := context.String("cwd"); cwd != "" {
			opts = append(opts, oci.WithProcessCwd(cwd))
		}
		if user := context.String("user"); user != "" {
			opts = append(opts, oci.WithUsername(user))
		}
		if context.Bool("tty") {
			opts = append(opts, oci.WithTTY)

			con := console.Current()
			size, err := con.Size()
			if err != nil {
				logrus.WithError(err).Error("console size")
			}
			opts = append(opts, oci.WithTTYSize(int(size.Width), int(size.Height)))
		}
		if context.Bool("net-host") {
			return nil, errors.New("Cannot use host mode networking with Windows containers")
		}
		if context.Bool("cni") {
			ns, err := netns.NewNetNS("")
			if err != nil {
				return nil, err
			}
			opts = append(opts, oci.WithWindowsNetworkNamespace(ns.GetPath()))
		}
		if context.Bool("isolated") {
			opts = append(opts, oci.WithWindowsHyperV)
		}
		limit := context.Uint64("memory-limit")
		if limit != 0 {
			opts = append(opts, oci.WithMemoryLimit(limit))
		}
		ccount := context.Uint64("cpu-count")
		if ccount != 0 {
			opts = append(opts, oci.WithWindowsCPUCount(ccount))
		}
		cshares := context.Uint64("cpu-shares")
		if cshares != 0 {
			opts = append(opts, oci.WithWindowsCPUShares(uint16(cshares)))
		}
		cmax := context.Uint64("cpu-max")
		if cmax != 0 {
			opts = append(opts, oci.WithWindowsCPUMaximum(uint16(cmax)))
		}
		for _, dev := range context.StringSlice("device") {
			parts := strings.Split(dev, "://")
			if len(parts) != 2 {
				return nil, errors.New("devices must be in the format IDType://ID")
			}
			if parts[0] == "" {
				return nil, errors.New("devices must have a non-empty IDType")
			}
			opts = append(opts, oci.WithWindowsDevice(parts[0], parts[1]))
		}
	}

	runtime := context.String("runtime")
	var runtimeOpts interface{}
	if runtime == "io.containerd.runhcs.v1" {
		runtimeOpts = &options.Options{
			Debug: context.GlobalBool("debug"),
		}
	}
	cOpts = append(cOpts, containerd.WithRuntime(runtime, runtimeOpts))

	var s specs.Spec
	spec = containerd.WithSpec(&s, opts...)

	cOpts = append(cOpts, spec)

	return client.NewContainer(ctx, id, cOpts...)
}

func getNewTaskOpts(_ *cli.Context) []containerd.NewTaskOpts {
	return nil
}

func getNetNSPath(ctx gocontext.Context, t containerd.Task) (string, error) {
	s, err := t.Spec(ctx)
	if err != nil {
		return "", err
	}
	if s.Windows == nil || s.Windows.Network == nil {
		return "", nil
	}
	return s.Windows.Network.NetworkNamespace, nil
}
<commit_msg>Forward ctr snapshotter flags on Windows<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package run

import (
	gocontext "context"
	"errors"
	"strings"

	"github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options"
	"github.com/containerd/console"
	"github.com/containerd/containerd"
	"github.com/containerd/containerd/cmd/ctr/commands"
	"github.com/containerd/containerd/oci"
	"github.com/containerd/containerd/pkg/netns"
	"github.com/containerd/containerd/snapshots"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/sirupsen/logrus"
	"github.com/urfave/cli"
)

var platformRunFlags = []cli.Flag{
	cli.BoolFlag{
		Name:  "isolated",
		Usage: "run the container with vm isolation",
	},
	cli.StringFlag{
		Name:  "user",
		Usage: "run the container as the specified user",
	},
}

// NewContainer creates a new container
func NewContainer(ctx gocontext.Context, client *containerd.Client, context *cli.Context) (containerd.Container, error) {
	var (
		id    string
		opts  []oci.SpecOpts
		cOpts []containerd.NewContainerOpts
		spec  containerd.NewContainerOpts

		config = context.IsSet("config")
	)

	if config {
		id = context.Args().First()
		opts = append(opts, oci.WithSpecFromFile(context.String("config")))
		cOpts = append(cOpts, containerd.WithContainerLabels(commands.LabelArgs(context.StringSlice("label"))))
	} else {
		var (
			ref  = context.Args().First()
			args = context.Args()[2:]
		)

		id = context.Args().Get(1)
		snapshotter := context.String("snapshotter")
		if snapshotter == "windows-lcow" {
			opts = append(opts, oci.WithDefaultSpecForPlatform("linux/amd64"))
			// Clear the rootfs section.
			opts = append(opts, oci.WithRootFSPath(""))
		} else {
			opts = append(opts, oci.WithDefaultSpec())
			opts = append(opts, oci.WithWindowNetworksAllowUnqualifiedDNSQuery())
			opts = append(opts, oci.WithWindowsIgnoreFlushesDuringBoot())
		}
		if ef := context.String("env-file"); ef != "" {
			opts = append(opts, oci.WithEnvFile(ef))
		}
		opts = append(opts, oci.WithEnv(context.StringSlice("env")))
		opts = append(opts, withMounts(context))

		image, err := client.GetImage(ctx, ref)
		if err != nil {
			return nil, err
		}
		unpacked, err := image.IsUnpacked(ctx, snapshotter)
		if err != nil {
			return nil, err
		}
		if !unpacked {
			if err := image.Unpack(ctx, snapshotter); err != nil {
				return nil, err
			}
		}
		opts = append(opts, oci.WithImageConfig(image))
		labels := buildLabels(commands.LabelArgs(context.StringSlice("label")), image.Labels())
		cOpts = append(cOpts,
			containerd.WithImage(image),
			containerd.WithImageConfigLabels(image),
			containerd.WithSnapshotter(snapshotter),
			containerd.WithNewSnapshot(
				id,
				image,
				snapshots.WithLabels(commands.LabelArgs(context.StringSlice("snapshotter-label")))),
			containerd.WithAdditionalContainerLabels(labels))

		if len(args) > 0 {
			opts = append(opts, oci.WithProcessArgs(args...))
		}
		if cwd := context.String("cwd"); cwd != "" {
			opts = append(opts, oci.WithProcessCwd(cwd))
		}
		if user := context.String("user"); user != "" {
			opts = append(opts, oci.WithUsername(user))
		}
		if context.Bool("tty") {
			opts = append(opts, oci.WithTTY)

			con := console.Current()
			size, err := con.Size()
			if err != nil {
				logrus.WithError(err).Error("console size")
			}
			opts = append(opts, oci.WithTTYSize(int(size.Width), int(size.Height)))
		}
		if context.Bool("net-host") {
			return nil, errors.New("Cannot use host mode networking with Windows containers")
		}
		if context.Bool("cni") {
			ns, err := netns.NewNetNS("")
			if err != nil {
				return nil, err
			}
			opts = append(opts, oci.WithWindowsNetworkNamespace(ns.GetPath()))
		}
		if context.Bool("isolated") {
			opts = append(opts, oci.WithWindowsHyperV)
		}
		limit := context.Uint64("memory-limit")
		if limit != 0 {
			opts = append(opts, oci.WithMemoryLimit(limit))
		}
		ccount := context.Uint64("cpu-count")
		if ccount != 0 {
			opts = append(opts, oci.WithWindowsCPUCount(ccount))
		}
		cshares := context.Uint64("cpu-shares")
		if cshares != 0 {
			opts = append(opts, oci.WithWindowsCPUShares(uint16(cshares)))
		}
		cmax := context.Uint64("cpu-max")
		if cmax != 0 {
			opts = append(opts, oci.WithWindowsCPUMaximum(uint16(cmax)))
		}
		for _, dev := range context.StringSlice("device") {
			parts := strings.Split(dev, "://")
			if len(parts) != 2 {
				return nil, errors.New("devices must be in the format IDType://ID")
			}
			if parts[0] == "" {
				return nil, errors.New("devices must have a non-empty IDType")
			}
			opts = append(opts, oci.WithWindowsDevice(parts[0], parts[1]))
		}
	}

	runtime := context.String("runtime")
	var runtimeOpts interface{}
	if runtime == "io.containerd.runhcs.v1" {
		runtimeOpts = &options.Options{
			Debug: context.GlobalBool("debug"),
		}
	}
	cOpts = append(cOpts, containerd.WithRuntime(runtime, runtimeOpts))

	var s specs.Spec
	spec = containerd.WithSpec(&s, opts...)

	cOpts = append(cOpts, spec)

	return client.NewContainer(ctx, id, cOpts...)
}

func getNewTaskOpts(_ *cli.Context) []containerd.NewTaskOpts {
	return nil
}

func getNetNSPath(ctx gocontext.Context, t containerd.Task) (string, error) {
	s, err := t.Spec(ctx)
	if err != nil {
		return "", err
	}
	if s.Windows == nil || s.Windows.Network == nil {
		return "", nil
	}
	return s.Windows.Network.NetworkNamespace, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"strings"
	"syscall"

	"github.com/docker/docker/pkg/reexec"
)

func init() {
	reexec.Register("namespaced", namespaced)

	if reexec.Init() {
		os.Exit(0)
	}
}

func namespaced() {
	dataDir := os.Args[1]

	cmd := exec.Command(os.Args[4], os.Args[5:]...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	mustRun(exec.Command("mount", "--make-slave", dataDir))

	if err := cmd.Run(); err != nil {
		fmt.Printf("%s: %s\n", cmd.Path, err)
		os.Exit(1)
	}
}

func main() {
	dataDir := os.Args[1]
	realGraphDir := os.Args[2]
	graphDir := os.Args[3]

	mustBindMountOnce(dataDir, dataDir)
	mustRun(exec.Command("mount", "--make-shared", dataDir))

	mustBindMountOnce(realGraphDir, graphDir)

	reexecInNamespace(os.Args[1:]...)
}

func reexecInNamespace(args ...string) {
	reexecArgs := append([]string{"namespaced"}, args...)
	cmd := reexec.Command(reexecArgs...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.SysProcAttr = &syscall.SysProcAttr{
		Cloneflags: syscall.CLONE_NEWNS,
	}

	if err := cmd.Run(); err != nil {
		fmt.Printf("exec secret garden: %s\n", err)
		os.Exit(1)
	}
}

func mustBindMountOnce(srcDir, dstDir string) {
	mounts := mustRun(exec.Command("mount"))
	alreadyMounted := strings.Contains(mounts, fmt.Sprintf("%s on %s", srcDir, dstDir))

	if !alreadyMounted {
		mustRun(exec.Command("mount", "--bind", srcDir, dstDir))
	}
}

func run(cmd *exec.Cmd) error {
	if out, err := cmd.CombinedOutput(); err != nil {
		return fmt.Errorf("%s: %s: %s", cmd.Path, err, string(out))
	}

	return nil
}

func mustRun(cmd *exec.Cmd) string {
	out, err := cmd.CombinedOutput()
	if err != nil {
		panic(fmt.Sprintf("%s: %s: %s", cmd.Path, err, string(out)))
	}
	return string(out)
}
<commit_msg>Add an extra new line to be consistent<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"strings"
	"syscall"

	"github.com/docker/docker/pkg/reexec"
)

func init() {
	reexec.Register("namespaced", namespaced)

	if reexec.Init() {
		os.Exit(0)
	}
}

func namespaced() {
	dataDir := os.Args[1]

	cmd := exec.Command(os.Args[4], os.Args[5:]...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	mustRun(exec.Command("mount", "--make-slave", dataDir))

	if err := cmd.Run(); err != nil {
		fmt.Printf("%s: %s\n", cmd.Path, err)
		os.Exit(1)
	}
}

func main() {
	dataDir := os.Args[1]
	realGraphDir := os.Args[2]
	graphDir := os.Args[3]

	mustBindMountOnce(dataDir, dataDir)
	mustRun(exec.Command("mount", "--make-shared", dataDir))

	mustBindMountOnce(realGraphDir, graphDir)

	reexecInNamespace(os.Args[1:]...)
}

func reexecInNamespace(args ...string) {
	reexecArgs := append([]string{"namespaced"}, args...)
	cmd := reexec.Command(reexecArgs...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.SysProcAttr = &syscall.SysProcAttr{
		Cloneflags: syscall.CLONE_NEWNS,
	}

	if err := cmd.Run(); err != nil {
		fmt.Printf("exec secret garden: %s\n", err)
		os.Exit(1)
	}
}

func mustBindMountOnce(srcDir, dstDir string) {
	mounts := mustRun(exec.Command("mount"))
	alreadyMounted := strings.Contains(mounts, fmt.Sprintf("%s on %s", srcDir, dstDir))

	if !alreadyMounted {
		mustRun(exec.Command("mount", "--bind", srcDir, dstDir))
	}
}

func run(cmd *exec.Cmd) error {
	if out, err := cmd.CombinedOutput(); err != nil {
		return fmt.Errorf("%s: %s: %s", cmd.Path, err, string(out))
	}

	return nil
}

func mustRun(cmd *exec.Cmd) string {
	out, err := cmd.CombinedOutput()
	if err != nil {
		panic(fmt.Sprintf("%s: %s: %s", cmd.Path, err, string(out)))
	}

	return string(out)
}
<|endoftext|>


<commit_before>package cluster

import (
	"github.com/smancke/guble/protocol"

	log "github.com/Sirupsen/logrus"
	"github.com/hashicorp/memberlist"

	"errors"
	"fmt"
	"io/ioutil"
)

// Config is a struct used by the local node when creating and running the guble cluster
type Config struct {
	ID                   int
	Host                 string
	Port                 int
	Remotes              []string
	HealthScoreThreshold int
}

type MessageHandler interface {
	HandleMessage(message *protocol.Message) error
}

// Cluster is a struct for managing the `local view` of the guble cluster, as seen by a node.
type Cluster struct {
	// Pointer to a Config struct, based on which the Cluster node is created and runs.
	Config *Config

	Name string

	// MessageHandler is used for dispatching messages received by this node.
	// Should be set after the node is created with New(), and before Start().
	MessageHandler MessageHandler

	memberlist *memberlist.Memberlist
	broadcasts [][]byte
}

//New returns a new instance of the cluster, created using the given Config.
func New(config *Config) (*Cluster, error) {
	c := &Cluster{Config: config, Name: fmt.Sprintf("%d", config.ID)}

	memberlistConfig := memberlist.DefaultLANConfig()
	memberlistConfig.Name = c.Name
	memberlistConfig.BindAddr = config.Host
	memberlistConfig.BindPort = config.Port
	memberlistConfig.Events = &eventDelegate{}

	//TODO Cosmin temporarily disabling any logging from memberlist, we might want to enable it again using logrus?
	memberlistConfig.LogOutput = ioutil.Discard

	memberlist, err := memberlist.Create(memberlistConfig)
	if err != nil {
		logger.WithField("error", err).Error("Error when creating the internal memberlist of the cluster")
		return nil, err
	}
	c.memberlist = memberlist
	memberlistConfig.Delegate = c
	memberlistConfig.Conflict = c
	return c, nil
}

// Start the cluster module.
func (cluster *Cluster) Start() error {
	logger.WithField("remotes", cluster.Config.Remotes).Debug("Starting Cluster")
	if cluster.MessageHandler == nil {
		errorMessage := "There should be a valid MessageHandler already set-up"
		logger.Error(errorMessage)
		return errors.New(errorMessage)
	}
	num, err := cluster.memberlist.Join(cluster.Config.Remotes)
	if err != nil {
		logger.WithField("error", err).Error("Error when this node wanted to join the cluster")
		return err
	}
	if num == 0 {
		errorMessage := "No remote hosts were successfuly contacted when this node wanted to join the cluster"
		logger.Error(errorMessage)
		return errors.New(errorMessage)
	}
	return nil
}

// Stop the cluster module.
func (cluster *Cluster) Stop() error {
	return cluster.memberlist.Shutdown()
}

// Check returns a non-nil error if the health status of the cluster (as seen by this node) is not perfect.
func (cluster *Cluster) Check() error {
	if healthScore := cluster.memberlist.GetHealthScore(); healthScore > cluster.Config.HealthScoreThreshold {
		errorMessage := "Cluster Health Score is not perfect"
		logger.WithField("healthScore", healthScore).Error(errorMessage)
		return errors.New(errorMessage)
	}
	return nil
}

// NotifyMsg is invoked each time a message is received by this node of the cluster; it decodes and dispatches the messages.
func (cluster *Cluster) NotifyMsg(msg []byte) {
	logger.WithField("msgAsBytes", msg).Debug("NotifyMsg")

	cmsg, err := decode(msg)
	if err != nil {
		logger.WithField("err", err).Error("Decoding of cluster message failed")
		return
	}
	logger.WithFields(log.Fields{
		"senderNodeID": cmsg.NodeID,
		"type":         cmsg.Type,
		"body":         string(cmsg.Body),
	}).Debug("NotifyMsg: Received cluster message")

	if cluster.MessageHandler != nil && cmsg.Type == gubleMessage {
		pMessage, err := protocol.ParseMessage(cmsg.Body)
		if err != nil {
			logger.WithField("err", err).Error("Parsing of guble-message contained in cluster-message failed")
			return
		}
		cluster.MessageHandler.HandleMessage(pMessage)
	}
}

func (cluster *Cluster) GetBroadcasts(overhead, limit int) [][]byte {
	b := cluster.broadcasts
	cluster.broadcasts = nil
	return b
}

func (cluster *Cluster) NodeMeta(limit int) []byte {
	return nil
}

func (cluster *Cluster) LocalState(join bool) []byte {
	return nil
}

func (cluster *Cluster) MergeRemoteState(s []byte, join bool) {
}

func (cluster *Cluster) NotifyConflict(existing, other *memberlist.Node) {
	logger.WithFields(log.Fields{
		"existing": *existing,
		"other":    *other,
	}).Panic("NotifyConflict")
}

// BroadcastString broadcasts a string to all the other nodes in the guble cluster
func (cluster *Cluster) BroadcastString(sMessage *string) error {
	logger.WithField("string", sMessage).Debug("BroadcastString")
	cMessage := &message{
		NodeID: cluster.Config.ID,
		Type:   stringMessage,
		Body:   []byte(*sMessage),
	}
	return cluster.broadcastClusterMessage(cMessage)
}

// BroadcastMessage broadcasts a guble-protocol-message to all the other nodes in the guble cluster
func (cluster *Cluster) BroadcastMessage(pMessage *protocol.Message) error {
	logger.WithField("message", pMessage).Debug("BroadcastMessage")
	cMessage := &message{
		NodeID: cluster.Config.ID,
		Type:   gubleMessage,
		Body:   pMessage.Bytes(),
	}
	return cluster.broadcastClusterMessage(cMessage)
}

func (cluster *Cluster) broadcastClusterMessage(cMessage *message) error {
	if cMessage == nil {
		errorMessage := "Could not broadcast a nil cluster-message"
		logger.Error(errorMessage)
		return errors.New(errorMessage)
	}
	cMessageBytes, err := cMessage.encode()
	if err != nil {
		logger.WithField("err", err).Error("Could not encode and broadcast cluster-message")
		return err
	}
	for _, node := range cluster.memberlist.Members() {
		if cluster.Name == node.Name {
			continue
		}
		logger.WithField("nodeName", node.Name).Debug("Sending cluster-message to a node")
		err := cluster.memberlist.SendToTCP(node, cMessageBytes)
		if err != nil {
			logger.WithFields(log.Fields{
				"err":  err,
				"node": node,
			}).Error("Error sending cluster-message to a node")
			return err
		}
	}
	return nil
}
<commit_msg>making name as private to Cluster<commit_after>package cluster

import (
	"github.com/smancke/guble/protocol"

	log "github.com/Sirupsen/logrus"
	"github.com/hashicorp/memberlist"

	"errors"
	"fmt"
	"io/ioutil"
)

// Config is a struct used by the local node when creating and running the guble cluster
type Config struct {
	ID                   int
	Host                 string
	Port                 int
	Remotes              []string
	HealthScoreThreshold int
}

type MessageHandler interface {
	HandleMessage(message *protocol.Message) error
}

// Cluster is a struct for managing the `local view` of the guble cluster, as seen by a node.
type Cluster struct {
	// Pointer to a Config struct, based on which the Cluster node is created and runs.
	Config *Config

	// MessageHandler is used for dispatching messages received by this node.
	// Should be set after the node is created with New(), and before Start().
	MessageHandler MessageHandler

	name       string
	memberlist *memberlist.Memberlist
	broadcasts [][]byte
}

//New returns a new instance of the cluster, created using the given Config.
func New(config *Config) (*Cluster, error) {
	c := &Cluster{Config: config, name: fmt.Sprintf("%d", config.ID)}

	memberlistConfig := memberlist.DefaultLANConfig()
	memberlistConfig.Name = c.name
	memberlistConfig.BindAddr = config.Host
	memberlistConfig.BindPort = config.Port
	memberlistConfig.Events = &eventDelegate{}

	//TODO Cosmin temporarily disabling any logging from memberlist, we might want to enable it again using logrus?
	memberlistConfig.LogOutput = ioutil.Discard

	memberlist, err := memberlist.Create(memberlistConfig)
	if err != nil {
		logger.WithField("error", err).Error("Error when creating the internal memberlist of the cluster")
		return nil, err
	}
	c.memberlist = memberlist
	memberlistConfig.Delegate = c
	memberlistConfig.Conflict = c
	return c, nil
}

// Start the cluster module.
func (cluster *Cluster) Start() error {
	logger.WithField("remotes", cluster.Config.Remotes).Debug("Starting Cluster")
	if cluster.MessageHandler == nil {
		errorMessage := "There should be a valid MessageHandler already set-up"
		logger.Error(errorMessage)
		return errors.New(errorMessage)
	}
	num, err := cluster.memberlist.Join(cluster.Config.Remotes)
	if err != nil {
		logger.WithField("error", err).Error("Error when this node wanted to join the cluster")
		return err
	}
	if num == 0 {
		errorMessage := "No remote hosts were successfuly contacted when this node wanted to join the cluster"
		logger.Error(errorMessage)
		return errors.New(errorMessage)
	}
	return nil
}

// Stop the cluster module.
func (cluster *Cluster) Stop() error {
	return cluster.memberlist.Shutdown()
}

// Check returns a non-nil error if the health status of the cluster (as seen by this node) is not perfect.
func (cluster *Cluster) Check() error {
	if healthScore := cluster.memberlist.GetHealthScore(); healthScore > cluster.Config.HealthScoreThreshold {
		errorMessage := "Cluster Health Score is not perfect"
		logger.WithField("healthScore", healthScore).Error(errorMessage)
		return errors.New(errorMessage)
	}
	return nil
}

// NotifyMsg is invoked each time a message is received by this node of the cluster; it decodes and dispatches the messages.
func (cluster *Cluster) NotifyMsg(msg []byte) {
	logger.WithField("msgAsBytes", msg).Debug("NotifyMsg")

	cmsg, err := decode(msg)
	if err != nil {
		logger.WithField("err", err).Error("Decoding of cluster message failed")
		return
	}
	logger.WithFields(log.Fields{
		"senderNodeID": cmsg.NodeID,
		"type":         cmsg.Type,
		"body":         string(cmsg.Body),
	}).Debug("NotifyMsg: Received cluster message")

	if cluster.MessageHandler != nil && cmsg.Type == gubleMessage {
		pMessage, err := protocol.ParseMessage(cmsg.Body)
		if err != nil {
			logger.WithField("err", err).Error("Parsing of guble-message contained in cluster-message failed")
			return
		}
		cluster.MessageHandler.HandleMessage(pMessage)
	}
}

func (cluster *Cluster) GetBroadcasts(overhead, limit int) [][]byte {
	b := cluster.broadcasts
	cluster.broadcasts = nil
	return b
}

func (cluster *Cluster) NodeMeta(limit int) []byte {
	return nil
}

func (cluster *Cluster) LocalState(join bool) []byte {
	return nil
}

func (cluster *Cluster) MergeRemoteState(s []byte, join bool) {
}

func (cluster *Cluster) NotifyConflict(existing, other *memberlist.Node) {
	logger.WithFields(log.Fields{
		"existing": *existing,
		"other":    *other,
	}).Panic("NotifyConflict")
}

// BroadcastString broadcasts a string to all the other nodes in the guble cluster
func (cluster *Cluster) BroadcastString(sMessage *string) error {
	logger.WithField("string", sMessage).Debug("BroadcastString")
	cMessage := &message{
		NodeID: cluster.Config.ID,
		Type:   stringMessage,
		Body:   []byte(*sMessage),
	}
	return cluster.broadcastClusterMessage(cMessage)
}

// BroadcastMessage broadcasts a guble-protocol-message to all the other nodes in the guble cluster
func (cluster *Cluster) BroadcastMessage(pMessage *protocol.Message) error {
	logger.WithField("message", pMessage).Debug("BroadcastMessage")
	cMessage := &message{
		NodeID: cluster.Config.ID,
		Type:   gubleMessage,
		Body:   pMessage.Bytes(),
	}
	return cluster.broadcastClusterMessage(cMessage)
}

func (cluster *Cluster) broadcastClusterMessage(cMessage *message) error {
	if cMessage == nil {
		errorMessage := "Could not broadcast a nil cluster-message"
		logger.Error(errorMessage)
		return errors.New(errorMessage)
	}
	cMessageBytes, err := cMessage.encode()
	if err != nil {
		logger.WithField("err", err).Error("Could not encode and broadcast cluster-message")
		return err
	}
	for _, node := range cluster.memberlist.Members() {
		if cluster.name == node.Name {
			continue
		}
		logger.WithField("nodeName", node.Name).Debug("Sending cluster-message to a node")
		err := cluster.memberlist.SendToTCP(node, cMessageBytes)
		if err != nil {
			logger.WithFields(log.Fields{
				"err":  err,
				"node": node,
			}).Error("Error sending cluster-message to a node")
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package etcdhttp

import (
	"context"
	"encoding/json"
	"net/http"
	"time"

	"go.etcd.io/etcd/etcdserver"
	"go.etcd.io/etcd/etcdserver/etcdserverpb"
	"go.etcd.io/etcd/raft"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

const (
	pathMetrics = "/metrics"
	PathHealth  = "/health"
)

// HandleMetricsHealth registers metrics and health handlers.
func HandleMetricsHealth(mux *http.ServeMux, srv etcdserver.ServerV2) {
	mux.Handle(pathMetrics, promhttp.Handler())
	mux.Handle(PathHealth, NewHealthHandler(func() Health { return checkHealth(srv) }))
}

// HandlePrometheus registers prometheus handler on '/metrics'.
func HandlePrometheus(mux *http.ServeMux) {
	mux.Handle(pathMetrics, promhttp.Handler())
}

// HandleHealth registers health handler on '/health'.
func HandleHealth(mux *http.ServeMux, srv etcdserver.ServerV2) {
	mux.Handle(PathHealth, NewHealthHandler(func() Health { return checkHealth(srv) }))
}

// NewHealthHandler handles '/health' requests.
func NewHealthHandler(hfunc func() Health) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.Method != http.MethodGet {
			w.Header().Set("Allow", http.MethodGet)
			http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
			return
		}
		h := hfunc()
		d, _ := json.Marshal(h)
		if h.Health != "true" {
			http.Error(w, string(d), http.StatusServiceUnavailable)
			return
		}
		w.WriteHeader(http.StatusOK)
		w.Write(d)
	}
}

var (
	healthSuccess = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "etcd",
		Subsystem: "server",
		Name:      "health_success",
		Help:      "The total number of successful health checks",
	})
	healthFailed = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "etcd",
		Subsystem: "server",
		Name:      "health_failures",
		Help:      "The total number of failed health checks",
	})
)

func init() {
	prometheus.MustRegister(healthSuccess)
	prometheus.MustRegister(healthFailed)
}

// Health defines etcd server health status.
// TODO: remove manual parsing in etcdctl cluster-health
type Health struct {
	Health string `json:"health"`
}

// TODO: server NOSPACE, etcdserver.ErrNoLeader in health API

func checkHealth(srv etcdserver.ServerV2) Health {
	h := Health{Health: "true"}

	as := srv.Alarms()
	if len(as) > 0 {
		h.Health = "false"
	}

	if h.Health == "true" {
		if uint64(srv.Leader()) == raft.None {
			h.Health = "false"
		}
	}

	if h.Health == "true" {
		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
		_, err := srv.Do(ctx, etcdserverpb.Request{Method: "QGET"})
		cancel()
		if err != nil {
			h.Health = "false"
		}
	}

	if h.Health == "true" {
		healthSuccess.Inc()
	} else {
		healthFailed.Inc()
	}
	return h
}
<commit_msg>etcdserver/api/etcdhttp: remove unused "HandleHealth" function<commit_after>// Copyright 2017 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package etcdhttp

import (
	"context"
	"encoding/json"
	"net/http"
	"time"

	"go.etcd.io/etcd/etcdserver"
	"go.etcd.io/etcd/etcdserver/etcdserverpb"
	"go.etcd.io/etcd/raft"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

const (
	pathMetrics = "/metrics"
	PathHealth  = "/health"
)

// HandleMetricsHealth registers metrics and health handlers.
func HandleMetricsHealth(mux *http.ServeMux, srv etcdserver.ServerV2) {
	mux.Handle(pathMetrics, promhttp.Handler())
	mux.Handle(PathHealth, NewHealthHandler(func() Health { return checkHealth(srv) }))
}

// HandlePrometheus registers prometheus handler on '/metrics'.
func HandlePrometheus(mux *http.ServeMux) {
	mux.Handle(pathMetrics, promhttp.Handler())
}

// NewHealthHandler handles '/health' requests.
func NewHealthHandler(hfunc func() Health) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.Method != http.MethodGet {
			w.Header().Set("Allow", http.MethodGet)
			http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
			return
		}
		h := hfunc()
		d, _ := json.Marshal(h)
		if h.Health != "true" {
			http.Error(w, string(d), http.StatusServiceUnavailable)
			return
		}
		w.WriteHeader(http.StatusOK)
		w.Write(d)
	}
}

var (
	healthSuccess = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "etcd",
		Subsystem: "server",
		Name:      "health_success",
		Help:      "The total number of successful health checks",
	})
	healthFailed = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "etcd",
		Subsystem: "server",
		Name:      "health_failures",
		Help:      "The total number of failed health checks",
	})
)

func init() {
	prometheus.MustRegister(healthSuccess)
	prometheus.MustRegister(healthFailed)
}

// Health defines etcd server health status.
// TODO: remove manual parsing in etcdctl cluster-health
type Health struct {
	Health string `json:"health"`
}

// TODO: server NOSPACE, etcdserver.ErrNoLeader in health API

func checkHealth(srv etcdserver.ServerV2) Health {
	h := Health{Health: "true"}

	as := srv.Alarms()
	if len(as) > 0 {
		h.Health = "false"
	}

	if h.Health == "true" {
		if uint64(srv.Leader()) == raft.None {
			h.Health = "false"
		}
	}

	if h.Health == "true" {
		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
		_, err := srv.Do(ctx, etcdserverpb.Request{Method: "QGET"})
		cancel()
		if err != nil {
			h.Health = "false"
		}
	}

	if h.Health == "true" {
		healthSuccess.Inc()
	} else {
		healthFailed.Inc()
	}
	return h
}
<|endoftext|>


<commit_before>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package snap

import (
	"errors"
	"fmt"
	"hash/crc32"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"go.etcd.io/etcd/v3/etcdserver/api/snap/snappb"
	pioutil "go.etcd.io/etcd/v3/pkg/ioutil"
	"go.etcd.io/etcd/v3/pkg/pbutil"
	"go.etcd.io/etcd/v3/raft"
	"go.etcd.io/etcd/v3/raft/raftpb"
	"go.etcd.io/etcd/v3/wal/walpb"

	"go.uber.org/zap"
)

const snapSuffix = ".snap"

var (
	ErrNoSnapshot    = errors.New("snap: no available snapshot")
	ErrEmptySnapshot = errors.New("snap: empty snapshot")
	ErrCRCMismatch   = errors.New("snap: crc mismatch")
	crcTable         = crc32.MakeTable(crc32.Castagnoli)

	// A map of valid files that can be present in the snap folder.
	validFiles = map[string]bool{
		"db": true,
	}
)

type Snapshotter struct {
	lg  *zap.Logger
	dir string
}

func New(lg *zap.Logger, dir string) *Snapshotter {
	if lg == nil {
		lg = zap.NewNop()
	}
	return &Snapshotter{
		lg:  lg,
		dir: dir,
	}
}

func (s *Snapshotter) SaveSnap(snapshot raftpb.Snapshot) error {
	if raft.IsEmptySnap(snapshot) {
		return nil
	}
	return s.save(&snapshot)
}

func (s *Snapshotter) save(snapshot *raftpb.Snapshot) error {
	start := time.Now()

	fname := fmt.Sprintf("%016x-%016x%s", snapshot.Metadata.Term, snapshot.Metadata.Index, snapSuffix)
	b := pbutil.MustMarshal(snapshot)
	crc := crc32.Update(0, crcTable, b)
	snap := snappb.Snapshot{Crc: crc, Data: b}
	d, err := snap.Marshal()
	if err != nil {
		return err
	}
	snapMarshallingSec.Observe(time.Since(start).Seconds())

	spath := filepath.Join(s.dir, fname)

	fsyncStart := time.Now()
	err = pioutil.WriteAndSyncFile(spath, d, 0666)
	snapFsyncSec.Observe(time.Since(fsyncStart).Seconds())

	if err != nil {
		s.lg.Warn("failed to write a snap file", zap.String("path", spath), zap.Error(err))
		rerr := os.Remove(spath)
		if rerr != nil {
			s.lg.Warn("failed to remove a broken snap file", zap.String("path", spath), zap.Error(err))
		}
		return err
	}

	snapSaveSec.Observe(time.Since(start).Seconds())
	return nil
}

// Load returns the newest snapshot.
func (s *Snapshotter) Load() (*raftpb.Snapshot, error) {
	return s.loadMatching(func(*raftpb.Snapshot) bool { return true })
}

// LoadNewestAvailable loads the newest snapshot available that is in walSnaps.
func (s *Snapshotter) LoadNewestAvailable(walSnaps []walpb.Snapshot) (*raftpb.Snapshot, error) {
	return s.loadMatching(func(snapshot *raftpb.Snapshot) bool {
		m := snapshot.Metadata
		for i := len(walSnaps) - 1; i >= 0; i-- {
			if m.Term == walSnaps[i].Term && m.Index == walSnaps[i].Index {
				return true
			}
		}
		return false
	})
}

// loadMatching returns the newest snapshot where matchFn returns true.
func (s *Snapshotter) loadMatching(matchFn func(*raftpb.Snapshot) bool) (*raftpb.Snapshot, error) {
	names, err := s.snapNames()
	if err != nil {
		return nil, err
	}
	var snap *raftpb.Snapshot
	for _, name := range names {
		if snap, err = loadSnap(s.lg, s.dir, name); err == nil && matchFn(snap) {
			return snap, nil
		}
	}
	return nil, ErrNoSnapshot
}

func loadSnap(lg *zap.Logger, dir, name string) (*raftpb.Snapshot, error) {
	fpath := filepath.Join(dir, name)
	snap, err := Read(lg, fpath)
	if err != nil {
		brokenPath := fpath + ".broken"
		if lg != nil {
			lg.Warn("failed to read a snap file", zap.String("path", fpath), zap.Error(err))
		}
		if rerr := os.Rename(fpath, brokenPath); rerr != nil {
			if lg != nil {
				lg.Warn("failed to rename a broken snap file", zap.String("path", fpath), zap.String("broken-path", brokenPath), zap.Error(rerr))
			}
		} else {
			if lg != nil {
				lg.Warn("renamed to a broken snap file", zap.String("path", fpath), zap.String("broken-path", brokenPath))
			}
		}
	}
	return snap, err
}

// Read reads the snapshot named by snapname and returns the snapshot.
func Read(lg *zap.Logger, snapname string) (*raftpb.Snapshot, error) {
	b, err := ioutil.ReadFile(snapname)
	if err != nil {
		if lg != nil {
			lg.Warn("failed to read a snap file", zap.String("path", snapname), zap.Error(err))
		}
		return nil, err
	}

	if len(b) == 0 {
		if lg != nil {
			lg.Warn("failed to read empty snapshot file", zap.String("path", snapname))
		}
		return nil, ErrEmptySnapshot
	}

	var serializedSnap snappb.Snapshot
	if err = serializedSnap.Unmarshal(b); err != nil {
		if lg != nil {
			lg.Warn("failed to unmarshal snappb.Snapshot", zap.String("path", snapname), zap.Error(err))
		}
		return nil, err
	}

	if len(serializedSnap.Data) == 0 || serializedSnap.Crc == 0 {
		if lg != nil {
			lg.Warn("failed to read empty snapshot data", zap.String("path", snapname))
		}
		return nil, ErrEmptySnapshot
	}

	crc := crc32.Update(0, crcTable, serializedSnap.Data)
	if crc != serializedSnap.Crc {
		if lg != nil {
			lg.Warn("snap file is corrupt",
				zap.String("path", snapname),
				zap.Uint32("prev-crc", serializedSnap.Crc),
				zap.Uint32("new-crc", crc),
			)
		}
		return nil, ErrCRCMismatch
	}

	var snap raftpb.Snapshot
	if err = snap.Unmarshal(serializedSnap.Data); err != nil {
		if lg != nil {
			lg.Warn("failed to unmarshal raftpb.Snapshot", zap.String("path", snapname), zap.Error(err))
		}
		return nil, err
	}
	return &snap, nil
}

// snapNames returns the filename of the snapshots in logical time order (from newest to oldest).
// If there is no available snapshots, an ErrNoSnapshot will be returned.
func (s *Snapshotter) snapNames() ([]string, error) {
	dir, err := os.Open(s.dir)
	if err != nil {
		return nil, err
	}
	defer dir.Close()
	names, err := dir.Readdirnames(-1)
	if err != nil {
		return nil, err
	}
	if err = s.cleanupSnapdir(names); err != nil {
		return nil, err
	}
	snaps := checkSuffix(s.lg, names)
	if len(snaps) == 0 {
		return nil, ErrNoSnapshot
	}
	sort.Sort(sort.Reverse(sort.StringSlice(snaps)))
	return snaps, nil
}

func checkSuffix(lg *zap.Logger, names []string) []string {
	snaps := []string{}
	for i := range names {
		if strings.HasSuffix(names[i], snapSuffix) {
			snaps = append(snaps, names[i])
		} else {
			// If we find a file which is not a snapshot then check if it's
			// a vaild file. If not throw out a warning.
			if _, ok := validFiles[names[i]]; !ok {
				if lg != nil {
					lg.Warn("found unexpected non-snap file; skipping", zap.String("path", names[i]))
				}
			}
		}
	}
	return snaps
}

// cleanupSnapdir removes any files that should not be in the snapshot directory:
// - db.tmp prefixed files that can be orphaned by defragmentation
func (s *Snapshotter) cleanupSnapdir(filenames []string) error {
	for _, filename := range filenames {
		if strings.HasPrefix(filename, "db.tmp") {
			s.lg.Info("found orphaned defragmentation file; deleting", zap.String("path", filename))
			if rmErr := os.Remove(filepath.Join(s.dir, filename)); rmErr != nil && !os.IsNotExist(rmErr) {
				return fmt.Errorf("failed to remove orphaned defragmentation file %s: %v", filename, rmErr)
			}
		}
	}
	return nil
}

func (s *Snapshotter) ReleaseSnapDBs(snap raftpb.Snapshot) error {
	dir, err := os.Open(s.dir)
	if err != nil {
		return err
	}
	defer dir.Close()
	filenames, err := dir.Readdirnames(-1)
	if err != nil {
		return err
	}
	for _, filename := range filenames {
		if strings.HasSuffix(filename, ".snap.db") {
			hexIndex := strings.TrimSuffix(filepath.Base(filename), ".snap.db")
			index, err := strconv.ParseUint(hexIndex, 16, 64)
			if err != nil {
				return fmt.Errorf("failed to parse index from .snap.db filename %q: %v", filename, err)
			}
			if index < snap.Metadata.Index {
				s.lg.Info("found orphaned .snap.db file; deleting", zap.String("path", filename))
				if rmErr := os.Remove(filepath.Join(s.dir, filename)); rmErr != nil && !os.IsNotExist(rmErr) {
					return fmt.Errorf("failed to remove orphaned defragmentation file %s: %v", filename, rmErr)
				}
			}
		}
	}
	return nil
}
<commit_msg>etcdserver: continue releasing snap db in case of error<commit_after>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package snap

import (
	"errors"
	"fmt"
	"hash/crc32"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"go.etcd.io/etcd/v3/etcdserver/api/snap/snappb"
	pioutil "go.etcd.io/etcd/v3/pkg/ioutil"
	"go.etcd.io/etcd/v3/pkg/pbutil"
	"go.etcd.io/etcd/v3/raft"
	"go.etcd.io/etcd/v3/raft/raftpb"
	"go.etcd.io/etcd/v3/wal/walpb"

	"go.uber.org/zap"
)

const snapSuffix = ".snap"

var (
	ErrNoSnapshot    = errors.New("snap: no available snapshot")
	ErrEmptySnapshot = errors.New("snap: empty snapshot")
	ErrCRCMismatch   = errors.New("snap: crc mismatch")
	crcTable         = crc32.MakeTable(crc32.Castagnoli)

	// A map of valid files that can be present in the snap folder.
	validFiles = map[string]bool{
		"db": true,
	}
)

type Snapshotter struct {
	lg  *zap.Logger
	dir string
}

func New(lg *zap.Logger, dir string) *Snapshotter {
	if lg == nil {
		lg = zap.NewNop()
	}
	return &Snapshotter{
		lg:  lg,
		dir: dir,
	}
}

func (s *Snapshotter) SaveSnap(snapshot raftpb.Snapshot) error {
	if raft.IsEmptySnap(snapshot) {
		return nil
	}
	return s.save(&snapshot)
}

func (s *Snapshotter) save(snapshot *raftpb.Snapshot) error {
	start := time.Now()

	fname := fmt.Sprintf("%016x-%016x%s", snapshot.Metadata.Term, snapshot.Metadata.Index, snapSuffix)
	b := pbutil.MustMarshal(snapshot)
	crc := crc32.Update(0, crcTable, b)
	snap := snappb.Snapshot{Crc: crc, Data: b}
	d, err := snap.Marshal()
	if err != nil {
		return err
	}
	snapMarshallingSec.Observe(time.Since(start).Seconds())

	spath := filepath.Join(s.dir, fname)

	fsyncStart := time.Now()
	err = pioutil.WriteAndSyncFile(spath, d, 0666)
	snapFsyncSec.Observe(time.Since(fsyncStart).Seconds())

	if err != nil {
		s.lg.Warn("failed to write a snap file", zap.String("path", spath), zap.Error(err))
		rerr := os.Remove(spath)
		if rerr != nil {
			s.lg.Warn("failed to remove a broken snap file", zap.String("path", spath), zap.Error(err))
		}
		return err
	}

	snapSaveSec.Observe(time.Since(start).Seconds())
	return nil
}

// Load returns the newest snapshot.
func (s *Snapshotter) Load() (*raftpb.Snapshot, error) {
	return s.loadMatching(func(*raftpb.Snapshot) bool { return true })
}

// LoadNewestAvailable loads the newest snapshot available that is in walSnaps.
func (s *Snapshotter) LoadNewestAvailable(walSnaps []walpb.Snapshot) (*raftpb.Snapshot, error) {
	return s.loadMatching(func(snapshot *raftpb.Snapshot) bool {
		m := snapshot.Metadata
		for i := len(walSnaps) - 1; i >= 0; i-- {
			if m.Term == walSnaps[i].Term && m.Index == walSnaps[i].Index {
				return true
			}
		}
		return false
	})
}

// loadMatching returns the newest snapshot where matchFn returns true.
func (s *Snapshotter) loadMatching(matchFn func(*raftpb.Snapshot) bool) (*raftpb.Snapshot, error) {
	names, err := s.snapNames()
	if err != nil {
		return nil, err
	}
	var snap *raftpb.Snapshot
	for _, name := range names {
		if snap, err = loadSnap(s.lg, s.dir, name); err == nil && matchFn(snap) {
			return snap, nil
		}
	}
	return nil, ErrNoSnapshot
}

func loadSnap(lg *zap.Logger, dir, name string) (*raftpb.Snapshot, error) {
	fpath := filepath.Join(dir, name)
	snap, err := Read(lg, fpath)
	if err != nil {
		brokenPath := fpath + ".broken"
		if lg != nil {
			lg.Warn("failed to read a snap file", zap.String("path", fpath), zap.Error(err))
		}
		if rerr := os.Rename(fpath, brokenPath); rerr != nil {
			if lg != nil {
				lg.Warn("failed to rename a broken snap file", zap.String("path", fpath), zap.String("broken-path", brokenPath), zap.Error(rerr))
			}
		} else {
			if lg != nil {
				lg.Warn("renamed to a broken snap file", zap.String("path", fpath), zap.String("broken-path", brokenPath))
			}
		}
	}
	return snap, err
}

// Read reads the snapshot named by snapname and returns the snapshot.
func Read(lg *zap.Logger, snapname string) (*raftpb.Snapshot, error) {
	b, err := ioutil.ReadFile(snapname)
	if err != nil {
		if lg != nil {
			lg.Warn("failed to read a snap file", zap.String("path", snapname), zap.Error(err))
		}
		return nil, err
	}

	if len(b) == 0 {
		if lg != nil {
			lg.Warn("failed to read empty snapshot file", zap.String("path", snapname))
		}
		return nil, ErrEmptySnapshot
	}

	var serializedSnap snappb.Snapshot
	if err = serializedSnap.Unmarshal(b); err != nil {
		if lg != nil {
			lg.Warn("failed to unmarshal snappb.Snapshot", zap.String("path", snapname), zap.Error(err))
		}
		return nil, err
	}

	if len(serializedSnap.Data) == 0 || serializedSnap.Crc == 0 {
		if lg != nil {
			lg.Warn("failed to read empty snapshot data", zap.String("path", snapname))
		}
		return nil, ErrEmptySnapshot
	}

	crc := crc32.Update(0, crcTable, serializedSnap.Data)
	if crc != serializedSnap.Crc {
		if lg != nil {
			lg.Warn("snap file is corrupt",
				zap.String("path", snapname),
				zap.Uint32("prev-crc", serializedSnap.Crc),
				zap.Uint32("new-crc", crc),
			)
		}
		return nil, ErrCRCMismatch
	}

	var snap raftpb.Snapshot
	if err = snap.Unmarshal(serializedSnap.Data); err != nil {
		if lg != nil {
			lg.Warn("failed to unmarshal raftpb.Snapshot", zap.String("path", snapname), zap.Error(err))
		}
		return nil, err
	}
	return &snap, nil
}

// snapNames returns the filename of the snapshots in logical time order (from newest to oldest).
// If there is no available snapshots, an ErrNoSnapshot will be returned.
func (s *Snapshotter) snapNames() ([]string, error) {
	dir, err := os.Open(s.dir)
	if err != nil {
		return nil, err
	}
	defer dir.Close()
	names, err := dir.Readdirnames(-1)
	if err != nil {
		return nil, err
	}
	if err = s.cleanupSnapdir(names); err != nil {
		return nil, err
	}
	snaps := checkSuffix(s.lg, names)
	if len(snaps) == 0 {
		return nil, ErrNoSnapshot
	}
	sort.Sort(sort.Reverse(sort.StringSlice(snaps)))
	return snaps, nil
}

func checkSuffix(lg *zap.Logger, names []string) []string {
	snaps := []string{}
	for i := range names {
		if strings.HasSuffix(names[i], snapSuffix) {
			snaps = append(snaps, names[i])
		} else {
			// If we find a file which is not a snapshot then check if it's
			// a vaild file. If not throw out a warning.
			if _, ok := validFiles[names[i]]; !ok {
				if lg != nil {
					lg.Warn("found unexpected non-snap file; skipping", zap.String("path", names[i]))
				}
			}
		}
	}
	return snaps
}

// cleanupSnapdir removes any files that should not be in the snapshot directory:
// - db.tmp prefixed files that can be orphaned by defragmentation
func (s *Snapshotter) cleanupSnapdir(filenames []string) error {
	for _, filename := range filenames {
		if strings.HasPrefix(filename, "db.tmp") {
			s.lg.Info("found orphaned defragmentation file; deleting", zap.String("path", filename))
			if rmErr := os.Remove(filepath.Join(s.dir, filename)); rmErr != nil && !os.IsNotExist(rmErr) {
				return fmt.Errorf("failed to remove orphaned .snap.db file %s: %v", filename, rmErr)
			}
		}
	}
	return nil
}

func (s *Snapshotter) ReleaseSnapDBs(snap raftpb.Snapshot) error {
	dir, err := os.Open(s.dir)
	if err != nil {
		return err
	}
	defer dir.Close()
	filenames, err := dir.Readdirnames(-1)
	if err != nil {
		return err
	}
	for _, filename := range filenames {
		if strings.HasSuffix(filename, ".snap.db") {
			hexIndex := strings.TrimSuffix(filepath.Base(filename), ".snap.db")
			index, err := strconv.ParseUint(hexIndex, 16, 64)
			if err != nil {
				s.lg.Error("failed to parse index from filename", zap.String("path", filename), zap.String("error", err.Error()))
				continue
			}
			if index < snap.Metadata.Index {
				s.lg.Info("found orphaned .snap.db file; deleting", zap.String("path", filename))
				if rmErr := os.Remove(filepath.Join(s.dir, filename)); rmErr != nil && !os.IsNotExist(rmErr) {
					s.lg.Error("failed to remove orphaned .snap.db file", zap.String("path", filename), zap.String("error", rmErr.Error()))
				}
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package browsers

import (
	"errors"
	"fmt"
	"github.com/HouzuoGuo/laitos/misc"
	"os"
	"path"
	"sync"
	"time"
)

const (
	// SlimerJSImageTag is the latet name and tag of the SlimerJS+Firefox docker image that works best on this version of laitos.
	SlimerJSImageTag = "registry.hub.docker.com/hzgl/slimerjs:20180520"

	// DockerMaintenanceIntervalSec is the interval between runs of docker maintenance routine - daemon startup and image pulling.
	DockerMaintenanceIntervalSec = 3600
)

var prepareDockerOnce = new(sync.Once)

// Instances manage lifecycle of a fixed number of browser server instances (SlimerJS via Docker).
type Instances struct {
	MaxInstances   int `json:"MaxInstances"`   // Maximum number of instances
	MaxLifetimeSec int `json:"MaxLifetimeSec"` // Unconditionally kill instance after this number of seconds elapse
	BasePortNumber int `json:"BasePortNumber"` // Browser instances listen on a port number beginning from this one

	browserMutex   *sync.Mutex // Protect against concurrent modification to browsers
	browsers       []*Instance // All browsers
	browserCounter int         // Increment only counter
	logger         misc.Logger
}

// Check configuration and initialise internal states.
func (instances *Instances) Initialise() error {
	instances.logger = misc.Logger{
		ComponentName: "browsers.Instances",
		ComponentID:   []misc.LoggerIDField{{"MaxInst", instances.MaxInstances}, {"MaxLifetime", instances.MaxLifetimeSec}},
	}
	if instances.MaxInstances < 1 {
		instances.MaxInstances = 5 // reasonable for a few consumers
	}
	if instances.MaxLifetimeSec < 1 {
		instances.MaxLifetimeSec = 1800 // half hour is quite reasonable
	}
	if instances.BasePortNumber < 1024 {
		return errors.New("browsers.Instances.Initialise: BasePortNumber must be greater than 1023")
	}

	instances.browserMutex = new(sync.Mutex)
	instances.browsers = make([]*Instance, instances.MaxInstances)
	instances.browserCounter = -1

	prepareDockerOnce.Do(func() {
		go func() {
			// Start this background routine in an infinite loop to keep docker running and image available
			for {
				// Enable and start docker daemon
				PrepareDocker(instances.logger)
				time.Sleep(DockerMaintenanceIntervalSec * time.Second)
			}
		}()
	})
	return nil
}

/*
PrepareDocker starts docker daemon, ensures that docker keeps running, and pulls the docker image for SlimerJS. The
routine requires root privilege to run.
*/
func PrepareDocker(logger misc.Logger) {
	if !misc.EnableStartDaemon("docker") {
		logger.Info("PrepareDocker", "", nil, "failed to enable/start docker daemon")
		// Nevertheless, move on.
	}
	// Download the SlimerJS docker image
	logger.Info("PrepareDocker", "", nil, "pulling %s", SlimerJSImageTag)
	out, err := misc.InvokeProgram(nil, 1800, "docker", "pull", SlimerJSImageTag)
	logger.Info("PrepareDocker", "", nil, "image pulling result: %v - %s", err, out)

}

// Acquire a new instance instance. If necessary, kill an existing instance to free up the space for the new instance.
func (instances *Instances) Acquire() (index int, browser *Instance, err error) {
	instances.browserMutex.Lock()
	defer instances.browserMutex.Unlock()
	instances.browserCounter++
	index = instances.browserCounter % len(instances.browsers)
	if instance := instances.browsers[index]; instance != nil {
		instance.Kill()
	}

	renderImageDir := path.Join(os.TempDir(), fmt.Sprintf("laitos-browser-instance-render-slimerjs-%d-%d", time.Now().Unix(), index))
	browser = &Instance{
		RenderImageDir:     renderImageDir,
		Port:               instances.BasePortNumber + int(index),
		AutoKillTimeoutSec: instances.MaxLifetimeSec,
		Index:              index,
	}
	instances.browsers[index] = browser
	err = browser.Start()
	return
}

/*
Return instance instance of the specified index and match its tag against expectation.
If instance instance does not exist or tag does not match, return nil.
*/
func (instances *Instances) Retrieve(index int, expectedTag string) *Instance {
	instances.browserMutex.Lock()
	defer instances.browserMutex.Unlock()
	browser := instances.browsers[index]
	if browser == nil || browser.Tag != expectedTag {
		return nil
	}
	return browser
}

/*
Forcibly stop all browser instances.
Be aware that, laitos does not use a persistent record of containers spawned, hence if laitos crashes, it will not be
able to kill containers spawned by previous laitos run, which causes failure when user attempts to start a new browser
instance. This is terribly unfortunate and there is not a good remedy other than manually running:

docker ps -a -q -f 'name=laitos-browsers.*' | xargs docker kill -f

Also be aware that the above statement must not be run automatically in KillAll function, because a computer host may
run more than one laitos programs and there is no way to tell whether any of the containers belongs to a crashed laitos.
*/
func (instances *Instances) KillAll() {
	instances.browserMutex.Lock()
	defer instances.browserMutex.Unlock()
	for _, instance := range instances.browsers {
		if instance != nil {
			instance.Kill()
		}
	}
}
<commit_msg>enable ip forwarding for docker container to access the internet<commit_after>package browsers

import (
	"errors"
	"fmt"
	"github.com/HouzuoGuo/laitos/misc"
	"os"
	"path"
	"sync"
	"time"
)

const (
	// SlimerJSImageTag is the latet name and tag of the SlimerJS+Firefox docker image that works best on this version of laitos.
	SlimerJSImageTag = "registry.hub.docker.com/hzgl/slimerjs:20180520"

	// DockerMaintenanceIntervalSec is the interval between runs of docker maintenance routine - daemon startup and image pulling.
	DockerMaintenanceIntervalSec = 3600
)

var prepareDockerOnce = new(sync.Once)

// Instances manage lifecycle of a fixed number of browser server instances (SlimerJS via Docker).
type Instances struct {
	MaxInstances   int `json:"MaxInstances"`   // Maximum number of instances
	MaxLifetimeSec int `json:"MaxLifetimeSec"` // Unconditionally kill instance after this number of seconds elapse
	BasePortNumber int `json:"BasePortNumber"` // Browser instances listen on a port number beginning from this one

	browserMutex   *sync.Mutex // Protect against concurrent modification to browsers
	browsers       []*Instance // All browsers
	browserCounter int         // Increment only counter
	logger         misc.Logger
}

// Check configuration and initialise internal states.
func (instances *Instances) Initialise() error {
	instances.logger = misc.Logger{
		ComponentName: "browsers.Instances",
		ComponentID:   []misc.LoggerIDField{{"MaxInst", instances.MaxInstances}, {"MaxLifetime", instances.MaxLifetimeSec}},
	}
	if instances.MaxInstances < 1 {
		instances.MaxInstances = 5 // reasonable for a few consumers
	}
	if instances.MaxLifetimeSec < 1 {
		instances.MaxLifetimeSec = 1800 // half hour is quite reasonable
	}
	if instances.BasePortNumber < 1024 {
		return errors.New("browsers.Instances.Initialise: BasePortNumber must be greater than 1023")
	}

	instances.browserMutex = new(sync.Mutex)
	instances.browsers = make([]*Instance, instances.MaxInstances)
	instances.browserCounter = -1

	prepareDockerOnce.Do(func() {
		go func() {
			// Start this background routine in an infinite loop to keep docker running and image available
			for {
				// Enable and start docker daemon
				PrepareDocker(instances.logger)
				time.Sleep(DockerMaintenanceIntervalSec * time.Second)
			}
		}()
	})
	return nil
}

/*
PrepareDocker starts docker daemon, ensures that docker keeps running, and pulls the docker image for SlimerJS. The
routine requires root privilege to run.
*/
func PrepareDocker(logger misc.Logger) {
	if !misc.EnableStartDaemon("docker") {
		logger.Info("PrepareDocker", "", nil, "failed to enable/start docker daemon")
		// Nevertheless, move on.
	}
	// Download the SlimerJS docker image
	logger.Info("PrepareDocker", "", nil, "pulling %s", SlimerJSImageTag)
	out, err := misc.InvokeProgram(nil, 1800, "docker", "pull", SlimerJSImageTag)
	logger.Info("PrepareDocker", "", nil, "image pulling result: %v - %s", err, out)
	// Turn on ip forwarding so that docker containers will have access to the Internet
	out, err = misc.InvokeProgram(nil, 30, "sysctl", "-w", "net.ipv4.ip_forward=1")
	logger.Info("PrepareDocker", "", nil, "enable ip forwarding result: %v - %s", err, out)
}

// Acquire a new instance instance. If necessary, kill an existing instance to free up the space for the new instance.
func (instances *Instances) Acquire() (index int, browser *Instance, err error) {
	instances.browserMutex.Lock()
	defer instances.browserMutex.Unlock()
	instances.browserCounter++
	index = instances.browserCounter % len(instances.browsers)
	if instance := instances.browsers[index]; instance != nil {
		instance.Kill()
	}

	renderImageDir := path.Join(os.TempDir(), fmt.Sprintf("laitos-browser-instance-render-slimerjs-%d-%d", time.Now().Unix(), index))
	browser = &Instance{
		RenderImageDir:     renderImageDir,
		Port:               instances.BasePortNumber + int(index),
		AutoKillTimeoutSec: instances.MaxLifetimeSec,
		Index:              index,
	}
	instances.browsers[index] = browser
	err = browser.Start()
	return
}

/*
Return instance instance of the specified index and match its tag against expectation.
If instance instance does not exist or tag does not match, return nil.
*/
func (instances *Instances) Retrieve(index int, expectedTag string) *Instance {
	instances.browserMutex.Lock()
	defer instances.browserMutex.Unlock()
	browser := instances.browsers[index]
	if browser == nil || browser.Tag != expectedTag {
		return nil
	}
	return browser
}

/*
Forcibly stop all browser instances.
Be aware that, laitos does not use a persistent record of containers spawned, hence if laitos crashes, it will not be
able to kill containers spawned by previous laitos run, which causes failure when user attempts to start a new browser
instance. This is terribly unfortunate and there is not a good remedy other than manually running:

docker ps -a -q -f 'name=laitos-browsers.*' | xargs docker kill -f

Also be aware that the above statement must not be run automatically in KillAll function, because a computer host may
run more than one laitos programs and there is no way to tell whether any of the containers belongs to a crashed laitos.
*/
func (instances *Instances) KillAll() {
	instances.browserMutex.Lock()
	defer instances.browserMutex.Unlock()
	for _, instance := range instances.browsers {
		if instance != nil {
			instance.Kill()
		}
	}
}
<|endoftext|>


<commit_before>package geomys

// Point -- represents a pair of geographic coordinates
// (latitude,longitude).
type Point struct {
	lat, lon float64
}

// Geo -- returns a point with the geographic latitude `lat`
// and the geographic longitude `lon`.
// This function causes a runtime panic when lat∉[-90,90]
// or lon∉[-180,180].
func Geo(lat, lon float64) Point {
	if !(-90 <= lat && lat <= 90) {
		panic("geomys.Geo: domain error: `lat`")
	}
	if !(-180 <= lon && lon <= 180) {
		panic("geomys.Geo: domain error: `lon`")
	}
	return Point{lat, lon}
}

// Geo -- returns the geographic coordinates of `p`.
func (p Point) Geo() (lat, lon float64) {
	lat, lon = p.lat, p.lon
	return
}
<commit_msg>Improve panic messages.<commit_after>package geomys

// Point -- represents a pair of geographic coordinates
// (latitude,longitude).
type Point struct {
	lat, lon float64
}

// Geo -- returns a point with the geographic latitude `lat`
// and the geographic longitude `lon`.
// This function causes a runtime panic when lat∉[-90,90]
// or lon∉[-180,180].
func Geo(lat, lon float64) Point {
	if !(-90 <= lat && lat <= 90) {
		panic("geomys.Geo: lat not in [-90,90]")
	}
	if !(-180 <= lon && lon <= 180) {
		panic("geomys.Geo: lon not in [-180,180]")
	}
	return Point{lat, lon}
}

// Geo -- returns the geographic coordinates of `p`.
func (p Point) Geo() (lat, lon float64) {
	lat, lon = p.lat, p.lon
	return
}
<|endoftext|>


<commit_before>package pinboard

import (
	"encoding/json"
	"errors"
	"net/url"
	"strings"
	"time"
)

// Aliasing this to get a custom UnmarshalJSON because Pinboard
// can return `"description": false` in the JSON output.
type descriptionType string

// Post represents a bookmark.
type Post struct {
	// URL of bookmark.
	Href *url.URL

	// Title of bookmark. This field is unfortunately named
	// 'description' for backwards compatibility with the
	// delicious API
	Description string

	// Description of the item. Called 'extended' for backwards
	// compatibility with delicious API.
	Extended []byte

	// Tags of bookmark.
	Tags []string

	// If the bookmark is private or public.
	Shared bool

	// If the bookmark is marked to read later.
	Toread bool

	// Create time for this bookmark.
	Time time.Time

	// Change detection signature of the bookmark.
	Meta []byte

	// Hash of the bookmark.
	Hash []byte

	// The number of other users who have bookmarked this same
	// item.
	Others int
}

// post represents intermediate post response data before type
// conversion.
type post struct {
	Href        string
	Description descriptionType
	Extended    string
	Tags        string
	Shared      string
	Toread      string
	Time        string
	Meta        string
	Hash        string
	Others      int
}

// toPost converts a post to a type correct Post.
func (p *post) toPost() (*Post, error) {
	href, err := url.Parse(p.Href)
	if err != nil {
		return nil, err
	}

	tags := strings.Split(p.Tags, " ")

	var shared, toread bool
	if p.Shared == "yes" {
		shared = true
	}

	if p.Toread == "yes" {
		toread = true
	}

	dt, err := time.Parse(time.RFC3339, p.Time)
	if err != nil {
		return nil, err
	}

	P := Post{
		Href:        href,
		Description: p.Description.String(),
		Extended:    []byte(p.Extended),
		Tags:        tags,
		Shared:      shared,
		Toread:      toread,
		Time:        dt,
		Meta:        []byte(p.Meta),
		Hash:        []byte(p.Hash),
		Others:      p.Others,
	}

	return &P, nil
}

// postsResponse represents a response from certain /posts/ endpoints.
type postsResponse struct {
	UpdateTime string `json:"update_time,omitempty"`
	ResultCode string `json:"result_code,omitempty"`
	Date       string `json:"date,omitempty"`
	User       string `json:"user,omitempty"`
	Posts      []post `json:"posts,omitempty"`
}

// PostsUpdate returns the most recent time a bookmark was added,
// updated or deleted.
//
// https://pinboard.in/api/#posts_update
func PostsUpdate() (time.Time, error) {
	resp, err := get("postsUpdate", nil)
	if err != nil {
		return time.Time{}, err
	}

	var pr postsResponse
	err = json.Unmarshal(resp, &pr)
	if err != nil {
		return time.Time{}, err
	}

	update, err := time.Parse(time.RFC3339, pr.UpdateTime)
	if err != nil {
		return time.Time{}, err
	}

	return update, nil
}

// PostsAddOptions represents the required and optional arguments for
// adding a bookmark.
type PostsAddOptions struct {
	// Required: The URL of the item.
	URL string

	// Required: Title of the item. This field is unfortunately
	// named 'description' for backwards compatibility with the
	// delicious API.
	Description string

	// Description of the item. Called 'extended' for backwards
	// compatibility with delicious API.
	Extended []byte

	// List of up to 100 tags.
	Tags []string

	// Creation time for this bookmark. Defaults to current
	// time. Datestamps more than 10 minutes ahead of server time
	// will be reset to current server time.
	Dt time.Time

	// Replace any existing bookmark with this URL. Default is
	// yes. If set to no, will throw an error if bookmark exists.
	Replace bool

	// Make bookmark public. Default is "yes" unless user has
	// enabled the "save all bookmarks as private" user setting,
	// in which case default is "no".
	Shared bool

	// Marks the bookmark as unread. Default is "no".
	Toread bool
}

// PostsAdd adds a bookmark.
//
// https://pinboard.in/api/#posts_add
func PostsAdd(opt *PostsAddOptions) error {
	if opt.URL == "" {
		return errors.New("error: missing url")
	}

	if opt.Description == "" {
		return errors.New("error: missing description")
	}

	resp, err := get("postsAdd", opt)
	if err != nil {
		return err
	}

	var pr postsResponse
	err = json.Unmarshal(resp, &pr)
	if err != nil {
		return err
	}

	if pr.ResultCode != "done" {
		return errors.New(pr.ResultCode)
	}

	return nil
}

// postsDeleteOptions represents the single required argument for
// deleting a bookmark.
type postsDeleteOptions struct {
	URL string
}

// PostsDelete deletes the bookmark by url.
//
// https://pinboard.in/api/#posts_delete
func PostsDelete(url string) error {
	resp, err := get("postsDelete", &postsDeleteOptions{URL: url})
	if err != nil {
		return err
	}

	var pr postsResponse
	err = json.Unmarshal(resp, &pr)
	if err != nil {
		return err
	}

	if pr.ResultCode != "done" {
		return errors.New(pr.ResultCode)
	}

	return nil
}

// PostsGetOptions represents the optional arguments for getting
// bookmarks.
type PostsGetOptions struct {
	// Filter by up to three tags.
	Tag []string

	// Return results bookmarked on this day. UTC date in this
	// format: 2010-12-11.
	Dt time.Time

	// Return bookmark for this URL.
	URL string

	// Include a change detection signature in a meta attribute.
	Meta bool
}

// PostsGet returns one or more posts (on a single day) matching the
// arguments. If no date or URL is given, date of most recent bookmark
// will be used.Returns one or more posts on a single day matching the
// arguments. If no date or URL is given, date of most recent bookmark
// will be used.
//
// https://pinboard.in/api/#posts_get
func PostsGet(opt *PostsGetOptions) ([]*Post, error) {
	resp, err := get("postsGet", opt)
	if err != nil {
		return nil, err
	}

	var pr postsResponse
	err = json.Unmarshal(resp, &pr)
	if err != nil {
		return nil, err
	}

	var posts []*Post
	for _, p := range pr.Posts {
		post, err := p.toPost()
		if err != nil {
			return nil, err
		}

		posts = append(posts, post)
	}

	return posts, nil
}

// PostsRecentOptions represents the optional arguments for returning
// the user's most recent posts.
type PostsRecentOptions struct {
	// Filter by up to three tags.
	Tag []string

	// Number of results to return. Default is 15, max is 100.
	Count int
}

// PostsRecent returns a list of the user's most recent posts,
// filtered by tag.
//
// https://pinboard.in/api/#posts_recent
func PostsRecent(opt *PostsRecentOptions) ([]*Post, error) {
	resp, err := get("postsRecent", opt)
	if err != nil {
		return nil, err
	}

	var pr postsResponse
	err = json.Unmarshal(resp, &pr)
	if err != nil {
		return nil, err
	}

	var posts []*Post
	for _, p := range pr.Posts {
		post, err := p.toPost()
		if err != nil {
			return nil, err
		}

		posts = append(posts, post)
	}

	return posts, nil
}

// postsDatesResponse represents the response from /posts/dates.
type postsDatesResponse struct {
	User  string            `json:"user"`
	Tag   string            `json:"tag"`
	Dates map[string]string `json:"dates"`
}

// PostsDatesOptions represents the single optional argument for
// returning a list of dates with the number of posts at each date.
type PostsDatesOptions struct {
	// Filter by up to three tags.
	Tag []string
}

// PostsDates returns a list of dates with the number of posts at each
// date.
//
// https://pinboard.in/api/#posts_dates
func PostsDates(opt *PostsDatesOptions) (map[string]string, error) {
	resp, err := get("postsDates", opt)
	if err != nil {
		return nil, err
	}

	var pr postsDatesResponse
	err = json.Unmarshal(resp, &pr)
	if err != nil {
		return nil, err
	}

	return pr.Dates, nil
}

// PostsAllOptions represents the optional arguments for returning all
// bookmarks in the user's account.
type PostsAllOptions struct {
	// Filter by up to three tags.
	Tag []string

	// Offset value (default is 0).
	Start int

	// Number of results to return. Default is all.
	Results int

	// Return only bookmarks created after this time.
	Fromdt time.Time

	// Return only bookmarks created before this time.
	Todt time.Time

	// Include a change detection signature for each bookmark.
	//
	// Note: This probably doesn't work. A meta field is always
	// returned. The Pinboard API says the datatype is an int but
	// changing the value has no impact on the results. Using a
	// yes/no string like all the other meta options doesn't work
	// either.
	Meta int
}

// PostsAll returns all bookmarks in the user's account.
//
// https://pinboard.in/api/#posts_all
func PostsAll(opt *PostsAllOptions) ([]*Post, error) {
	resp, err := get("postsAll", opt)
	if err != nil {
		return nil, err
	}

	var pr []post
	err = json.Unmarshal(resp, &pr)
	if err != nil {
		return nil, err
	}

	var posts []*Post
	for _, p := range pr {
		post, err := p.toPost()
		if err != nil {
			return nil, err
		}

		posts = append(posts, post)
	}

	return posts, nil
}

// postSuggestResponse represents the response from /posts/suggest.
type postsSuggestResponse struct {
	Popular     []string `json:"popular"`
	Recommended []string `json:"recommended"`
}

// postSuggestOptions represents the single required argument, url,
// for suggesting tags for a post.
type postsSuggestOptions struct {
	URL string
}

// PostsSuggestPopular returns a slice of popular tags for a given
// URL. Popular tags are tags used site-wide for the url.
//
// https://pinboard.in/api/#posts_suggest
func PostsSuggestPopular(url string) ([]string, error) {
	resp, err := get("postsSuggest", &postsSuggestOptions{URL: url})
	if err != nil {
		return nil, err
	}

	var pr []postsSuggestResponse
	err = json.Unmarshal(resp, &pr)
	if err != nil {
		return nil, err
	}

	return pr[0].Popular, nil
}

// PostsSuggestRecommended returns a slice of recommended tags for a
// given URL. Recommended tags are drawn from the user's own tags.
//
// https://pinboard.in/api/#posts_suggest
func PostsSuggestRecommended(url string) ([]string, error) {
	resp, err := get("postsSuggest", &postsSuggestOptions{URL: url})
	if err != nil {
		return nil, err
	}

	var pr []postsSuggestResponse
	err = json.Unmarshal(resp, &pr)
	if err != nil {
		return nil, err
	}

	return pr[1].Recommended, nil
}

// UnmarshalJSON converts a `descriptionType` into a `string`.
func (d *descriptionType) UnmarshalJSON(data []byte) error {
	// Have to do the type dance to avoid an infinite loop.
	type descriptionTypeAlias descriptionType
	var d2 descriptionTypeAlias

	if err := json.Unmarshal(data, &d2); err != nil {
		d2 = ""
	}
	*d = descriptionType(d2)

	return nil
}

// String returns the `string` value of our `descriptionType`.
func (d *descriptionType) String() string {
	return string(*d)
}
<commit_msg>Make post dates use an int<commit_after>package pinboard

import (
	"encoding/json"
	"errors"
	"net/url"
	"strings"
	"time"
)

// Aliasing this to get a custom UnmarshalJSON because Pinboard
// can return `"description": false` in the JSON output.
type descriptionType string

// Post represents a bookmark.
type Post struct {
	// URL of bookmark.
	Href *url.URL

	// Title of bookmark. This field is unfortunately named
	// 'description' for backwards compatibility with the
	// delicious API
	Description string

	// Description of the item. Called 'extended' for backwards
	// compatibility with delicious API.
	Extended []byte

	// Tags of bookmark.
	Tags []string

	// If the bookmark is private or public.
	Shared bool

	// If the bookmark is marked to read later.
	Toread bool

	// Create time for this bookmark.
	Time time.Time

	// Change detection signature of the bookmark.
	Meta []byte

	// Hash of the bookmark.
	Hash []byte

	// The number of other users who have bookmarked this same
	// item.
	Others int
}

// post represents intermediate post response data before type
// conversion.
type post struct {
	Href        string
	Description descriptionType
	Extended    string
	Tags        string
	Shared      string
	Toread      string
	Time        string
	Meta        string
	Hash        string
	Others      int
}

// toPost converts a post to a type correct Post.
func (p *post) toPost() (*Post, error) {
	href, err := url.Parse(p.Href)
	if err != nil {
		return nil, err
	}

	tags := strings.Split(p.Tags, " ")

	var shared, toread bool
	if p.Shared == "yes" {
		shared = true
	}

	if p.Toread == "yes" {
		toread = true
	}

	dt, err := time.Parse(time.RFC3339, p.Time)
	if err != nil {
		return nil, err
	}

	P := Post{
		Href:        href,
		Description: p.Description.String(),
		Extended:    []byte(p.Extended),
		Tags:        tags,
		Shared:      shared,
		Toread:      toread,
		Time:        dt,
		Meta:        []byte(p.Meta),
		Hash:        []byte(p.Hash),
		Others:      p.Others,
	}

	return &P, nil
}

// postsResponse represents a response from certain /posts/ endpoints.
type postsResponse struct {
	UpdateTime string `json:"update_time,omitempty"`
	ResultCode string `json:"result_code,omitempty"`
	Date       string `json:"date,omitempty"`
	User       string `json:"user,omitempty"`
	Posts      []post `json:"posts,omitempty"`
}

// PostsUpdate returns the most recent time a bookmark was added,
// updated or deleted.
//
// https://pinboard.in/api/#posts_update
func PostsUpdate() (time.Time, error) {
	resp, err := get("postsUpdate", nil)
	if err != nil {
		return time.Time{}, err
	}

	var pr postsResponse
	err = json.Unmarshal(resp, &pr)
	if err != nil {
		return time.Time{}, err
	}

	update, err := time.Parse(time.RFC3339, pr.UpdateTime)
	if err != nil {
		return time.Time{}, err
	}

	return update, nil
}

// PostsAddOptions represents the required and optional arguments for
// adding a bookmark.
type PostsAddOptions struct {
	// Required: The URL of the item.
	URL string

	// Required: Title of the item. This field is unfortunately
	// named 'description' for backwards compatibility with the
	// delicious API.
	Description string

	// Description of the item. Called 'extended' for backwards
	// compatibility with delicious API.
	Extended []byte

	// List of up to 100 tags.
	Tags []string

	// Creation time for this bookmark. Defaults to current
	// time. Datestamps more than 10 minutes ahead of server time
	// will be reset to current server time.
	Dt time.Time

	// Replace any existing bookmark with this URL. Default is
	// yes. If set to no, will throw an error if bookmark exists.
	Replace bool

	// Make bookmark public. Default is "yes" unless user has
	// enabled the "save all bookmarks as private" user setting,
	// in which case default is "no".
	Shared bool

	// Marks the bookmark as unread. Default is "no".
	Toread bool
}

// PostsAdd adds a bookmark.
//
// https://pinboard.in/api/#posts_add
func PostsAdd(opt *PostsAddOptions) error {
	if opt.URL == "" {
		return errors.New("error: missing url")
	}

	if opt.Description == "" {
		return errors.New("error: missing description")
	}

	resp, err := get("postsAdd", opt)
	if err != nil {
		return err
	}

	var pr postsResponse
	err = json.Unmarshal(resp, &pr)
	if err != nil {
		return err
	}

	if pr.ResultCode != "done" {
		return errors.New(pr.ResultCode)
	}

	return nil
}

// postsDeleteOptions represents the single required argument for
// deleting a bookmark.
type postsDeleteOptions struct {
	URL string
}

// PostsDelete deletes the bookmark by url.
//
// https://pinboard.in/api/#posts_delete
func PostsDelete(url string) error {
	resp, err := get("postsDelete", &postsDeleteOptions{URL: url})
	if err != nil {
		return err
	}

	var pr postsResponse
	err = json.Unmarshal(resp, &pr)
	if err != nil {
		return err
	}

	if pr.ResultCode != "done" {
		return errors.New(pr.ResultCode)
	}

	return nil
}

// PostsGetOptions represents the optional arguments for getting
// bookmarks.
type PostsGetOptions struct {
	// Filter by up to three tags.
	Tag []string

	// Return results bookmarked on this day. UTC date in this
	// format: 2010-12-11.
	Dt time.Time

	// Return bookmark for this URL.
	URL string

	// Include a change detection signature in a meta attribute.
	Meta bool
}

// PostsGet returns one or more posts (on a single day) matching the
// arguments. If no date or URL is given, date of most recent bookmark
// will be used.Returns one or more posts on a single day matching the
// arguments. If no date or URL is given, date of most recent bookmark
// will be used.
//
// https://pinboard.in/api/#posts_get
func PostsGet(opt *PostsGetOptions) ([]*Post, error) {
	resp, err := get("postsGet", opt)
	if err != nil {
		return nil, err
	}

	var pr postsResponse
	err = json.Unmarshal(resp, &pr)
	if err != nil {
		return nil, err
	}

	var posts []*Post
	for _, p := range pr.Posts {
		post, err := p.toPost()
		if err != nil {
			return nil, err
		}

		posts = append(posts, post)
	}

	return posts, nil
}

// PostsRecentOptions represents the optional arguments for returning
// the user's most recent posts.
type PostsRecentOptions struct {
	// Filter by up to three tags.
	Tag []string

	// Number of results to return. Default is 15, max is 100.
	Count int
}

// PostsRecent returns a list of the user's most recent posts,
// filtered by tag.
//
// https://pinboard.in/api/#posts_recent
func PostsRecent(opt *PostsRecentOptions) ([]*Post, error) {
	resp, err := get("postsRecent", opt)
	if err != nil {
		return nil, err
	}

	var pr postsResponse
	err = json.Unmarshal(resp, &pr)
	if err != nil {
		return nil, err
	}

	var posts []*Post
	for _, p := range pr.Posts {
		post, err := p.toPost()
		if err != nil {
			return nil, err
		}

		posts = append(posts, post)
	}

	return posts, nil
}

// postsDatesResponse represents the response from /posts/dates.
type postsDatesResponse struct {
	User  string         `json:"user"`
	Tag   string         `json:"tag"`
	Dates map[string]int `json:"dates"`
}

// PostsDatesOptions represents the single optional argument for
// returning a list of dates with the number of posts at each date.
type PostsDatesOptions struct {
	// Filter by up to three tags.
	Tag []string
}

// PostsDates returns a list of dates with the number of posts at each
// date.
//
// https://pinboard.in/api/#posts_dates
func PostsDates(opt *PostsDatesOptions) (map[string]int, error) {
	resp, err := get("postsDates", opt)
	if err != nil {
		return nil, err
	}

	var pr postsDatesResponse
	err = json.Unmarshal(resp, &pr)
	if err != nil {
		return nil, err
	}

	return pr.Dates, nil
}

// PostsAllOptions represents the optional arguments for returning all
// bookmarks in the user's account.
type PostsAllOptions struct {
	// Filter by up to three tags.
	Tag []string

	// Offset value (default is 0).
	Start int

	// Number of results to return. Default is all.
	Results int

	// Return only bookmarks created after this time.
	Fromdt time.Time

	// Return only bookmarks created before this time.
	Todt time.Time

	// Include a change detection signature for each bookmark.
	//
	// Note: This probably doesn't work. A meta field is always
	// returned. The Pinboard API says the datatype is an int but
	// changing the value has no impact on the results. Using a
	// yes/no string like all the other meta options doesn't work
	// either.
	Meta int
}

// PostsAll returns all bookmarks in the user's account.
//
// https://pinboard.in/api/#posts_all
func PostsAll(opt *PostsAllOptions) ([]*Post, error) {
	resp, err := get("postsAll", opt)
	if err != nil {
		return nil, err
	}

	var pr []post
	err = json.Unmarshal(resp, &pr)
	if err != nil {
		return nil, err
	}

	var posts []*Post
	for _, p := range pr {
		post, err := p.toPost()
		if err != nil {
			return nil, err
		}

		posts = append(posts, post)
	}

	return posts, nil
}

// postSuggestResponse represents the response from /posts/suggest.
type postsSuggestResponse struct {
	Popular     []string `json:"popular"`
	Recommended []string `json:"recommended"`
}

// postSuggestOptions represents the single required argument, url,
// for suggesting tags for a post.
type postsSuggestOptions struct {
	URL string
}

// PostsSuggestPopular returns a slice of popular tags for a given
// URL. Popular tags are tags used site-wide for the url.
//
// https://pinboard.in/api/#posts_suggest
func PostsSuggestPopular(url string) ([]string, error) {
	resp, err := get("postsSuggest", &postsSuggestOptions{URL: url})
	if err != nil {
		return nil, err
	}

	var pr []postsSuggestResponse
	err = json.Unmarshal(resp, &pr)
	if err != nil {
		return nil, err
	}

	return pr[0].Popular, nil
}

// PostsSuggestRecommended returns a slice of recommended tags for a
// given URL. Recommended tags are drawn from the user's own tags.
//
// https://pinboard.in/api/#posts_suggest
func PostsSuggestRecommended(url string) ([]string, error) {
	resp, err := get("postsSuggest", &postsSuggestOptions{URL: url})
	if err != nil {
		return nil, err
	}

	var pr []postsSuggestResponse
	err = json.Unmarshal(resp, &pr)
	if err != nil {
		return nil, err
	}

	return pr[1].Recommended, nil
}

// UnmarshalJSON converts a `descriptionType` into a `string`.
func (d *descriptionType) UnmarshalJSON(data []byte) error {
	// Have to do the type dance to avoid an infinite loop.
	type descriptionTypeAlias descriptionType
	var d2 descriptionTypeAlias

	if err := json.Unmarshal(data, &d2); err != nil {
		d2 = ""
	}
	*d = descriptionType(d2)

	return nil
}

// String returns the `string` value of our `descriptionType`.
func (d *descriptionType) String() string {
	return string(*d)
}
<|endoftext|>