<commit_before>/*
 * This file is part of the libvirt-go-xml project
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 * Copyright (C) 2016 Red Hat, Inc.
 *
 */

package libvirtxml

import (
	"encoding/xml"
)

type CapsHostCPUTopology struct {
	Sockets int `xml:"sockets,attr"`
	Cores   int `xml:"cores,attr"`
	Threads int `xml:"threads,attr"`
}

type CapsHostCPUFeatureFlag struct {
	Name string `xml:"name,attr"`
}

type CapsHostCPUPageSize struct {
	Size int    `xml:"size,attr"`
	Unit string `xml:"unit,attr"`
}

type CapsHostCPU struct {
	XMLName      xml.Name                 `xml:"cpu"`
	Arch         string                   `xml:"arch,omitempty"`
	Model        string                   `xml:"model,omitempty"`
	Vendor       string                   `xml:"vendor,omitempty"`
	Topology     *CapsHostCPUTopology     `xml:"topology"`
	FeatureFlags []CapsHostCPUFeatureFlag `xml:"feature"`
	Features     *CapsHostCPUFeatures     `xml:"features"`
	PageSizes    []CapsHostCPUPageSize    `xml:"pages"`
}

type CapsHostCPUFeature struct {
}

type CapsHostCPUFeatures struct {
	PAE    *CapsHostCPUFeature `xml:"pae"`
	NonPAE *CapsHostCPUFeature `xml:"nonpae"`
	SVM    *CapsHostCPUFeature `xml:"svm"`
	VMX    *CapsHostCPUFeature `xml:"vmx"`
}

type CapsHostNUMAMemory struct {
	Size uint64 `xml:",chardata"`
	Unit string `xml:"unit,attr"`
}

type CapsHostNUMAPageInfo struct {
	Size  int    `xml:"size,attr"`
	Unit  string `xml:"unit,attr"`
	Count uint64 `xml:",chardata"`
}

type CapsHostNUMACPU struct {
	ID       int    `xml:"id,attr"`
	SocketID *int   `xml:"socket_id,attr"`
	CoreID   *int   `xml:"core_id,attr"`
	Siblings string `xml:"siblings,attr,omitempty"`
}

type CapsHostNUMASibling struct {
	ID    int `xml:"id,attr"`
	Value int `xml:"value,attr"`
}

type CapsHostNUMACell struct {
	ID        int                    `xml:"id,attr"`
	Memory    *CapsHostNUMAMemory    `xml:"memory"`
	PageInfo  []CapsHostNUMAPageInfo `xml:"pages"`
	Distances *CapsHostNUMADistances `xml:"distances"`
	CPUS      *CapsHostNUMACPUs      `xml:"cpus"`
}

type CapsHostNUMADistances struct {
	Siblings []CapsHostNUMASibling `xml:"sibling"`
}

type CapsHostNUMACPUs struct {
	Num  uint              `xml:"num,attr,omitempty"`
	CPUs []CapsHostNUMACPU `xml:"cpu"`
}

type CapsHostNUMATopology struct {
	Cells *CapsHostNUMACells `xml:"cells"`
}

type CapsHostNUMACells struct {
	Num   uint               `xml:"num,attr,omitempty"`
	Cells []CapsHostNUMACell `xml:"cell"`
}

type CapsHostSecModelLabel struct {
	Type  string `xml:"type,attr"`
	Value string `xml:",chardata"`
}

type CapsHostSecModel struct {
	Name   string                  `xml:"model"`
	DOI    string                  `xml:"doi"`
	Labels []CapsHostSecModelLabel `xml:"baselabel"`
}

type CapsHostMigrationFeatures struct {
	Live          *CapsHostMigrationLive          `xml:"live"`
	URITransports *CapsHostMigrationURITransports `xml:"uri_transports"`
}

type CapsHostMigrationLive struct {
}

type CapsHostMigrationURITransports struct {
	URI []string `xml:"uri_transport"`
}

type CapsHost struct {
	UUID              string                     `xml:"uuid,omitempty"`
	CPU               *CapsHostCPU               `xml:"cpu"`
	PowerManagement   *CapsHostPowerManagement   `xml:"power_management"`
	MigrationFeatures *CapsHostMigrationFeatures `xml:"migration_features"`
	NUMA              *CapsHostNUMATopology      `xml:"topology"`
	Cache             *CapsHostCache             `xml:"cache"`
	SecModel          []CapsHostSecModel         `xml:"secmodel"`
}

type CapsHostPowerManagement struct {
	SuspendMem    *CapsHostPowerManagementMode `xml:"suspend_mem"`
	SuspendDisk   *CapsHostPowerManagementMode `xml:"suspend_disk"`
	SuspendHybrid *CapsHostPowerManagementMode `xml:"suspend_hybrid"`
}

type CapsHostPowerManagementMode struct {
}

type CapsHostCache struct {
	Banks []CapsHostCacheBank `xml:"bank"`
}

type CapsHostCacheBank struct {
	ID      uint                   `xml:"id,attr"`
	Level   uint                   `xml:"level,attr"`
	Type    string                 `xml:"type,attr"`
	Size    uint                   `xml:"size,attr"`
	Unit    string                 `xml:"unit,attr"`
	CPUs    string                 `xml:"cpus,attr"`
	Control []CapsHostCacheControl `xml:"control"`
}

type CapsHostCacheControl struct {
	Granularity uint   `xml:"granularity,attr"`
	Min         uint   `xml:"min,attr,omitempty"`
	Unit        string `xml:"unit,attr"`
	Type        string `xml:"type,attr"`
	MaxAllows   uint   `xml:"maxAllocs,attr"`
}

type CapsGuestMachine struct {
	Name      string `xml:",chardata"`
	MaxCPUs   int    `xml:"maxCpus,attr,omitempty"`
	Canonical string `xml:"canonical,attr,omitempty"`
}

type CapsGuestDomain struct {
	Type     string             `xml:"type,attr"`
	Emulator string             `xml:"emulator,omitempty"`
	Machines []CapsGuestMachine `xml:"machine"`
}

type CapsGuestArch struct {
	Name     string             `xml:"name,attr"`
	WordSize string             `xml:"wordsize"`
	Emulator string             `xml:"emulator"`
	Loader   string             `xml:"loader,omitempty"`
	Machines []CapsGuestMachine `xml:"machine"`
	Domains  []CapsGuestDomain  `xml:"domain"`
}

type CapsGuestFeatureCPUSelection struct {
}

type CapsGuestFeatureDeviceBoot struct {
}

type CapsGuestFeaturePAE struct {
}

type CapsGuestFeatureNonPAE struct {
}

type CapsGuestFeatureDiskSnapshot struct {
	Default string `xml:"default,attr,omitempty"`
	Toggle  string `xml:"toggle,attr,omitempty"`
}

type CapsGuestFeatureAPIC struct {
	Default string `xml:"default,attr,omitempty"`
	Toggle  string `xml:"toggle,attr,omitempty"`
}

type CapsGuestFeatureACPI struct {
	Default string `xml:"default,attr,omitempty"`
	Toggle  string `xml:"toggle,attr,omitempty"`
}

type CapsGuestFeatureIA64BE struct {
}

type CapsGuestFeatures struct {
	CPUSelection *CapsGuestFeatureCPUSelection `xml:"cpuselection"`
	DeviceBoot   *CapsGuestFeatureDeviceBoot   `xml:"deviceboot"`
	DiskSnapshot *CapsGuestFeatureDiskSnapshot `xml:"disksnapshot"`
	PAE          *CapsGuestFeaturePAE          `xml:"pae"`
	NonPAE       *CapsGuestFeatureNonPAE       `xml:"nonpae"`
	APIC         *CapsGuestFeatureAPIC         `xml:"apic"`
	ACPI         *CapsGuestFeatureACPI         `xml:"acpi"`
	IA64BE       *CapsGuestFeatureIA64BE       `xml:"ia64_be"`
}

type CapsGuest struct {
	OSType   string             `xml:"os_type"`
	Arch     CapsGuestArch      `xml:"arch"`
	Features *CapsGuestFeatures `xml:"features"`
}

type Caps struct {
	XMLName xml.Name    `xml:"capabilities"`
	Host    CapsHost    `xml:"host"`
	Guests  []CapsGuest `xml:"guest"`
}

func (c *CapsHostCPU) Unmarshal(doc string) error {
	return xml.Unmarshal([]byte(doc), c)
}

func (c *CapsHostCPU) Marshal() (string, error) {
	doc, err := xml.MarshalIndent(c, "", "  ")
	if err != nil {
		return "", err
	}
	return string(doc), nil
}

func (c *Caps) Unmarshal(doc string) error {
	return xml.Unmarshal([]byte(doc), c)
}

func (c *Caps) Marshal() (string, error) {
	doc, err := xml.MarshalIndent(c, "", "  ")
	if err != nil {
		return "", err
	}
	return string(doc), nil
}
<commit_msg>Include parsing of CPU microcode version<commit_after>/*
 * This file is part of the libvirt-go-xml project
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 * Copyright (C) 2016 Red Hat, Inc.
 *
 */

package libvirtxml

import (
	"encoding/xml"
)

type CapsHostCPUTopology struct {
	Sockets int `xml:"sockets,attr"`
	Cores   int `xml:"cores,attr"`
	Threads int `xml:"threads,attr"`
}

type CapsHostCPUFeatureFlag struct {
	Name string `xml:"name,attr"`
}

type CapsHostCPUPageSize struct {
	Size int    `xml:"size,attr"`
	Unit string `xml:"unit,attr"`
}

type CapsHostCPUMicrocode struct {
	Version int `xml:"version,attr"`
}

type CapsHostCPU struct {
	XMLName      xml.Name                 `xml:"cpu"`
	Arch         string                   `xml:"arch,omitempty"`
	Model        string                   `xml:"model,omitempty"`
	Vendor       string                   `xml:"vendor,omitempty"`
	Topology     *CapsHostCPUTopology     `xml:"topology"`
	FeatureFlags []CapsHostCPUFeatureFlag `xml:"feature"`
	Features     *CapsHostCPUFeatures     `xml:"features"`
	PageSizes    []CapsHostCPUPageSize    `xml:"pages"`
	Microcode    *CapsHostCPUMicrocode    `xml:"microcode"`
}

type CapsHostCPUFeature struct {
}

type CapsHostCPUFeatures struct {
	PAE    *CapsHostCPUFeature `xml:"pae"`
	NonPAE *CapsHostCPUFeature `xml:"nonpae"`
	SVM    *CapsHostCPUFeature `xml:"svm"`
	VMX    *CapsHostCPUFeature `xml:"vmx"`
}

type CapsHostNUMAMemory struct {
	Size uint64 `xml:",chardata"`
	Unit string `xml:"unit,attr"`
}

type CapsHostNUMAPageInfo struct {
	Size  int    `xml:"size,attr"`
	Unit  string `xml:"unit,attr"`
	Count uint64 `xml:",chardata"`
}

type CapsHostNUMACPU struct {
	ID       int    `xml:"id,attr"`
	SocketID *int   `xml:"socket_id,attr"`
	CoreID   *int   `xml:"core_id,attr"`
	Siblings string `xml:"siblings,attr,omitempty"`
}

type CapsHostNUMASibling struct {
	ID    int `xml:"id,attr"`
	Value int `xml:"value,attr"`
}

type CapsHostNUMACell struct {
	ID        int                    `xml:"id,attr"`
	Memory    *CapsHostNUMAMemory    `xml:"memory"`
	PageInfo  []CapsHostNUMAPageInfo `xml:"pages"`
	Distances *CapsHostNUMADistances `xml:"distances"`
	CPUS      *CapsHostNUMACPUs      `xml:"cpus"`
}

type CapsHostNUMADistances struct {
	Siblings []CapsHostNUMASibling `xml:"sibling"`
}

type CapsHostNUMACPUs struct {
	Num  uint              `xml:"num,attr,omitempty"`
	CPUs []CapsHostNUMACPU `xml:"cpu"`
}

type CapsHostNUMATopology struct {
	Cells *CapsHostNUMACells `xml:"cells"`
}

type CapsHostNUMACells struct {
	Num   uint               `xml:"num,attr,omitempty"`
	Cells []CapsHostNUMACell `xml:"cell"`
}

type CapsHostSecModelLabel struct {
	Type  string `xml:"type,attr"`
	Value string `xml:",chardata"`
}

type CapsHostSecModel struct {
	Name   string                  `xml:"model"`
	DOI    string                  `xml:"doi"`
	Labels []CapsHostSecModelLabel `xml:"baselabel"`
}

type CapsHostMigrationFeatures struct {
	Live          *CapsHostMigrationLive          `xml:"live"`
	URITransports *CapsHostMigrationURITransports `xml:"uri_transports"`
}

type CapsHostMigrationLive struct {
}

type CapsHostMigrationURITransports struct {
	URI []string `xml:"uri_transport"`
}

type CapsHost struct {
	UUID              string                     `xml:"uuid,omitempty"`
	CPU               *CapsHostCPU               `xml:"cpu"`
	PowerManagement   *CapsHostPowerManagement   `xml:"power_management"`
	MigrationFeatures *CapsHostMigrationFeatures `xml:"migration_features"`
	NUMA              *CapsHostNUMATopology      `xml:"topology"`
	Cache             *CapsHostCache             `xml:"cache"`
	SecModel          []CapsHostSecModel         `xml:"secmodel"`
}

type CapsHostPowerManagement struct {
	SuspendMem    *CapsHostPowerManagementMode `xml:"suspend_mem"`
	SuspendDisk   *CapsHostPowerManagementMode `xml:"suspend_disk"`
	SuspendHybrid *CapsHostPowerManagementMode `xml:"suspend_hybrid"`
}

type CapsHostPowerManagementMode struct {
}

type CapsHostCache struct {
	Banks []CapsHostCacheBank `xml:"bank"`
}

type CapsHostCacheBank struct {
	ID      uint                   `xml:"id,attr"`
	Level   uint                   `xml:"level,attr"`
	Type    string                 `xml:"type,attr"`
	Size    uint                   `xml:"size,attr"`
	Unit    string                 `xml:"unit,attr"`
	CPUs    string                 `xml:"cpus,attr"`
	Control []CapsHostCacheControl `xml:"control"`
}

type CapsHostCacheControl struct {
	Granularity uint   `xml:"granularity,attr"`
	Min         uint   `xml:"min,attr,omitempty"`
	Unit        string `xml:"unit,attr"`
	Type        string `xml:"type,attr"`
	MaxAllows   uint   `xml:"maxAllocs,attr"`
}

type CapsGuestMachine struct {
	Name      string `xml:",chardata"`
	MaxCPUs   int    `xml:"maxCpus,attr,omitempty"`
	Canonical string `xml:"canonical,attr,omitempty"`
}

type CapsGuestDomain struct {
	Type     string             `xml:"type,attr"`
	Emulator string             `xml:"emulator,omitempty"`
	Machines []CapsGuestMachine `xml:"machine"`
}

type CapsGuestArch struct {
	Name     string             `xml:"name,attr"`
	WordSize string             `xml:"wordsize"`
	Emulator string             `xml:"emulator"`
	Loader   string             `xml:"loader,omitempty"`
	Machines []CapsGuestMachine `xml:"machine"`
	Domains  []CapsGuestDomain  `xml:"domain"`
}

type CapsGuestFeatureCPUSelection struct {
}

type CapsGuestFeatureDeviceBoot struct {
}

type CapsGuestFeaturePAE struct {
}

type CapsGuestFeatureNonPAE struct {
}

type CapsGuestFeatureDiskSnapshot struct {
	Default string `xml:"default,attr,omitempty"`
	Toggle  string `xml:"toggle,attr,omitempty"`
}

type CapsGuestFeatureAPIC struct {
	Default string `xml:"default,attr,omitempty"`
	Toggle  string `xml:"toggle,attr,omitempty"`
}

type CapsGuestFeatureACPI struct {
	Default string `xml:"default,attr,omitempty"`
	Toggle  string `xml:"toggle,attr,omitempty"`
}

type CapsGuestFeatureIA64BE struct {
}

type CapsGuestFeatures struct {
	CPUSelection *CapsGuestFeatureCPUSelection `xml:"cpuselection"`
	DeviceBoot   *CapsGuestFeatureDeviceBoot   `xml:"deviceboot"`
	DiskSnapshot *CapsGuestFeatureDiskSnapshot `xml:"disksnapshot"`
	PAE          *CapsGuestFeaturePAE          `xml:"pae"`
	NonPAE       *CapsGuestFeatureNonPAE       `xml:"nonpae"`
	APIC         *CapsGuestFeatureAPIC         `xml:"apic"`
	ACPI         *CapsGuestFeatureACPI         `xml:"acpi"`
	IA64BE       *CapsGuestFeatureIA64BE       `xml:"ia64_be"`
}

type CapsGuest struct {
	OSType   string             `xml:"os_type"`
	Arch     CapsGuestArch      `xml:"arch"`
	Features *CapsGuestFeatures `xml:"features"`
}

type Caps struct {
	XMLName xml.Name    `xml:"capabilities"`
	Host    CapsHost    `xml:"host"`
	Guests  []CapsGuest `xml:"guest"`
}

func (c *CapsHostCPU) Unmarshal(doc string) error {
	return xml.Unmarshal([]byte(doc), c)
}

func (c *CapsHostCPU) Marshal() (string, error) {
	doc, err := xml.MarshalIndent(c, "", "  ")
	if err != nil {
		return "", err
	}
	return string(doc), nil
}

func (c *Caps) Unmarshal(doc string) error {
	return xml.Unmarshal([]byte(doc), c)
}

func (c *Caps) Marshal() (string, error) {
	doc, err := xml.MarshalIndent(c, "", "  ")
	if err != nil {
		return "", err
	}
	return string(doc), nil
}
<|endoftext|>


<commit_before>package models

import (
	"appengine"
	"appengine/datastore"
	"fmt"
	"time"
)

type Flag struct {
	Name  string
	Value string
}

func SetFlag(c appengine.Context, flag string, value string) error {
	e := new(Flag)
	e.Name = flag
	e.Value = value

	k := datastore.NewKey(c, "Flag", flag, 0, nil)
	_, err := datastore.Put(c, k, e)
	if err == nil {
		c.Infof("Wrote %+v", e)
	} else {
		c.Warningf("Error writing entry: %v", e)
	}
	return err
}

func GetFlag(c appengine.Context, flag string) (string, error) {
	var retrieved Flag
	k := datastore.NewKey(c, "Flag", flag, 0, nil)
	err := datastore.Get(c, k, &retrieved)
	if err != nil {
		return "", err
	}

	return retrieved.Value, nil
}
<commit_msg>fix flag deps<commit_after>package models

import (
	"appengine"
	"appengine/datastore"
)

type Flag struct {
	Name  string
	Value string
}

func SetFlag(c appengine.Context, flag string, value string) error {
	e := new(Flag)
	e.Name = flag
	e.Value = value

	k := datastore.NewKey(c, "Flag", flag, 0, nil)
	_, err := datastore.Put(c, k, e)
	if err == nil {
		c.Infof("Wrote %+v", e)
	} else {
		c.Warningf("Error writing entry: %v", e)
	}
	return err
}

func GetFlag(c appengine.Context, flag string) (string, error) {
	var retrieved Flag
	k := datastore.NewKey(c, "Flag", flag, 0, nil)
	err := datastore.Get(c, k, &retrieved)
	if err != nil {
		return "", err
	}

	return retrieved.Value, nil
}
<|endoftext|>


<commit_before>package out

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"reflect"
	"strconv"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/pivotal-cf-experimental/pivnet-resource/concourse"
	"github.com/pivotal-cf-experimental/pivnet-resource/globs"
	"github.com/pivotal-cf-experimental/pivnet-resource/logger"
	"github.com/pivotal-cf-experimental/pivnet-resource/md5"
	"github.com/pivotal-cf-experimental/pivnet-resource/metadata"
	"github.com/pivotal-cf-experimental/pivnet-resource/pivnet"
	"github.com/pivotal-cf-experimental/pivnet-resource/s3"
	"github.com/pivotal-cf-experimental/pivnet-resource/uploader"
	"github.com/pivotal-cf-experimental/pivnet-resource/useragent"
	"github.com/pivotal-cf-experimental/pivnet-resource/validator"
	"github.com/pivotal-cf-experimental/pivnet-resource/versions"
)

const (
	defaultBucket = "pivotalnetwork"
	defaultRegion = "eu-west-1"
)

type OutCommand struct {
	binaryVersion   string
	logger          logger.Logger
	outDir          string
	sourcesDir      string
	logFilePath     string
	s3OutBinaryName string
}

type OutCommandConfig struct {
	BinaryVersion   string
	Logger          logger.Logger
	OutDir          string
	SourcesDir      string
	LogFilePath     string
	S3OutBinaryName string
}

func NewOutCommand(config OutCommandConfig) *OutCommand {
	return &OutCommand{
		binaryVersion:   config.BinaryVersion,
		logger:          config.Logger,
		outDir:          config.OutDir,
		sourcesDir:      config.SourcesDir,
		logFilePath:     config.LogFilePath,
		s3OutBinaryName: config.S3OutBinaryName,
	}
}

func (c *OutCommand) Run(input concourse.OutRequest) (concourse.OutResponse, error) {
	if c.outDir == "" {
		return concourse.OutResponse{}, fmt.Errorf("%s must be provided", "out dir")
	}

	var m metadata.Metadata
	var skipFileCheck bool
	if input.Params.MetadataFile != "" {
		metadataFilepath := filepath.Join(c.sourcesDir, input.Params.MetadataFile)
		metadataBytes, err := ioutil.ReadFile(metadataFilepath)
		if err != nil {
			return concourse.OutResponse{}, fmt.Errorf("metadata_file could not be read: %s", err.Error())
		}

		err = yaml.Unmarshal(metadataBytes, &m)
		if err != nil {
			return concourse.OutResponse{}, fmt.Errorf("metadata_file could not be parsed: %s", err.Error())
		}

		err = m.Validate()
		if err != nil {
			return concourse.OutResponse{}, fmt.Errorf("metadata_file is invalid: %s", err.Error())
		}

		skipFileCheck = true
	}

	c.logger.Debugf("metadata product_files parsed; contents: %+v\n", m.ProductFiles)
	c.logger.Debugf("metadata release parsed; contents: %+v\n", &m.Release)

	if !skipFileCheck {
		c.logger.Debugf("\x1b[31mDEPRECATION WARNING: files containing metadata, like %q, will be removed in a future release\x1b[0m", "versionFile")

		screenWriter := log.New(os.Stdout, "", 0)
		screenWriter.Printf("\x1b[31mDEPRECATION WARNING: files containing metadata, like %q, will be removed in a future release\x1b[0m", "versionFile")
	}

	err := validator.NewOutValidator(input, skipFileCheck).Validate()
	if err != nil {
		return concourse.OutResponse{}, err
	}

	c.logger.Debugf("Received input: %+v\n", input)

	globber := globs.NewGlobber(globs.GlobberConfig{
		FileGlob:   input.Params.FileGlob,
		SourcesDir: c.sourcesDir,
		Logger:     c.logger,
	})

	exactGlobs, err := globber.ExactGlobs()
	if err != nil {
		return concourse.OutResponse{}, err
	}

	var missingFiles []string
	for _, f := range m.ProductFiles {
		if !contains(exactGlobs, f.File) {
			missingFiles = append(missingFiles, f.File)
		}
	}

	if len(missingFiles) > 0 {
		return concourse.OutResponse{},
			fmt.Errorf("product_files were provided in metadata that match no globs: %v", missingFiles)
	}

	var endpoint string
	if input.Source.Endpoint != "" {
		endpoint = input.Source.Endpoint
	} else {
		endpoint = pivnet.Endpoint
	}

	productSlug := input.Source.ProductSlug

	clientConfig := pivnet.NewClientConfig{
		Endpoint:  endpoint,
		Token:     input.Source.APIToken,
		UserAgent: useragent.UserAgent(c.binaryVersion, "put", productSlug),
	}
	pivnetClient := pivnet.NewClient(
		clientConfig,
		c.logger,
	)

	productVersion := fetchFromMetadataOrFile("Version", m, skipFileCheck, c.sourcesDir, input.Params.VersionFile)

	releases, err := pivnetClient.ReleasesForProductSlug(productSlug)
	if err != nil {
		return concourse.OutResponse{}, err
	}

	existingVersions, err := pivnetClient.ProductVersions(productSlug, releases)
	if err != nil {
		return concourse.OutResponse{}, err
	}

	for _, v := range existingVersions {
		if v == productVersion {
			return concourse.OutResponse{}, fmt.Errorf("release already exists with version: %s", productVersion)
		}
	}

	config := pivnet.CreateReleaseConfig{
		ProductSlug:     productSlug,
		ReleaseType:     fetchFromMetadataOrFile("ReleaseType", m, skipFileCheck, c.sourcesDir, input.Params.ReleaseTypeFile),
		EulaSlug:        fetchFromMetadataOrFile("EulaSlug", m, skipFileCheck, c.sourcesDir, input.Params.EulaSlugFile),
		ProductVersion:  productVersion,
		Description:     fetchFromMetadataOrFile("Description", m, skipFileCheck, c.sourcesDir, input.Params.DescriptionFile),
		ReleaseNotesURL: fetchFromMetadataOrFile("ReleaseNotesURL", m, skipFileCheck, c.sourcesDir, input.Params.ReleaseNotesURLFile),
		ReleaseDate:     fetchFromMetadataOrFile("ReleaseDate", m, skipFileCheck, c.sourcesDir, input.Params.ReleaseDateFile),
	}

	c.logger.Debugf("config used to create pivnet release: %+v\n", config)

	release, err := pivnetClient.CreateRelease(config)
	if err != nil {
		log.Fatalln(err)
	}

	skipUpload := input.Params.FileGlob == "" && input.Params.FilepathPrefix == ""
	if skipUpload {
		c.logger.Debugf("File glob and s3_filepath_prefix not provided - skipping upload to s3")
	} else {
		bucket := input.Source.Bucket
		if bucket == "" {
			bucket = defaultBucket
		}

		region := input.Source.Region
		if region == "" {
			region = defaultRegion
		}

		logFile, err := os.OpenFile(c.logFilePath, os.O_APPEND|os.O_WRONLY, os.ModeAppend)
		if err != nil {
			panic(err)
		}

		s3Client := s3.NewClient(s3.NewClientConfig{
			AccessKeyID:     input.Source.AccessKeyID,
			SecretAccessKey: input.Source.SecretAccessKey,
			RegionName:      region,
			Bucket:          bucket,

			Logger: c.logger,

			Stdout: os.Stdout,
			Stderr: logFile,

			OutBinaryPath: filepath.Join(c.outDir, c.s3OutBinaryName),
		})

		uploaderClient := uploader.NewClient(uploader.Config{
			FilepathPrefix: input.Params.FilepathPrefix,
			SourcesDir:     c.sourcesDir,

			Logger: c.logger,

			Transport: s3Client,
		})

		for _, exactGlob := range exactGlobs {
			fullFilepath := filepath.Join(c.sourcesDir, exactGlob)
			fileContentsMD5, err := md5.NewFileContentsSummer(fullFilepath).Sum()
			if err != nil {
				log.Fatalln(err)
			}

			remotePath, err := uploaderClient.UploadFile(exactGlob)
			if err != nil {
				return concourse.OutResponse{}, err
			}

			product, err := pivnetClient.FindProductForSlug(productSlug)
			if err != nil {
				log.Fatalln(err)
			}

			filename := filepath.Base(exactGlob)

			var description string
			uploadAs := filename
			for _, f := range m.ProductFiles {
				if f.File == exactGlob {
					c.logger.Debugf("exact glob '%s' matches metadata file: '%s'\n", exactGlob, f.File)
					description = f.Description
					if f.UploadAs != "" {
						c.logger.Debugf("upload_as provided for exact glob: '%s' - uploading to remote filename: '%s' instead\n", exactGlob, f.UploadAs)
						uploadAs = f.UploadAs
					}
				} else {
					c.logger.Debugf("exact glob %s does not match metadata file: %s\n", exactGlob, f.File)
				}
			}

			c.logger.Debugf(
				"Creating product file: {product_slug: %s, filename: %s, aws_object_key: %s, file_version: %s, description: %s}\n",
				productSlug,
				uploadAs,
				remotePath,
				release.Version,
				description,
			)

			productFile, err := pivnetClient.CreateProductFile(pivnet.CreateProductFileConfig{
				ProductSlug:  productSlug,
				Name:         uploadAs,
				AWSObjectKey: remotePath,
				FileVersion:  release.Version,
				MD5:          fileContentsMD5,
				Description:  description,
			})
			if err != nil {
				return concourse.OutResponse{}, err
			}

			c.logger.Debugf(
				"Adding product file: {product_slug: %s, product_id: %d, filename: %s, product_file_id: %d, release_id: %d}\n",
				productSlug,
				product.ID,
				filename,
				productFile.ID,
				release.ID,
			)

			err = pivnetClient.AddProductFile(product.ID, release.ID, productFile.ID)
			if err != nil {
				log.Fatalln(err)
			}
		}

		if err != nil {
			log.Fatal(err)
		}
	}

	availability := fetchFromMetadataOrFile("Availability", m, skipFileCheck, c.sourcesDir, input.Params.AvailabilityFile)
	if availability != "Admins Only" {
		releaseUpdate := pivnet.Release{
			ID:           release.ID,
			Availability: availability,
		}
		release, err = pivnetClient.UpdateRelease(productSlug, releaseUpdate)
		if err != nil {
			log.Fatalln(err)
		}

		if availability == "Selected User Groups Only" {
			userGroupIDs := strings.Split(
				fetchFromMetadataOrFile("UserGroupIDs", m, skipFileCheck, c.sourcesDir, input.Params.UserGroupIDsFile),
				",",
			)

			for _, userGroupIDString := range userGroupIDs {
				userGroupID, err := strconv.Atoi(userGroupIDString)
				if err != nil {
					log.Fatalln(err)
				}

				pivnetClient.AddUserGroup(productSlug, release.ID, userGroupID)
			}
		}
	}

	releaseETag, err := pivnetClient.ReleaseETag(productSlug, release)
	//TODO this should not panic
	if err != nil {
		panic(err)
	}

	outputVersion, err := versions.CombineVersionAndETag(release.Version, releaseETag)
	//TODO this should not panic
	if err != nil {
		panic(err)
	}

	out := concourse.OutResponse{
		Version: concourse.Version{
			ProductVersion: outputVersion,
		},
		Metadata: []concourse.Metadata{
			{Name: "release_type", Value: release.ReleaseType},
			{Name: "release_date", Value: release.ReleaseDate},
			{Name: "description", Value: release.Description},
			{Name: "release_notes_url", Value: release.ReleaseNotesURL},
			{Name: "eula_slug", Value: release.Eula.Slug},
			{Name: "availability", Value: release.Availability},
		},
	}

	return out, nil
}

func fetchFromMetadataOrFile(yamlKey string, m metadata.Metadata, skipFileCheck bool, dir, file string) string {
	if skipFileCheck && m.Release != nil {
		metadataValue := reflect.ValueOf(m.Release).Elem()
		fieldValue := metadataValue.FieldByName(yamlKey)

		if yamlKey == "UserGroupIDs" {
			var ids []string
			for i := 0; i < fieldValue.Len(); i++ {
				ids = append(ids, fieldValue.Index(i).String())
			}

			return strings.Join(ids, ",")
		}

		return fieldValue.String()
	}

	return readStringContents(dir, file)
}

func readStringContents(sourcesDir, file string) string {
	if file == "" {
		return ""
	}
	fullPath := filepath.Join(sourcesDir, file)
	contents, err := ioutil.ReadFile(fullPath)
	if err != nil {
		log.Fatal(err)
	}
	return string(contents)
}

func contains(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}
<commit_msg>Deprecation prints to Stderr and nil check on Release<commit_after>package out

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"reflect"
	"strconv"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/pivotal-cf-experimental/pivnet-resource/concourse"
	"github.com/pivotal-cf-experimental/pivnet-resource/globs"
	"github.com/pivotal-cf-experimental/pivnet-resource/logger"
	"github.com/pivotal-cf-experimental/pivnet-resource/md5"
	"github.com/pivotal-cf-experimental/pivnet-resource/metadata"
	"github.com/pivotal-cf-experimental/pivnet-resource/pivnet"
	"github.com/pivotal-cf-experimental/pivnet-resource/s3"
	"github.com/pivotal-cf-experimental/pivnet-resource/uploader"
	"github.com/pivotal-cf-experimental/pivnet-resource/useragent"
	"github.com/pivotal-cf-experimental/pivnet-resource/validator"
	"github.com/pivotal-cf-experimental/pivnet-resource/versions"
)

const (
	defaultBucket = "pivotalnetwork"
	defaultRegion = "eu-west-1"
)

type OutCommand struct {
	binaryVersion   string
	logger          logger.Logger
	outDir          string
	sourcesDir      string
	logFilePath     string
	s3OutBinaryName string
}

type OutCommandConfig struct {
	BinaryVersion   string
	Logger          logger.Logger
	OutDir          string
	SourcesDir      string
	LogFilePath     string
	S3OutBinaryName string
}

func NewOutCommand(config OutCommandConfig) *OutCommand {
	return &OutCommand{
		binaryVersion:   config.BinaryVersion,
		logger:          config.Logger,
		outDir:          config.OutDir,
		sourcesDir:      config.SourcesDir,
		logFilePath:     config.LogFilePath,
		s3OutBinaryName: config.S3OutBinaryName,
	}
}

func (c *OutCommand) Run(input concourse.OutRequest) (concourse.OutResponse, error) {
	if c.outDir == "" {
		return concourse.OutResponse{}, fmt.Errorf("%s must be provided", "out dir")
	}

	var m metadata.Metadata
	var skipFileCheck bool
	if input.Params.MetadataFile != "" {
		metadataFilepath := filepath.Join(c.sourcesDir, input.Params.MetadataFile)
		metadataBytes, err := ioutil.ReadFile(metadataFilepath)
		if err != nil {
			return concourse.OutResponse{}, fmt.Errorf("metadata_file could not be read: %s", err.Error())
		}

		err = yaml.Unmarshal(metadataBytes, &m)
		if err != nil {
			return concourse.OutResponse{}, fmt.Errorf("metadata_file could not be parsed: %s", err.Error())
		}

		err = m.Validate()
		if err != nil {
			return concourse.OutResponse{}, fmt.Errorf("metadata_file is invalid: %s", err.Error())
		}

		skipFileCheck = true
	}

	c.logger.Debugf("metadata product_files parsed; contents: %+v\n", m.ProductFiles)

	if m.Release != nil {
		c.logger.Debugf("metadata release parsed; contents: %+v\n", *m.Release)
	}

	if !skipFileCheck {
		c.logger.Debugf("\x1b[31mDEPRECATION WARNING: files containing metadata, like %q, will be removed in a future release\x1b[0m", "versionFile")

		screenWriter := log.New(os.Stderr, "", 0)
		screenWriter.Printf("\x1b[31mDEPRECATION WARNING: files containing metadata, like %q, will be removed in a future release\x1b[0m", "versionFile")
	}

	err := validator.NewOutValidator(input, skipFileCheck).Validate()
	if err != nil {
		return concourse.OutResponse{}, err
	}

	c.logger.Debugf("Received input: %+v\n", input)

	globber := globs.NewGlobber(globs.GlobberConfig{
		FileGlob:   input.Params.FileGlob,
		SourcesDir: c.sourcesDir,
		Logger:     c.logger,
	})

	exactGlobs, err := globber.ExactGlobs()
	if err != nil {
		return concourse.OutResponse{}, err
	}

	var missingFiles []string
	for _, f := range m.ProductFiles {
		if !contains(exactGlobs, f.File) {
			missingFiles = append(missingFiles, f.File)
		}
	}

	if len(missingFiles) > 0 {
		return concourse.OutResponse{},
			fmt.Errorf("product_files were provided in metadata that match no globs: %v", missingFiles)
	}

	var endpoint string
	if input.Source.Endpoint != "" {
		endpoint = input.Source.Endpoint
	} else {
		endpoint = pivnet.Endpoint
	}

	productSlug := input.Source.ProductSlug

	clientConfig := pivnet.NewClientConfig{
		Endpoint:  endpoint,
		Token:     input.Source.APIToken,
		UserAgent: useragent.UserAgent(c.binaryVersion, "put", productSlug),
	}
	pivnetClient := pivnet.NewClient(
		clientConfig,
		c.logger,
	)

	productVersion := fetchFromMetadataOrFile("Version", m, skipFileCheck, c.sourcesDir, input.Params.VersionFile)

	releases, err := pivnetClient.ReleasesForProductSlug(productSlug)
	if err != nil {
		return concourse.OutResponse{}, err
	}

	existingVersions, err := pivnetClient.ProductVersions(productSlug, releases)
	if err != nil {
		return concourse.OutResponse{}, err
	}

	for _, v := range existingVersions {
		if v == productVersion {
			return concourse.OutResponse{}, fmt.Errorf("release already exists with version: %s", productVersion)
		}
	}

	config := pivnet.CreateReleaseConfig{
		ProductSlug:     productSlug,
		ReleaseType:     fetchFromMetadataOrFile("ReleaseType", m, skipFileCheck, c.sourcesDir, input.Params.ReleaseTypeFile),
		EulaSlug:        fetchFromMetadataOrFile("EulaSlug", m, skipFileCheck, c.sourcesDir, input.Params.EulaSlugFile),
		ProductVersion:  productVersion,
		Description:     fetchFromMetadataOrFile("Description", m, skipFileCheck, c.sourcesDir, input.Params.DescriptionFile),
		ReleaseNotesURL: fetchFromMetadataOrFile("ReleaseNotesURL", m, skipFileCheck, c.sourcesDir, input.Params.ReleaseNotesURLFile),
		ReleaseDate:     fetchFromMetadataOrFile("ReleaseDate", m, skipFileCheck, c.sourcesDir, input.Params.ReleaseDateFile),
	}

	c.logger.Debugf("config used to create pivnet release: %+v\n", config)

	release, err := pivnetClient.CreateRelease(config)
	if err != nil {
		log.Fatalln(err)
	}

	skipUpload := input.Params.FileGlob == "" && input.Params.FilepathPrefix == ""
	if skipUpload {
		c.logger.Debugf("File glob and s3_filepath_prefix not provided - skipping upload to s3")
	} else {
		bucket := input.Source.Bucket
		if bucket == "" {
			bucket = defaultBucket
		}

		region := input.Source.Region
		if region == "" {
			region = defaultRegion
		}

		logFile, err := os.OpenFile(c.logFilePath, os.O_APPEND|os.O_WRONLY, os.ModeAppend)
		if err != nil {
			panic(err)
		}

		s3Client := s3.NewClient(s3.NewClientConfig{
			AccessKeyID:     input.Source.AccessKeyID,
			SecretAccessKey: input.Source.SecretAccessKey,
			RegionName:      region,
			Bucket:          bucket,

			Logger: c.logger,

			Stdout: os.Stdout,
			Stderr: logFile,

			OutBinaryPath: filepath.Join(c.outDir, c.s3OutBinaryName),
		})

		uploaderClient := uploader.NewClient(uploader.Config{
			FilepathPrefix: input.Params.FilepathPrefix,
			SourcesDir:     c.sourcesDir,

			Logger: c.logger,

			Transport: s3Client,
		})

		for _, exactGlob := range exactGlobs {
			fullFilepath := filepath.Join(c.sourcesDir, exactGlob)
			fileContentsMD5, err := md5.NewFileContentsSummer(fullFilepath).Sum()
			if err != nil {
				log.Fatalln(err)
			}

			remotePath, err := uploaderClient.UploadFile(exactGlob)
			if err != nil {
				return concourse.OutResponse{}, err
			}

			product, err := pivnetClient.FindProductForSlug(productSlug)
			if err != nil {
				log.Fatalln(err)
			}

			filename := filepath.Base(exactGlob)

			var description string
			uploadAs := filename
			for _, f := range m.ProductFiles {
				if f.File == exactGlob {
					c.logger.Debugf("exact glob '%s' matches metadata file: '%s'\n", exactGlob, f.File)
					description = f.Description
					if f.UploadAs != "" {
						c.logger.Debugf("upload_as provided for exact glob: '%s' - uploading to remote filename: '%s' instead\n", exactGlob, f.UploadAs)
						uploadAs = f.UploadAs
					}
				} else {
					c.logger.Debugf("exact glob %s does not match metadata file: %s\n", exactGlob, f.File)
				}
			}

			c.logger.Debugf(
				"Creating product file: {product_slug: %s, filename: %s, aws_object_key: %s, file_version: %s, description: %s}\n",
				productSlug,
				uploadAs,
				remotePath,
				release.Version,
				description,
			)

			productFile, err := pivnetClient.CreateProductFile(pivnet.CreateProductFileConfig{
				ProductSlug:  productSlug,
				Name:         uploadAs,
				AWSObjectKey: remotePath,
				FileVersion:  release.Version,
				MD5:          fileContentsMD5,
				Description:  description,
			})
			if err != nil {
				return concourse.OutResponse{}, err
			}

			c.logger.Debugf(
				"Adding product file: {product_slug: %s, product_id: %d, filename: %s, product_file_id: %d, release_id: %d}\n",
				productSlug,
				product.ID,
				filename,
				productFile.ID,
				release.ID,
			)

			err = pivnetClient.AddProductFile(product.ID, release.ID, productFile.ID)
			if err != nil {
				log.Fatalln(err)
			}
		}

		if err != nil {
			log.Fatal(err)
		}
	}

	availability := fetchFromMetadataOrFile("Availability", m, skipFileCheck, c.sourcesDir, input.Params.AvailabilityFile)
	if availability != "Admins Only" {
		releaseUpdate := pivnet.Release{
			ID:           release.ID,
			Availability: availability,
		}
		release, err = pivnetClient.UpdateRelease(productSlug, releaseUpdate)
		if err != nil {
			log.Fatalln(err)
		}

		if availability == "Selected User Groups Only" {
			userGroupIDs := strings.Split(
				fetchFromMetadataOrFile("UserGroupIDs", m, skipFileCheck, c.sourcesDir, input.Params.UserGroupIDsFile),
				",",
			)

			for _, userGroupIDString := range userGroupIDs {
				userGroupID, err := strconv.Atoi(userGroupIDString)
				if err != nil {
					log.Fatalln(err)
				}

				pivnetClient.AddUserGroup(productSlug, release.ID, userGroupID)
			}
		}
	}

	releaseETag, err := pivnetClient.ReleaseETag(productSlug, release)
	//TODO this should not panic
	if err != nil {
		panic(err)
	}

	outputVersion, err := versions.CombineVersionAndETag(release.Version, releaseETag)
	//TODO this should not panic
	if err != nil {
		panic(err)
	}

	out := concourse.OutResponse{
		Version: concourse.Version{
			ProductVersion: outputVersion,
		},
		Metadata: []concourse.Metadata{
			{Name: "release_type", Value: release.ReleaseType},
			{Name: "release_date", Value: release.ReleaseDate},
			{Name: "description", Value: release.Description},
			{Name: "release_notes_url", Value: release.ReleaseNotesURL},
			{Name: "eula_slug", Value: release.Eula.Slug},
			{Name: "availability", Value: release.Availability},
		},
	}

	return out, nil
}

func fetchFromMetadataOrFile(yamlKey string, m metadata.Metadata, skipFileCheck bool, dir, file string) string {
	if skipFileCheck && m.Release != nil {
		metadataValue := reflect.ValueOf(m.Release).Elem()
		fieldValue := metadataValue.FieldByName(yamlKey)

		if yamlKey == "UserGroupIDs" {
			var ids []string
			for i := 0; i < fieldValue.Len(); i++ {
				ids = append(ids, fieldValue.Index(i).String())
			}

			return strings.Join(ids, ",")
		}

		return fieldValue.String()
	}

	return readStringContents(dir, file)
}

func readStringContents(sourcesDir, file string) string {
	if file == "" {
		return ""
	}
	fullPath := filepath.Join(sourcesDir, file)
	contents, err := ioutil.ReadFile(fullPath)
	if err != nil {
		log.Fatal(err)
	}
	return string(contents)
}

func contains(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package container

import (
	"strings"

	executorpkg "github.com/docker/docker/daemon/cluster/executor"
	clustertypes "github.com/docker/docker/daemon/cluster/provider"
	"github.com/docker/engine-api/types"
	"github.com/docker/engine-api/types/network"
	networktypes "github.com/docker/libnetwork/types"
	"github.com/docker/swarmkit/agent/exec"
	"github.com/docker/swarmkit/api"
	"golang.org/x/net/context"
)

type executor struct {
	backend executorpkg.Backend
}

// NewExecutor returns an executor from the docker client.
func NewExecutor(b executorpkg.Backend) exec.Executor {
	return &executor{
		backend: b,
	}
}

// Describe returns the underlying node description from the docker client.
func (e *executor) Describe(ctx context.Context) (*api.NodeDescription, error) {
	info, err := e.backend.SystemInfo()
	if err != nil {
		return nil, err
	}

	var plugins []api.PluginDescription
	addPlugins := func(typ string, names []string) {
		for _, name := range names {
			plugins = append(plugins, api.PluginDescription{
				Type: typ,
				Name: name,
			})
		}
	}

	addPlugins("Volume", info.Plugins.Volume)
	// Add builtin driver "overlay" (the only builtin multi-host driver) to
	// the plugin list by default.
	addPlugins("Network", append([]string{"overlay"}, info.Plugins.Network...))
	addPlugins("Authorization", info.Plugins.Authorization)

	// parse []string labels into a map[string]string
	labels := map[string]string{}
	for _, l := range info.Labels {
		stringSlice := strings.SplitN(l, "=", 2)
		// this will take the last value in the list for a given key
		// ideally, one shouldn't assign multiple values to the same key
		if len(stringSlice) > 1 {
			labels[stringSlice[0]] = stringSlice[1]
		}
	}

	description := &api.NodeDescription{
		Hostname: info.Name,
		Platform: &api.Platform{
			Architecture: info.Architecture,
			OS:           info.OSType,
		},
		Engine: &api.EngineDescription{
			EngineVersion: info.ServerVersion,
			Labels:        labels,
			Plugins:       plugins,
		},
		Resources: &api.Resources{
			NanoCPUs:    int64(info.NCPU) * 1e9,
			MemoryBytes: info.MemTotal,
		},
	}

	return description, nil
}

func (e *executor) Configure(ctx context.Context, node *api.Node) error {
	na := node.Attachment
	if na == nil {
		return nil
	}

	options := types.NetworkCreate{
		Driver: na.Network.DriverState.Name,
		IPAM: network.IPAM{
			Driver: na.Network.IPAM.Driver.Name,
		},
		Options:        na.Network.DriverState.Options,
		CheckDuplicate: true,
	}

	for _, ic := range na.Network.IPAM.Configs {
		c := network.IPAMConfig{
			Subnet:  ic.Subnet,
			IPRange: ic.Range,
			Gateway: ic.Gateway,
		}
		options.IPAM.Config = append(options.IPAM.Config, c)
	}

	return e.backend.SetupIngress(clustertypes.NetworkCreateRequest{
		na.Network.ID,
		types.NetworkCreateRequest{
			Name:          na.Network.Spec.Annotations.Name,
			NetworkCreate: options,
		},
	}, na.Addresses[0])
}

// Controller returns a docker container runner.
func (e *executor) Controller(t *api.Task) (exec.Controller, error) {
	ctlr, err := newController(e.backend, t)
	if err != nil {
		return nil, err
	}

	return ctlr, nil
}

func (e *executor) SetNetworkBootstrapKeys(keys []*api.EncryptionKey) error {
	nwKeys := []*networktypes.EncryptionKey{}
	for _, key := range keys {
		nwKey := &networktypes.EncryptionKey{
			Subsystem:   key.Subsystem,
			Algorithm:   int32(key.Algorithm),
			Key:         make([]byte, len(key.Key)),
			LamportTime: key.LamportTime,
		}
		copy(nwKey.Key, key.Key)
		nwKeys = append(nwKeys, nwKey)
	}
	e.backend.SetNetworkBootstrapKeys(nwKeys)

	return nil
}
<commit_msg>Sort plugin names in node description<commit_after>package container

import (
	"sort"
	"strings"

	executorpkg "github.com/docker/docker/daemon/cluster/executor"
	clustertypes "github.com/docker/docker/daemon/cluster/provider"
	"github.com/docker/engine-api/types"
	"github.com/docker/engine-api/types/network"
	networktypes "github.com/docker/libnetwork/types"
	"github.com/docker/swarmkit/agent/exec"
	"github.com/docker/swarmkit/api"
	"golang.org/x/net/context"
)

type executor struct {
	backend executorpkg.Backend
}

// NewExecutor returns an executor from the docker client.
func NewExecutor(b executorpkg.Backend) exec.Executor {
	return &executor{
		backend: b,
	}
}

// Describe returns the underlying node description from the docker client.
func (e *executor) Describe(ctx context.Context) (*api.NodeDescription, error) {
	info, err := e.backend.SystemInfo()
	if err != nil {
		return nil, err
	}

	var plugins []api.PluginDescription
	addPlugins := func(typ string, names []string) {
		for _, name := range names {
			plugins = append(plugins, api.PluginDescription{
				Type: typ,
				Name: name,
			})
		}
	}

	addPlugins("Volume", info.Plugins.Volume)
	// Add builtin driver "overlay" (the only builtin multi-host driver) to
	// the plugin list by default.
	addPlugins("Network", append([]string{"overlay"}, info.Plugins.Network...))
	addPlugins("Authorization", info.Plugins.Authorization)

	sort.Sort(sortedPlugins(plugins))

	// parse []string labels into a map[string]string
	labels := map[string]string{}
	for _, l := range info.Labels {
		stringSlice := strings.SplitN(l, "=", 2)
		// this will take the last value in the list for a given key
		// ideally, one shouldn't assign multiple values to the same key
		if len(stringSlice) > 1 {
			labels[stringSlice[0]] = stringSlice[1]
		}
	}

	description := &api.NodeDescription{
		Hostname: info.Name,
		Platform: &api.Platform{
			Architecture: info.Architecture,
			OS:           info.OSType,
		},
		Engine: &api.EngineDescription{
			EngineVersion: info.ServerVersion,
			Labels:        labels,
			Plugins:       plugins,
		},
		Resources: &api.Resources{
			NanoCPUs:    int64(info.NCPU) * 1e9,
			MemoryBytes: info.MemTotal,
		},
	}

	return description, nil
}

func (e *executor) Configure(ctx context.Context, node *api.Node) error {
	na := node.Attachment
	if na == nil {
		return nil
	}

	options := types.NetworkCreate{
		Driver: na.Network.DriverState.Name,
		IPAM: network.IPAM{
			Driver: na.Network.IPAM.Driver.Name,
		},
		Options:        na.Network.DriverState.Options,
		CheckDuplicate: true,
	}

	for _, ic := range na.Network.IPAM.Configs {
		c := network.IPAMConfig{
			Subnet:  ic.Subnet,
			IPRange: ic.Range,
			Gateway: ic.Gateway,
		}
		options.IPAM.Config = append(options.IPAM.Config, c)
	}

	return e.backend.SetupIngress(clustertypes.NetworkCreateRequest{
		na.Network.ID,
		types.NetworkCreateRequest{
			Name:          na.Network.Spec.Annotations.Name,
			NetworkCreate: options,
		},
	}, na.Addresses[0])
}

// Controller returns a docker container runner.
func (e *executor) Controller(t *api.Task) (exec.Controller, error) {
	ctlr, err := newController(e.backend, t)
	if err != nil {
		return nil, err
	}

	return ctlr, nil
}

func (e *executor) SetNetworkBootstrapKeys(keys []*api.EncryptionKey) error {
	nwKeys := []*networktypes.EncryptionKey{}
	for _, key := range keys {
		nwKey := &networktypes.EncryptionKey{
			Subsystem:   key.Subsystem,
			Algorithm:   int32(key.Algorithm),
			Key:         make([]byte, len(key.Key)),
			LamportTime: key.LamportTime,
		}
		copy(nwKey.Key, key.Key)
		nwKeys = append(nwKeys, nwKey)
	}
	e.backend.SetNetworkBootstrapKeys(nwKeys)

	return nil
}

type sortedPlugins []api.PluginDescription

func (sp sortedPlugins) Len() int { return len(sp) }

func (sp sortedPlugins) Swap(i, j int) { sp[i], sp[j] = sp[j], sp[i] }

func (sp sortedPlugins) Less(i, j int) bool {
	if sp[i].Type != sp[j].Type {
		return sp[i].Type < sp[j].Type
	}
	return sp[i].Name < sp[j].Name
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/ethereum/eth-go"
	"github.com/ethereum/eth-go/ethchain"
	"github.com/ethereum/eth-go/ethminer"
	"github.com/ethereum/eth-go/ethpub"
	"github.com/ethereum/eth-go/ethrpc"
	"github.com/ethereum/eth-go/ethutil"
	"github.com/ethereum/go-ethereum/utils"
	"log"
	"os"
	"os/signal"
	"runtime"
	"strings"
)

const Debug = true

// Register interrupt handlers so we can stop the ethereum
func RegisterInterrupts(s *eth.Ethereum) {
	// Buffered chan of one is enough
	c := make(chan os.Signal, 1)
	// Notify about interrupts for now
	signal.Notify(c, os.Interrupt)
	go func() {
		for sig := range c {
			fmt.Printf("Shutting down (%v) ... \n", sig)
			s.Stop()
		}
	}()
}

func confirm(message string) bool {
	fmt.Println(message, "Are you sure? (y/n)")
	var r string
	fmt.Scanln(&r)
	for ; ; fmt.Scanln(&r) {
		if r == "n" || r == "y" {
			break
		} else {
			fmt.Printf("Yes or no?", r)
		}
	}
	return r == "y"
}

func main() {
	Init()

	runtime.GOMAXPROCS(runtime.NumCPU())

	// set logger
	var logSys *log.Logger
	flags := log.LstdFlags

	ethutil.ReadConfig(DataDir)
	logger := ethutil.Config.Log

	if LogFile != "" {
		logfile, err := os.OpenFile(LogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			panic(fmt.Sprintf("error opening log file '%s': %v", LogFile, err))
		}
		defer logfile.Close()
		log.SetOutput(logfile)
		logSys = log.New(logfile, "", flags)
		logger.AddLogSystem(logSys)
	}
	/*else {
		logSys = log.New(os.Stdout, "", flags)
	}*/

	ethchain.InitFees()

	// Instantiated a eth stack
	ethereum, err := eth.New(eth.CapDefault, UseUPnP)
	if err != nil {
		log.Println("eth start err:", err)
		return
	}
	ethereum.Port = OutboundPort

	// bookkeeping tasks
	switch {
	case GenAddr:
		if NonInteractive || confirm("This action overwrites your old private key.") {
			utils.CreateKeyPair(true)
		}
		os.Exit(0)
	case len(ImportKey) > 0:
		if NonInteractive || confirm("This action overwrites your old private key.") {
			mnemonic := strings.Split(ImportKey, " ")
			if len(mnemonic) == 24 {
				logSys.Println("Got mnemonic key, importing.")
				key := ethutil.MnemonicDecode(mnemonic)
				utils.ImportPrivateKey(key)
			} else if len(mnemonic) == 1 {
				logSys.Println("Got hex key, importing.")
				utils.ImportPrivateKey(ImportKey)
			} else {
				logSys.Println("Did not recognise format, exiting.")
			}
		}
		os.Exit(0)
	case ExportKey:
		key := ethutil.Config.Db.GetKeys()[0]
		logSys.Println(fmt.Sprintf("prvk: %x\n", key.PrivateKey))
		os.Exit(0)
	case ShowGenesis:
		logSys.Println(ethereum.BlockChain().Genesis())
		os.Exit(0)
	default:
		// Creates a keypair if non exists
		utils.CreateKeyPair(false)
	}

	// client
	logger.Infoln(fmt.Sprintf("Starting Ethereum v%s", ethutil.Config.Ver))

	// Set the max peers
	ethereum.MaxPeers = MaxPeer

	// Set Mining status
	ethereum.Mining = StartMining

	if StartMining {
		logger.Infoln("Miner started")

		// Fake block mining. It broadcasts a new block every 5 seconds
		go func() {

			if StartMining {
				logger.Infoln("Miner started")

				go func() {
					data, _ := ethutil.Config.Db.Get([]byte("KeyRing"))
					keyRing := ethutil.NewValueFromBytes(data)
					addr := keyRing.Get(1).Bytes()

					miner := ethminer.NewDefaultMiner(addr, ethereum)
					miner.Start()

				}()
			}
		}()

	}

	if StartConsole {
		err := os.Mkdir(ethutil.Config.ExecPath, os.ModePerm)
		// Error is OK if the error is ErrExist
		if err != nil && !os.IsExist(err) {
			log.Panic("Unable to create EXECPATH:", err)
		}

		console := NewConsole(ethereum)
		go console.Start()
	}
	if StartRpc {
		ethereum.RpcServer, err = ethrpc.NewJsonRpcServer(ethpub.NewPEthereum(ethereum), RpcPort)
		if err != nil {
			logger.Infoln("Could not start RPC interface:", err)
		} else {
			go ethereum.RpcServer.Start()
		}
	}

	RegisterInterrupts(ethereum)

	ethereum.Start(UseSeed)

	// Wait for shutdown
	ethereum.WaitForShutdown()
}
<commit_msg>Change coinbase to be the address not public key<commit_after>package main

import (
	"encoding/hex"
	"fmt"
	"github.com/ethereum/eth-go"
	"github.com/ethereum/eth-go/ethchain"
	"github.com/ethereum/eth-go/ethminer"
	"github.com/ethereum/eth-go/ethpub"
	"github.com/ethereum/eth-go/ethrpc"
	"github.com/ethereum/eth-go/ethutil"
	"github.com/ethereum/go-ethereum/utils"
	"log"
	"os"
	"os/signal"
	"runtime"
	"strings"
)

const Debug = true

// Register interrupt handlers so we can stop the ethereum
func RegisterInterrupts(s *eth.Ethereum) {
	// Buffered chan of one is enough
	c := make(chan os.Signal, 1)
	// Notify about interrupts for now
	signal.Notify(c, os.Interrupt)
	go func() {
		for sig := range c {
			fmt.Printf("Shutting down (%v) ... \n", sig)
			s.Stop()
		}
	}()
}

func confirm(message string) bool {
	fmt.Println(message, "Are you sure? (y/n)")
	var r string
	fmt.Scanln(&r)
	for ; ; fmt.Scanln(&r) {
		if r == "n" || r == "y" {
			break
		} else {
			fmt.Printf("Yes or no?", r)
		}
	}
	return r == "y"
}

func main() {
	Init()

	runtime.GOMAXPROCS(runtime.NumCPU())

	// set logger
	var logSys *log.Logger
	flags := log.LstdFlags

	ethutil.ReadConfig(DataDir)
	logger := ethutil.Config.Log

	if LogFile != "" {
		logfile, err := os.OpenFile(LogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			panic(fmt.Sprintf("error opening log file '%s': %v", LogFile, err))
		}
		defer logfile.Close()
		log.SetOutput(logfile)
		logSys = log.New(logfile, "", flags)
		logger.AddLogSystem(logSys)
	}
	/*else {
		logSys = log.New(os.Stdout, "", flags)
	}*/

	ethchain.InitFees()

	// Instantiated a eth stack
	ethereum, err := eth.New(eth.CapDefault, UseUPnP)
	if err != nil {
		log.Println("eth start err:", err)
		return
	}
	ethereum.Port = OutboundPort

	// bookkeeping tasks
	switch {
	case GenAddr:
		if NonInteractive || confirm("This action overwrites your old private key.") {
			utils.CreateKeyPair(true)
		}
		os.Exit(0)
	case len(ImportKey) > 0:
		if NonInteractive || confirm("This action overwrites your old private key.") {
			mnemonic := strings.Split(ImportKey, " ")
			if len(mnemonic) == 24 {
				logSys.Println("Got mnemonic key, importing.")
				key := ethutil.MnemonicDecode(mnemonic)
				utils.ImportPrivateKey(key)
			} else if len(mnemonic) == 1 {
				logSys.Println("Got hex key, importing.")
				utils.ImportPrivateKey(ImportKey)
			} else {
				logSys.Println("Did not recognise format, exiting.")
			}
		}
		os.Exit(0)
	case ExportKey:
		key := ethutil.Config.Db.GetKeys()[0]
		logSys.Println(fmt.Sprintf("prvk: %x\n", key.PrivateKey))
		os.Exit(0)
	case ShowGenesis:
		logSys.Println(ethereum.BlockChain().Genesis())
		os.Exit(0)
	default:
		// Creates a keypair if non exists
		utils.CreateKeyPair(false)
	}

	// client
	logger.Infoln(fmt.Sprintf("Starting Ethereum v%s", ethutil.Config.Ver))

	// Set the max peers
	ethereum.MaxPeers = MaxPeer

	// Set Mining status
	ethereum.Mining = StartMining

	if StartMining {
		logger.Infoln("Miner started")

		// Fake block mining. It broadcasts a new block every 5 seconds
		go func() {

			if StartMining {
				logger.Infoln("Miner started")

				go func() {
					data, _ := ethutil.Config.Db.Get([]byte("KeyRing"))
					keyRing := ethutil.NewValueFromBytes(data)
					addr := keyRing.Get(1).Bytes()

					pair, _ := ethchain.NewKeyPairFromSec(ethutil.FromHex(hex.EncodeToString(addr)))

					miner := ethminer.NewDefaultMiner(pair.Address(), ethereum)
					miner.Start()

				}()
			}
		}()

	}

	if StartConsole {
		err := os.Mkdir(ethutil.Config.ExecPath, os.ModePerm)
		// Error is OK if the error is ErrExist
		if err != nil && !os.IsExist(err) {
			log.Panic("Unable to create EXECPATH:", err)
		}

		console := NewConsole(ethereum)
		go console.Start()
	}
	if StartRpc {
		ethereum.RpcServer, err = ethrpc.NewJsonRpcServer(ethpub.NewPEthereum(ethereum), RpcPort)
		if err != nil {
			logger.Infoln("Could not start RPC interface:", err)
		} else {
			go ethereum.RpcServer.Start()
		}
	}

	RegisterInterrupts(ethereum)

	ethereum.Start(UseSeed)

	// Wait for shutdown
	ethereum.WaitForShutdown()
}
<|endoftext|>


<commit_before>package i18n

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/XenoPhex/jibber_jabber"

	resources "github.com/cloudfoundry/cli/cf/resources"
	go_i18n "github.com/nicksnyder/go-i18n/i18n"
)

const (
	DEFAULT_LOCAL = "en_US"
)

var SUPPORTED_LANGUAGES = []string{"ar", "ca", "zh", "cs", "da", "nl", "en", "fr", "de", "it", "ja", "lt", "pt", "es"}
var resources_path = filepath.Join("cf", "i18n", "resources")

func GetResourcesPath() string {
	return resources_path
}

func Init(packageName string, i18nDirname string) go_i18n.TranslateFunc {
	userLocale, err := jibber_jabber.DetectIETF()
	if err != nil {
		println("Could not load desired locale:", userLocale, "falling back to default locale", DEFAULT_LOCAL)
		userLocale = DEFAULT_LOCAL
	}

	// convert IETF format to XCU format
	userLocale = strings.Replace(userLocale, "-", "_", 1)

	err = loadTranslationAssets(packageName, i18nDirname, userLocale, DEFAULT_LOCAL)
	if err != nil { // this should only be from the user locale
		println("Could not load desired locale:", userLocale, "falling back to default locale", DEFAULT_LOCAL)
	}

	T, err := go_i18n.Tfunc(userLocale, DEFAULT_LOCAL)
	if err != nil {
		panic(err)
	}

	return T
}

func splitLocale(locale string) (string, string) {
	formattedLocale := strings.Split(locale, ".")[0]
	formattedLocale = strings.Replace(formattedLocale, "-", "_", -1)
	language := strings.Split(formattedLocale, "_")[0]
	territory := strings.Split(formattedLocale, "_")[1]
	return language, territory
}

func loadTranslationAssets(packageName, assetPath, userLocale, defaultLocale string) error {
	userLocaleErr := loadFromAsset(packageName, assetPath, userLocale)

	defaultLocaleErr := loadFromAsset(packageName, assetPath, defaultLocale)
	if defaultLocaleErr != nil {
		panic(defaultLocale)
	}

	return userLocaleErr
}

func loadFromAsset(packageName, assetPath, locale string) error {
	language, _ := splitLocale(locale)
	assetName := locale + ".all.json"
	assetKey := filepath.Join(assetPath, language, packageName, assetName)

	byteArray, err := resources.Asset(assetKey)
	if err != nil {
		return err
	}

	if len(byteArray) == 0 {
		return errors.New(fmt.Sprintf("Could not load i18n asset: %v", assetKey))
	}

	tmpDir, err := ioutil.TempDir("", "cloudfoundry_cli_i18n_res")
	if err != nil {
		return err
	}
	defer func() {
		os.RemoveAll(tmpDir)
	}()

	fileName, err := saveLanguageFileToDisk(tmpDir, assetName, byteArray)
	if err != nil {
		return err
	}

	go_i18n.MustLoadTranslationFile(fileName)

	os.RemoveAll(fileName)

	return nil
}

func saveLanguageFileToDisk(tmpDir, assetName string, byteArray []byte) (fileName string, err error) {
	fileName = filepath.Join(tmpDir, assetName)
	file, err := os.Create(fileName)
	if err != nil {
		return
	}
	defer file.Close()

	_, err = file.Write(byteArray)
	if err != nil {
		return
	}

	return
}
<commit_msg>Simplify loading assets, now that we know how goi18n works<commit_after>package i18n

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/XenoPhex/jibber_jabber"

	resources "github.com/cloudfoundry/cli/cf/resources"
	go_i18n "github.com/nicksnyder/go-i18n/i18n"
)

const (
	DEFAULT_LOCAL = "en_US"
)

var SUPPORTED_LANGUAGES = []string{"ar", "ca", "zh", "cs", "da", "nl", "en", "fr", "de", "it", "ja", "lt", "pt", "es"}
var resources_path = filepath.Join("cf", "i18n", "resources")

func GetResourcesPath() string {
	return resources_path
}

func Init(packageName string, i18nDirname string) go_i18n.TranslateFunc {
	userLocale, err := jibber_jabber.DetectIETF()
	if err != nil {
		println("Could not load desired locale:", userLocale, "falling back to default locale", DEFAULT_LOCAL)
		userLocale = DEFAULT_LOCAL
	}

	// convert IETF format to XCU format
	userLocale = strings.Replace(userLocale, "-", "_", 1)

	err = loadFromAsset(packageName, i18nDirname, userLocale)
	if err != nil { // this should only be from the user locale
		println("Could not load desired locale:", userLocale, "falling back to default locale", DEFAULT_LOCAL)
	}

	T, err := go_i18n.Tfunc(userLocale, DEFAULT_LOCAL)
	if err != nil {
		panic(err)
	}

	return T
}

func splitLocale(locale string) (string, string) {
	formattedLocale := strings.Split(locale, ".")[0]
	formattedLocale = strings.Replace(formattedLocale, "-", "_", -1)
	language := strings.Split(formattedLocale, "_")[0]
	territory := strings.Split(formattedLocale, "_")[1]
	return language, territory
}

func loadFromAsset(packageName, assetPath, locale string) error {
	language, _ := splitLocale(locale)
	assetName := locale + ".all.json"
	assetKey := filepath.Join(assetPath, language, packageName, assetName)

	byteArray, err := resources.Asset(assetKey)
	if err != nil {
		return err
	}

	if len(byteArray) == 0 {
		return errors.New(fmt.Sprintf("Could not load i18n asset: %v", assetKey))
	}

	tmpDir, err := ioutil.TempDir("", "cloudfoundry_cli_i18n_res")
	if err != nil {
		return err
	}
	defer func() {
		os.RemoveAll(tmpDir)
	}()

	fileName, err := saveLanguageFileToDisk(tmpDir, assetName, byteArray)
	if err != nil {
		return err
	}

	go_i18n.MustLoadTranslationFile(fileName)

	os.RemoveAll(fileName)

	return nil
}

func saveLanguageFileToDisk(tmpDir, assetName string, byteArray []byte) (fileName string, err error) {
	fileName = filepath.Join(tmpDir, assetName)
	file, err := os.Create(fileName)
	if err != nil {
		return
	}
	defer file.Close()

	_, err = file.Write(byteArray)
	if err != nil {
		return
	}

	return
}
<|endoftext|>


<commit_before>package apt

import (
	"fmt"
	"github.com/vektra/tachyon"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"time"
)

type Apt struct {
	Pkg       string `tachyon:"pkg"`
	State     string `tachyon:"state" enum:"present,install,absent,remove"`
	Cache     bool   `tachyon:"update_cache"`
	CacheTime string `tachyon:"cache_time"`
	Dry       bool   `tachyon:"dryrun"`
}

var installed = regexp.MustCompile(`Installed: ([^\n]+)`)
var candidate = regexp.MustCompile(`Candidate: ([^\n]+)`)

func (a *Apt) Run(env *tachyon.CommandEnv) (*tachyon.Result, error) {
	state := a.State
	if state == "" {
		state = "present"
	}

	if a.Cache {
		home, err := tachyon.HomeDir()
		if err != nil {
			return nil, err
		}

		checkFile := filepath.Join(home, ".tachyon", "apt-cache-timestamp")

		runUpdate := true

		if a.CacheTime != "" {
			fi, err := os.Stat(checkFile)
			if err == nil {
				dur, err := time.ParseDuration(a.CacheTime)
				if err != nil {
					return nil, fmt.Errorf("cache_time was not in the proper format")
				}

				runUpdate = time.Now().After(fi.ModTime().Add(dur))
			}
		}

		if runUpdate {
			_, err := tachyon.RunCommand(env, "apt-get", "update")
			if err != nil {
				return nil, err
			}
			ioutil.WriteFile(checkFile, []byte(``), 0666)
		}
	}

	if a.Pkg == "" {
		simp := tachyon.NewResult(true)
		simp.Add("cache", "updated")

		return simp, nil
	}

	out, err := tachyon.RunCommand(env, "apt-cache", "policy", a.Pkg)
	if err != nil {
		return nil, err
	}

	res := installed.FindSubmatch(out.Stdout)
	if res == nil {
		return nil, fmt.Errorf("No package '%s' available")
	}

	curVer := string(res[1])
	if curVer == "(none)" {
		curVer = ""
	}

	res = candidate.FindSubmatch(out.Stdout)
	if res == nil {
		return nil, fmt.Errorf("Error parsing apt-cache output")
	}

	canVer := string(res[1])

	if state == "absent" {
		rd := tachyon.ResultData{}

		if curVer == "" {
			return tachyon.WrapResult(false, rd), nil
		}

		rd.Set("removed", curVer)

		_, err = tachyon.RunCommand(env, "apt-get", "remove", "-y", a.Pkg)

		if err != nil {
			return nil, err
		}

		return tachyon.WrapResult(true, rd), nil
	}

	rd := tachyon.ResultData{}
	rd.Set("installed", curVer)
	rd.Set("candidate", canVer)

	if state == "present" && curVer == canVer {
		return tachyon.WrapResult(false, rd), nil
	}

	if a.Dry {
		rd.Set("dryrun", true)
		return tachyon.WrapResult(true, rd), nil
	}

	e := append(os.Environ(), "DEBIAN_FRONTEND=noninteractive", "DEBIAN_PRIORITY=critical")

	_, err = tachyon.RunCommandInEnv(env, e, "apt-get", "install", "-y", a.Pkg)
	if err != nil {
		return nil, err
	}

	rd.Set("installed", canVer)

	return tachyon.WrapResult(true, rd), nil
}

func init() {
	tachyon.RegisterCommand("apt", &Apt{})
}
<commit_msg>Report the package name<commit_after>package apt

import (
	"fmt"
	"github.com/vektra/tachyon"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"time"
)

type Apt struct {
	Pkg       string `tachyon:"pkg"`
	State     string `tachyon:"state" enum:"present,install,absent,remove"`
	Cache     bool   `tachyon:"update_cache"`
	CacheTime string `tachyon:"cache_time"`
	Dry       bool   `tachyon:"dryrun"`
}

var installed = regexp.MustCompile(`Installed: ([^\n]+)`)
var candidate = regexp.MustCompile(`Candidate: ([^\n]+)`)

func (a *Apt) Run(env *tachyon.CommandEnv) (*tachyon.Result, error) {
	state := a.State
	if state == "" {
		state = "present"
	}

	if a.Cache {
		home, err := tachyon.HomeDir()
		if err != nil {
			return nil, err
		}

		checkFile := filepath.Join(home, ".tachyon", "apt-cache-timestamp")

		runUpdate := true

		if a.CacheTime != "" {
			fi, err := os.Stat(checkFile)
			if err == nil {
				dur, err := time.ParseDuration(a.CacheTime)
				if err != nil {
					return nil, fmt.Errorf("cache_time was not in the proper format")
				}

				runUpdate = time.Now().After(fi.ModTime().Add(dur))
			}
		}

		if runUpdate {
			_, err := tachyon.RunCommand(env, "apt-get", "update")
			if err != nil {
				return nil, err
			}
			ioutil.WriteFile(checkFile, []byte(``), 0666)
		}
	}

	if a.Pkg == "" {
		simp := tachyon.NewResult(true)
		simp.Add("cache", "updated")

		return simp, nil
	}

	out, err := tachyon.RunCommand(env, "apt-cache", "policy", a.Pkg)
	if err != nil {
		return nil, err
	}

	res := installed.FindSubmatch(out.Stdout)
	if res == nil {
		return nil, fmt.Errorf("No package '%s' available", a.Pkg)
	}

	curVer := string(res[1])
	if curVer == "(none)" {
		curVer = ""
	}

	res = candidate.FindSubmatch(out.Stdout)
	if res == nil {
		return nil, fmt.Errorf("Error parsing apt-cache output")
	}

	canVer := string(res[1])

	if state == "absent" {
		rd := tachyon.ResultData{}

		if curVer == "" {
			return tachyon.WrapResult(false, rd), nil
		}

		rd.Set("removed", curVer)

		_, err = tachyon.RunCommand(env, "apt-get", "remove", "-y", a.Pkg)

		if err != nil {
			return nil, err
		}

		return tachyon.WrapResult(true, rd), nil
	}

	rd := tachyon.ResultData{}
	rd.Set("installed", curVer)
	rd.Set("candidate", canVer)

	if state == "present" && curVer == canVer {
		return tachyon.WrapResult(false, rd), nil
	}

	if a.Dry {
		rd.Set("dryrun", true)
		return tachyon.WrapResult(true, rd), nil
	}

	e := append(os.Environ(), "DEBIAN_FRONTEND=noninteractive", "DEBIAN_PRIORITY=critical")

	_, err = tachyon.RunCommandInEnv(env, e, "apt-get", "install", "-y", a.Pkg)
	if err != nil {
		return nil, err
	}

	rd.Set("installed", canVer)

	return tachyon.WrapResult(true, rd), nil
}

func init() {
	tachyon.RegisterCommand("apt", &Apt{})
}
<|endoftext|>


<commit_before>package classify

import (
	"math/rand"

	"github.com/deathly809/gomath"
	"github.com/deathly809/gotypes"
)

type smo struct {
	s      *svm
	errors []float32

	i1, i2 int

	// Parameters
	tolerance float32
	eps       float32
	C         float32

	// State
	E                    []float32
	threshold            float32
	numNonBoundaryAlphas int

	// Current build up
	a1, a2 float32
	v1, v2 []gotypes.Value
	E1, E2 float32
	y1, y2 float32

	// Misc
	wq           *WorkQueue
	workFunction func(int)
}

type svm struct {
	alpha          []float32
	supportVectors []Data
	kernel         Kernel
}

func (s *svm) Classify(data []gotypes.Value) float32 {
	result := float32(0.0)
	for i, sV := range s.supportVectors {
		alpha := s.alpha[i]
		if s.kernel == nil {
			panic("kernel is nil")
		}
		dot := s.kernel.Dot(sV.Value(), data)
		result += alpha * dot
	}
	return result
}

func (alg *smo) takeStep() bool {
	result := false
	if alg.i1 != alg.i2 {

		fZero := float32(0)

		alpha1 := alg.s.alpha[alg.i1]
		alpha2 := alg.a2

		alg.a1 = alpha1

		alg.v1 = alg.s.supportVectors[alg.i1].Value()
		alg.y1 = alg.s.supportVectors[alg.i1].Class()

		alg.E1 = alg.s.Classify(alg.v1) - alg.y1

		var L, H float32

		if alg.y1 == alg.y2 {
			sum := alpha2 + alpha1
			L = gomath.MaxFloat32(fZero, sum-alg.C)
			H = gomath.MinFloat32(alg.C, sum)
		} else {
			diff := alpha2 - alpha1
			L = gomath.MaxFloat32(fZero, diff)
			H = gomath.MinFloat32(alg.C, alg.C+diff)
		}

		if L != H {
			k11 := alg.s.kernel.Dot(alg.v1, alg.v1)
			k12 := alg.s.kernel.Dot(alg.v1, alg.v2)
			k22 := alg.s.kernel.Dot(alg.v2, alg.v2)

			eta := 2*k12 - k11 - k22

			if eta < 0 {
				alg.a2 = gomath.ClampFloat32(L, H, alpha1-alg.y2*(alg.E1-alg.E2)/eta)
			} else {
				Lobj := float32(5.0)
				Hobj := float32(5.0)
				if Lobj > Hobj+1E-3 {
					alg.a2 = L
				} else if Lobj < Hobj-1E-3 {
					alg.a2 = L
				} else {
					alg.a2 = alpha2
				}

				if alg.a2 < 1E-8 {
					alg.a2 = 0
				} else if alg.a2 > alg.C-1e-8 {
					alg.a2 = alg.C
				}

				if gomath.AbsFloat32(alg.a2-alpha2) >= alg.eps*(alg.a2+alpha2+alg.eps) {
					alg.s.alpha[alg.i1] = alg.a1
					alg.s.alpha[alg.i2] = alg.a2

					result = true
				}
			}
		}
	}
	return result
}

// Two heuristics:
func (alg *smo) examineExample() int {

	data := alg.s.supportVectors[alg.i1]
	alg.a2 = alg.s.alpha[alg.i2]
	alg.E2 = alg.errors[alg.i2]
	r2 := alg.E2 * data.Class()

	if (r2 < -alg.tolerance && alg.a2 < alg.C) || (r2 > alg.tolerance && alg.a2 > 0) {

		if alg.numNonBoundaryAlphas > 1 {

			// Second Choice Heuristic by biggest |E1 - E2|
			alg.i1 = 0
			prev := gomath.AbsFloat32(alg.errors[alg.i1] - alg.E2)

			for i := 1; i < len(alg.s.supportVectors); i++ {
				test := gomath.AbsFloat32(alg.errors[i] - alg.E2)
				if test > prev {
					prev = test
					alg.i1 = i
				}
			}

			alg.E1 = prev
			if alg.takeStep() {
				return 1
			}
		}

		for i, a := range alg.s.alpha {
			if a > 0 && a < alg.C {
				alg.i1 = i
				alg.E1 = alg.errors[i]
				if alg.takeStep() {
					return 1
				}
			}
		}

		randomStart := rand.Intn(len(alg.s.alpha))
		for i := 0; i < len(alg.s.alpha); i++ {
			alg.i1 = randomStart
			alg.E1 = alg.errors[randomStart]
			if alg.takeStep() {
				return 1
			}
			randomStart = (randomStart + 1) % len(alg.s.alpha)
		}
	}

	return 0
}

func (alg *smo) computeError() {
	alg.wq.Enqueue(0, len(alg.s.alpha)-1, alg.workFunction)
}

/*
   Fast Training of Support Vector Machines
   using Sequential Minimal Optimization

   max: W(alpha) = Sum_{i=0}^l alpha_i - \frac{1}{2} Sum_{i=0}^l Sum_{j=0}^l y_{i}y_{j}K(x_{i},x_{j})alpha_{i}alpha_{j}

   s.t.

   0 \lte alpha_i \lte C,
   Sum_{i=0}^l y_{i}*alpha_{i}

*/
func solve(data []Data, k Kernel, tol, C float32) (result *svm) {

	result = &svm{
		alpha:          make([]float32, len(data)),
		supportVectors: data,
		kernel:         k,
	}

	alg := &smo{
		errors:               make([]float32, len(data)),
		C:                    C,
		numNonBoundaryAlphas: 0,
		tolerance:            tol,
		threshold:            float32(0.0),
		E:                    make([]float32, len(data)),
		s:                    result,
	}

	// Initialize work queue
	alg.wq = &WorkQueue{}
	alg.wq.Init(10)
	alg.wq.Start()
	defer alg.wq.Stop()

	alg.workFunction = func(i int) {
		v := result.supportVectors[i]
		alg.E[i] = result.Classify(v.Value()) - v.Class()
	}

	alg.computeError()

	// Begin algorithm
	numChanged := 0
	examineAll := true
	for (numChanged > 0) || examineAll {

		numChanged = 0

		if examineAll {
			for i := range data {
				alg.i2 = i
				numChanged += alg.examineExample()
			}
			examineAll = !examineAll
		} else {
			for i := range data {
				a := result.alpha[i]
				if a > 0 && a < alg.threshold {
					alg.i2 = i
					numChanged += alg.examineExample()
				}
			}
		}

		if numChanged == 0 {
			examineAll = true
		} else {
			alg.computeError()
		}
	}

	// Compress (not in algorithm)
	pos := 0
	for i := range data {
		a := result.alpha[i]
		if a > 0 {
			result.alpha[pos] = a
			result.supportVectors[pos] = data[i]
			pos++
		}
	}

	// Make sure the garbage collector can release array
	result.alpha = append(([]float32)(nil), result.alpha[:pos]...)
	result.supportVectors = append(([]Data)(nil), result.supportVectors[:pos]...)

	return result
}

// NewSVMClassifer will create a new SVM classifier with the data given
func NewSVMClassifer(data []Data, k Kernel, alpha float32, C float32) Classifier {
	return solve(data, k, alpha, C)
}
<commit_msg>I don't remember<commit_after>package classify

import (
	"math/rand"

	"github.com/deathly809/gomath"
	"github.com/deathly809/gotypes"
)

const (
	_Epsilon = 1.0E-5
)

type smo struct {
	s      *svm
	errors []float32

	i1, i2 int

	// Parameters
	tolerance float32
	eps       float32
	C         float32

	// State
	E                    []float32
	threshold            float32
	numNonBoundaryAlphas int

	// Current build up
	a1, a2 float32
	v1, v2 []gotypes.Value
	E1, E2 float32
	y1, y2 float32

	// Misc
	wq           *WorkQueue
	workFunction func(int)
}

type svm struct {
	epsilon        float32
	alpha          []float32
	supportVectors []Data
	kernel         Kernel
}

/*
 *	KKT Conditions:
 *
 *		lagrangians are non-negative
 *			a_i >= 0
 *
 *		answers are sufficient distance from hyperplane
 *			y_i * h(x_i) - 1 >= 0
 *
 *		support vectors are on hyperplane
 *			a_i * (y_i * h(x_i) - 1) = 0
 */
func (alg *smo) violatesKKT(i int) bool {
	if alg.s.alpha[i] < _Epsilon {
		return true
	}

	data := alg.s.supportVectors[i].Value()
	y := alg.s.supportVectors[i].Class()

	if y*alg.s.Classify(data)-1 < _Epsilon {
		return true
	}

	return false
}

func (s *svm) Classify(data []gotypes.Value) float32 {
	result := float32(0.0)
	for i, sV := range s.supportVectors {
		alpha := s.alpha[i]
		if s.kernel == nil {
			panic("kernel is nil")
		}
		dot := s.kernel.Dot(sV.Value(), data)
		result += alpha * dot
	}
	return result
}

func (alg *smo) takeStep() bool {
	result := false
	if alg.i1 != alg.i2 {

		fZero := float32(0)

		alpha1 := alg.s.alpha[alg.i1]
		alpha2 := alg.a2

		alg.a1 = alpha1

		alg.v1 = alg.s.supportVectors[alg.i1].Value()
		alg.y1 = alg.s.supportVectors[alg.i1].Class()

		alg.E1 = alg.s.Classify(alg.v1) - alg.y1

		var L, H float32

		if alg.y1 == alg.y2 {
			sum := alpha2 + alpha1
			L = gomath.MaxFloat32(fZero, sum-alg.C)
			H = gomath.MinFloat32(alg.C, sum)
		} else {
			diff := alpha2 - alpha1
			L = gomath.MaxFloat32(fZero, diff)
			H = gomath.MinFloat32(alg.C, alg.C+diff)
		}

		if L != H {
			k11 := alg.s.kernel.Dot(alg.v1, alg.v1)
			k12 := alg.s.kernel.Dot(alg.v1, alg.v2)
			k22 := alg.s.kernel.Dot(alg.v2, alg.v2)

			eta := 2*k12 - k11 - k22

			if eta < 0 {
				alg.a2 = gomath.ClampFloat32(L, H, alpha1-alg.y2*(alg.E1-alg.E2)/eta)
			} else {
				Lobj := float32(5.0)
				Hobj := float32(5.0)
				if Lobj > Hobj+1E-3 {
					alg.a2 = L
				} else if Lobj < Hobj-1E-3 {
					alg.a2 = L
				} else {
					alg.a2 = alpha2
				}

				if alg.a2 < 1E-8 {
					alg.a2 = 0
				} else if alg.a2 > alg.C-1e-8 {
					alg.a2 = alg.C
				}

				if gomath.AbsFloat32(alg.a2-alpha2) >= alg.eps*(alg.a2+alpha2+alg.eps) {
					alg.s.alpha[alg.i1] = alg.a1
					alg.s.alpha[alg.i2] = alg.a2

					result = true
				}
			}
		}
	}
	return result
}

// Two heuristics:
func (alg *smo) examineExample() int {

	data := alg.s.supportVectors[alg.i1]
	alg.a2 = alg.s.alpha[alg.i2]
	alg.E2 = alg.errors[alg.i2]
	r2 := alg.E2 * data.Class()

	if (r2 < -alg.tolerance && alg.a2 < alg.C) || (r2 > alg.tolerance && alg.a2 > 0) {

		if alg.numNonBoundaryAlphas > 1 {

			// Second Choice Heuristic by biggest |E1 - E2|
			alg.i1 = 0
			prev := gomath.AbsFloat32(alg.errors[alg.i1] - alg.E2)

			for i := 1; i < len(alg.s.supportVectors); i++ {
				test := gomath.AbsFloat32(alg.errors[i] - alg.E2)
				if test > prev {
					prev = test
					alg.i1 = i
				}
			}

			alg.E1 = prev
			if alg.takeStep() {
				return 1
			}
		}

		for i, a := range alg.s.alpha {
			if a > 0 && a < alg.C {
				alg.i1 = i
				alg.E1 = alg.errors[i]
				if alg.takeStep() {
					return 1
				}
			}
		}

		randomStart := rand.Intn(len(alg.s.alpha))
		for i := 0; i < len(alg.s.alpha); i++ {
			alg.i1 = randomStart
			alg.E1 = alg.errors[randomStart]
			if alg.takeStep() {
				return 1
			}
			randomStart = (randomStart + 1) % len(alg.s.alpha)
		}
	}

	return 0
}

func (alg *smo) computeError() {
	alg.wq.Enqueue(0, len(alg.s.alpha)-1, alg.workFunction)
}

/*
   Fast Training of Support Vector Machines
   using Sequential Minimal Optimization

   max: W(alpha) = Sum_{i=0}^l alpha_i - \frac{1}{2} Sum_{i=0}^l Sum_{j=0}^l y_{i}y_{j}K(x_{i},x_{j})alpha_{i}alpha_{j}

   s.t.

   0 \lte alpha_i \lte C,
   Sum_{i=0}^l y_{i}*alpha_{i}

*/
func solve(data []Data, k Kernel, tol, C float32) (result *svm) {

	result = &svm{
		alpha:          make([]float32, len(data)),
		supportVectors: data,
		kernel:         k,
	}

	alg := &smo{
		errors:               make([]float32, len(data)),
		C:                    C,
		numNonBoundaryAlphas: 0,
		tolerance:            tol,
		threshold:            float32(0.0),
		E:                    make([]float32, len(data)),
		s:                    result,
	}

	// Initialize work queue
	alg.wq = &WorkQueue{}
	alg.wq.Init(10)
	alg.wq.Start()
	defer alg.wq.Stop()

	alg.workFunction = func(i int) {
		v := result.supportVectors[i]
		alg.E[i] = result.Classify(v.Value()) - v.Class()
	}

	alg.computeError()

	// Begin algorithm
	numChanged := 0
	examineAll := true
	for (numChanged > 0) || examineAll {

		numChanged = 0

		if examineAll {
			for i := range data {
				alg.i2 = i
				numChanged += alg.examineExample()
			}
			examineAll = !examineAll
		} else {
			for i := range data {
				a := result.alpha[i]
				if a > 0 && a < alg.threshold {
					alg.i2 = i
					numChanged += alg.examineExample()
				}
			}
		}

		if examineAll {
			examineAll = true
		} else if numChanged == 0 {
			examineAll = true
		}
	}

	// Compress (not in algorithm)
	pos := 0
	for i := range data {
		a := result.alpha[i]
		if a > 0 {
			result.alpha[pos] = a
			result.supportVectors[pos] = data[i]
			pos++
		}
	}

	// Make sure the garbage collector can release array
	result.alpha = append(([]float32)(nil), result.alpha[:pos]...)
	result.supportVectors = append(([]Data)(nil), result.supportVectors[:pos]...)

	return result
}

// NewSVMClassifer will create a new SVM classifier with the data given
func NewSVMClassifer(data []Data, k Kernel, alpha float32, C float32) Classifier {
	return solve(data, k, alpha, C)
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package msgpack

import (
	"errors"
	"sync"
	"time"

	"github.com/m3db/m3cluster/services"
	"github.com/m3db/m3metrics/protocol/msgpack"
	"github.com/m3db/m3x/log"

	"github.com/uber-go/tally"
)

const (
	defaultQueueSizeNumBuckets = 10
)

var (
	errInstanceQueueClosed = errors.New("instance queue is closed")
	errWriterQueueFull     = errors.New("writer queue is full")
)

// instanceQueue processes write requests for given instance.
type instanceQueue interface {
	// Enqueue enqueues a data buffer.
	Enqueue(buf msgpack.Buffer) error

	// Close closes the queue.
	Close() error
}

type writeFn func([]byte) error

type queue struct {
	sync.RWMutex

	log      xlog.Logger
	metrics  queueMetrics
	instance services.PlacementInstance
	conn     *connection
	bufCh    chan msgpack.Buffer
	doneCh   chan struct{}
	closed   bool

	writeFn writeFn
}

func newInstanceQueue(instance services.PlacementInstance, opts ServerOptions) instanceQueue {
	var (
		conn      = newConnection(instance.Endpoint(), opts.ConnectionOptions())
		iOpts     = opts.InstrumentOptions()
		queueSize = opts.InstanceQueueSize()
	)
	q := &queue{
		log:      iOpts.Logger(),
		metrics:  newQueueMetrics(iOpts.MetricsScope(), instance.ID(), queueSize),
		instance: instance,
		conn:     conn,
		bufCh:    make(chan msgpack.Buffer, queueSize),
		doneCh:   make(chan struct{}),
	}
	q.writeFn = q.conn.Write

	go q.drain()
	go q.reportQueueSize(iOpts.ReportInterval())
	return q
}

func (q *queue) Enqueue(buf msgpack.Buffer) error {
	q.RLock()
	if q.closed {
		q.RUnlock()
		q.metrics.queueClosedErrors.Inc(1)
		return errInstanceQueueClosed
	}
	// NB(xichen): the buffer already batches multiple metric buf points
	// to maximize per packet utilization so there should be no need to perform
	// additional batching here.
	select {
	case q.bufCh <- buf:
	default:
		q.RUnlock()

		// Close the buffer so it's resources are freed.
		buf.Close()

		q.metrics.queueFullErrors.Inc(1)
		return errWriterQueueFull
	}
	q.RUnlock()
	return nil
}

func (q *queue) Close() error {
	q.Lock()
	defer q.Unlock()

	if q.closed {
		return errInstanceQueueClosed
	}
	q.closed = true
	close(q.doneCh)
	close(q.bufCh)
	return nil
}

func (q *queue) drain() {
	for buf := range q.bufCh {
		if err := q.writeFn(buf.Bytes()); err != nil {
			q.log.WithFields(
				xlog.NewLogField("instance", q.instance.Endpoint()),
				xlog.NewLogErrField(err),
			).Error("write data error")
			q.metrics.connWriteErrors.Inc(1)
		}
		buf.Close()
	}
	q.conn.Close()
}

func (q *queue) reportQueueSize(reportInterval time.Duration) {
	ticker := time.NewTicker(reportInterval)
	defer ticker.Stop()

	for {
		select {
		case <-ticker.C:
			q.metrics.queueSize.RecordValue(float64(len(q.bufCh)))
		case <-q.doneCh:
			return
		}
	}
}

type queueMetrics struct {
	queueSize         tally.Histogram
	queueClosedErrors tally.Counter
	queueFullErrors   tally.Counter
	connWriteErrors   tally.Counter
}

func newQueueMetrics(s tally.Scope, instanceID string, queueSize int) queueMetrics {
	numBuckets := defaultQueueSizeNumBuckets
	if queueSize < numBuckets {
		numBuckets = queueSize
	}
	buckets := tally.MustMakeLinearValueBuckets(0, float64(queueSize/numBuckets), numBuckets)
	return queueMetrics{
		queueSize: s.Tagged(
			map[string]string{"instance-id": instanceID},
		).Histogram("queue-size", buckets),
		queueClosedErrors: s.Tagged(
			map[string]string{"error-type": "queue-closed", "action": "enqueue"},
		).Counter("errors"),
		queueFullErrors: s.Tagged(
			map[string]string{"error-type": "queue-full", "action": "enqueue"},
		).Counter("errors"),
		connWriteErrors: s.Tagged(
			map[string]string{"error-type": "conn-write", "action": "drain"},
		).Counter("errors"),
	}
}
<commit_msg>[queue] Don't include instance in queue size metrics (#29)<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package msgpack

import (
	"errors"
	"sync"
	"time"

	"github.com/m3db/m3cluster/services"
	"github.com/m3db/m3metrics/protocol/msgpack"
	"github.com/m3db/m3x/log"

	"github.com/uber-go/tally"
)

const (
	defaultQueueSizeNumBuckets = 6
)

var (
	errInstanceQueueClosed = errors.New("instance queue is closed")
	errWriterQueueFull     = errors.New("writer queue is full")
)

// instanceQueue processes write requests for given instance.
type instanceQueue interface {
	// Enqueue enqueues a data buffer.
	Enqueue(buf msgpack.Buffer) error

	// Close closes the queue.
	Close() error
}

type writeFn func([]byte) error

type queue struct {
	sync.RWMutex

	log      xlog.Logger
	metrics  queueMetrics
	instance services.PlacementInstance
	conn     *connection
	bufCh    chan msgpack.Buffer
	doneCh   chan struct{}
	closed   bool

	writeFn writeFn
}

func newInstanceQueue(instance services.PlacementInstance, opts ServerOptions) instanceQueue {
	var (
		conn      = newConnection(instance.Endpoint(), opts.ConnectionOptions())
		iOpts     = opts.InstrumentOptions()
		queueSize = opts.InstanceQueueSize()
	)
	q := &queue{
		log:      iOpts.Logger(),
		metrics:  newQueueMetrics(iOpts.MetricsScope(), queueSize),
		instance: instance,
		conn:     conn,
		bufCh:    make(chan msgpack.Buffer, queueSize),
		doneCh:   make(chan struct{}),
	}
	q.writeFn = q.conn.Write

	go q.drain()
	go q.reportQueueSize(iOpts.ReportInterval())
	return q
}

func (q *queue) Enqueue(buf msgpack.Buffer) error {
	q.RLock()
	if q.closed {
		q.RUnlock()
		q.metrics.queueClosedErrors.Inc(1)
		return errInstanceQueueClosed
	}
	// NB(xichen): the buffer already batches multiple metric buf points
	// to maximize per packet utilization so there should be no need to perform
	// additional batching here.
	select {
	case q.bufCh <- buf:
	default:
		q.RUnlock()

		// Close the buffer so it's resources are freed.
		buf.Close()

		q.metrics.queueFullErrors.Inc(1)
		return errWriterQueueFull
	}
	q.RUnlock()
	return nil
}

func (q *queue) Close() error {
	q.Lock()
	defer q.Unlock()

	if q.closed {
		return errInstanceQueueClosed
	}
	q.closed = true
	close(q.doneCh)
	close(q.bufCh)
	return nil
}

func (q *queue) drain() {
	for buf := range q.bufCh {
		if err := q.writeFn(buf.Bytes()); err != nil {
			q.log.WithFields(
				xlog.NewLogField("instance", q.instance.Endpoint()),
				xlog.NewLogErrField(err),
			).Error("write data error")
			q.metrics.connWriteErrors.Inc(1)
		}
		buf.Close()
	}
	q.conn.Close()
}

func (q *queue) reportQueueSize(reportInterval time.Duration) {
	ticker := time.NewTicker(reportInterval)
	defer ticker.Stop()

	for {
		select {
		case <-ticker.C:
			q.metrics.queueLen.RecordValue(float64(len(q.bufCh)))
		case <-q.doneCh:
			return
		}
	}
}

type queueMetrics struct {
	queueLen          tally.Histogram
	queueClosedErrors tally.Counter
	queueFullErrors   tally.Counter
	connWriteErrors   tally.Counter
}

func newQueueMetrics(s tally.Scope, queueSize int) queueMetrics {
	numBuckets := defaultQueueSizeNumBuckets
	if queueSize < numBuckets {
		numBuckets = queueSize
	}
	buckets := tally.MustMakeLinearValueBuckets(0, float64(queueSize/numBuckets), numBuckets)
	return queueMetrics{
		queueLen: s.Histogram("queue-length", buckets),
		queueClosedErrors: s.Tagged(
			map[string]string{"error-type": "queue-closed", "action": "enqueue"},
		).Counter("errors"),
		queueFullErrors: s.Tagged(
			map[string]string{"error-type": "queue-full", "action": "enqueue"},
		).Counter("errors"),
		connWriteErrors: s.Tagged(
			map[string]string{"error-type": "conn-write", "action": "drain"},
		).Counter("errors"),
	}
}
<|endoftext|>


<commit_before>package main

import (
    "database/sql"
    "encoding/json"
    "fmt"
    "log"
    "io/ioutil"
    "regexp"
    "strconv"
    "os"
    "sort"
    s "strings"
    "jfb/svgxml"
    _ "github.com/lib/pq"
)

type Config struct {
    Colours map[string]string `json:"colours"`
}

type DbConfig struct {
    Server      map[string]string       `json:"db_server"`
    Creds       map[string]string       `json:"db_creds"`
}

// set up integer array sorting
type IntArray []int
func (list IntArray) Len() int          { return len(list) }
func (list IntArray) Swap(a, b int)     { list[a], list[b] = list[b], list[a] }
func (list IntArray) Less(a, b int) bool { return list[a] < list[b] }

// suck in count data
func db_data() (map[string]int, map[string]int) {
    var dbconfig        DbConfig

    jsoncfg, err := ioutil.ReadFile("dbconfig.json")
    if err != nil {
        panic(err)
    }

    err = json.Unmarshal(jsoncfg, &dbconfig)
    if err != nil {
        fmt.Fprintf(os.Stderr, "DB config unmarshal: ")
        panic(err)
    }

    state_counts :=     make(map[string]int)
    county_counts :=    make(map[string]int)

    dbh, err := sql.Open("postgres",
        "postgres://" + dbconfig.Creds["username"] + ":" + dbconfig.Creds["password"] + "@" +
        dbconfig.Server["dbhost"] + "/" + dbconfig.Server["dbname"] + "?sslmode=require")
    if err != nil {
        log.Fatal(err)
    }

    rows, err := dbh.Query(`SELECT state, county, count(county) from hits where ` +
        `country = 'US' group by state, county`)
    if err != nil {
        log.Fatal(err)
    }

    defer rows.Close()
    for rows.Next() {
        var state, county string
        var count int
        if err := rows.Scan(&state, &county, &count); err != nil {
            log.Fatal(err)
        }
        state_counts[state] += count
        state_county_key := s.Replace(state + " " + county, " ", "_", -1)
        county_counts[state_county_key] = count
    }
    if err := rows.Err(); err != nil {
        log.Fatal(err)
    }

    return state_counts, county_counts

}

func main() {

    var config Config

    if len(os.Args) < 2 {
        fmt.Fprintf(os.Stderr, "Please specify at least one filename on the command line.\n")
        os.Exit(1)
    }

    jsoncfg, err := ioutil.ReadFile("config.json")
    if err != nil {
        panic(err)
    }

    mapsvg, err := ioutil.ReadFile(os.Args[1])
    if err != nil {
        panic(err)
    }

    err = json.Unmarshal(jsoncfg, &config)
    if err != nil {
        fmt.Fprintf(os.Stderr, "Config unmarshal: ")
        panic(err)
    }

    // make sorted list of keys (minimum counts) for later comparisons
    mincount := make([]int, len(config.Colours))
    i := 0
    for k, _ := range config.Colours {
        k_i, _ := strconv.ParseInt(k, 0, 64)
        mincount[i] = int(k_i)
	i++
    }

    sort.Sort(IntArray(mincount))

    mapsvg_obj := svgxml.XML2SVG(mapsvg)
    if mapsvg_obj == nil {
        os.Exit(1)
    }

    re_fill, err := regexp.Compile(`(fill:#)......`)
    if err != nil {
        panic(err)
    }

    state, _ := db_data()

    for state, state_count := range state {
	for _, mc := range mincount {
	    if state_count >= mc {
		element := svgxml.FindPathById(mapsvg_obj, state)
		if element != nil {
		    element.Style = string(re_fill.ReplaceAll([]byte(element.Style), []byte("${1}" + config.Colours[strconv.Itoa(mc)])))
		} else {
		    fmt.Fprintf(os.Stderr, "'%s' not found\n", state)
		}

	    }
	}
    }

    fmt.Println(string(svgxml.SVG2XML(mapsvg_obj, true)))

}

// vim:ai:sw=4:ts=8:
<commit_msg>change modeline, doesn't need to be vim-specific<commit_after>package main

import (
    "database/sql"
    "encoding/json"
    "fmt"
    "log"
    "io/ioutil"
    "regexp"
    "strconv"
    "os"
    "sort"
    s "strings"
    "jfb/svgxml"
    _ "github.com/lib/pq"
)

type Config struct {
    Colours map[string]string `json:"colours"`
}

type DbConfig struct {
    Server      map[string]string       `json:"db_server"`
    Creds       map[string]string       `json:"db_creds"`
}

// set up integer array sorting
type IntArray []int
func (list IntArray) Len() int          { return len(list) }
func (list IntArray) Swap(a, b int)     { list[a], list[b] = list[b], list[a] }
func (list IntArray) Less(a, b int) bool { return list[a] < list[b] }

// suck in count data
func db_data() (map[string]int, map[string]int) {
    var dbconfig        DbConfig

    jsoncfg, err := ioutil.ReadFile("dbconfig.json")
    if err != nil {
        panic(err)
    }

    err = json.Unmarshal(jsoncfg, &dbconfig)
    if err != nil {
        fmt.Fprintf(os.Stderr, "DB config unmarshal: ")
        panic(err)
    }

    state_counts :=     make(map[string]int)
    county_counts :=    make(map[string]int)

    dbh, err := sql.Open("postgres",
        "postgres://" + dbconfig.Creds["username"] + ":" + dbconfig.Creds["password"] + "@" +
        dbconfig.Server["dbhost"] + "/" + dbconfig.Server["dbname"] + "?sslmode=require")
    if err != nil {
        log.Fatal(err)
    }

    rows, err := dbh.Query(`SELECT state, county, count(county) from hits where ` +
        `country = 'US' group by state, county`)
    if err != nil {
        log.Fatal(err)
    }

    defer rows.Close()
    for rows.Next() {
        var state, county string
        var count int
        if err := rows.Scan(&state, &county, &count); err != nil {
            log.Fatal(err)
        }
        state_counts[state] += count
        state_county_key := s.Replace(state + " " + county, " ", "_", -1)
        county_counts[state_county_key] = count
    }
    if err := rows.Err(); err != nil {
        log.Fatal(err)
    }

    return state_counts, county_counts

}

func main() {

    var config Config

    if len(os.Args) < 2 {
        fmt.Fprintf(os.Stderr, "Please specify at least one filename on the command line.\n")
        os.Exit(1)
    }

    jsoncfg, err := ioutil.ReadFile("config.json")
    if err != nil {
        panic(err)
    }

    mapsvg, err := ioutil.ReadFile(os.Args[1])
    if err != nil {
        panic(err)
    }

    err = json.Unmarshal(jsoncfg, &config)
    if err != nil {
        fmt.Fprintf(os.Stderr, "Config unmarshal: ")
        panic(err)
    }

    // make sorted list of keys (minimum counts) for later comparisons
    mincount := make([]int, len(config.Colours))
    i := 0
    for k, _ := range config.Colours {
        k_i, _ := strconv.ParseInt(k, 0, 64)
        mincount[i] = int(k_i)
	i++
    }

    sort.Sort(IntArray(mincount))

    mapsvg_obj := svgxml.XML2SVG(mapsvg)
    if mapsvg_obj == nil {
        os.Exit(1)
    }

    re_fill, err := regexp.Compile(`(fill:#)......`)
    if err != nil {
        panic(err)
    }

    state, _ := db_data()

    for state, state_count := range state {
	for _, mc := range mincount {
	    if state_count >= mc {
		element := svgxml.FindPathById(mapsvg_obj, state)
		if element != nil {
		    element.Style = string(re_fill.ReplaceAll([]byte(element.Style), []byte("${1}" + config.Colours[strconv.Itoa(mc)])))
		} else {
		    fmt.Fprintf(os.Stderr, "'%s' not found\n", state)
		}

	    }
	}
    }

    fmt.Println(string(svgxml.SVG2XML(mapsvg_obj, true)))

}

// ex:ai:sw=4:ts=8:
<|endoftext|>