<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package handlers

import (
	"bytes"
	"context"
	"fmt"
	"net/http"
	"strings"
	"time"
	"unicode"
	"unicode/utf8"

	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metainternalversionscheme "k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/validation"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apiserver/pkg/admission"
	"k8s.io/apiserver/pkg/audit"
	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
	"k8s.io/apiserver/pkg/endpoints/handlers/finisher"
	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
	"k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/apiserver/pkg/features"
	"k8s.io/apiserver/pkg/registry/rest"
	"k8s.io/apiserver/pkg/util/dryrun"
	utilfeature "k8s.io/apiserver/pkg/util/feature"
	utiltrace "k8s.io/utils/trace"
)

var namespaceGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Namespace"}

func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Interface, includeName bool) http.HandlerFunc {
	return func(w http.ResponseWriter, req *http.Request) {
		// For performance tracking purposes.
		trace := utiltrace.New("Create", traceFields(req)...)
		defer trace.LogIfLong(500 * time.Millisecond)

		if isDryRun(req.URL) && !utilfeature.DefaultFeatureGate.Enabled(features.DryRun) {
			scope.err(errors.NewBadRequest("the dryRun feature is disabled"), w, req)
			return
		}

		namespace, name, err := scope.Namer.Name(req)
		if err != nil {
			if includeName {
				// name was required, return
				scope.err(err, w, req)
				return
			}

			// otherwise attempt to look up the namespace
			namespace, err = scope.Namer.Namespace(req)
			if err != nil {
				scope.err(err, w, req)
				return
			}
		}

		// enforce a timeout of at most requestTimeoutUpperBound (34s) or less if the user-provided
		// timeout inside the parent context is lower than requestTimeoutUpperBound.
		ctx, cancel := context.WithTimeout(req.Context(), requestTimeoutUpperBound)
		defer cancel()
		outputMediaType, _, err := negotiation.NegotiateOutputMediaType(req, scope.Serializer, scope)
		if err != nil {
			scope.err(err, w, req)
			return
		}

		gv := scope.Kind.GroupVersion()
		s, err := negotiation.NegotiateInputSerializer(req, false, scope.Serializer)
		if err != nil {
			scope.err(err, w, req)
			return
		}

		decoder := scope.Serializer.DecoderToVersion(s.Serializer, scope.HubGroupVersion)

		body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
		if err != nil {
			scope.err(err, w, req)
			return
		}

		options := &metav1.CreateOptions{}
		values := req.URL.Query()
		if err := metainternalversionscheme.ParameterCodec.DecodeParameters(values, scope.MetaGroupVersion, options); err != nil {
			err = errors.NewBadRequest(err.Error())
			scope.err(err, w, req)
			return
		}
		if errs := validation.ValidateCreateOptions(options); len(errs) > 0 {
			err := errors.NewInvalid(schema.GroupKind{Group: metav1.GroupName, Kind: "CreateOptions"}, "", errs)
			scope.err(err, w, req)
			return
		}
		options.TypeMeta.SetGroupVersionKind(metav1.SchemeGroupVersion.WithKind("CreateOptions"))

		defaultGVK := scope.Kind
		original := r.New()
		trace.Step("About to convert to expected version")
		obj, gvk, err := decoder.Decode(body, &defaultGVK, original)
		if err != nil {
			err = transformDecodeError(scope.Typer, err, original, gvk, body)
			scope.err(err, w, req)
			return
		}

		objGV := gvk.GroupVersion()
		if !scope.AcceptsGroupVersion(objGV) {
			err = errors.NewBadRequest(fmt.Sprintf("the API version in the data (%s) does not match the expected API version (%v)", objGV.String(), gv.String()))
			scope.err(err, w, req)
			return
		}
		trace.Step("Conversion done")

		// On create, get name from new object if unset
		if len(name) == 0 {
			_, name, _ = scope.Namer.ObjectName(obj)
		}
		if len(namespace) == 0 && *gvk == namespaceGVK {
			namespace = name
		}
		ctx = request.WithNamespace(ctx, namespace)

		ae := request.AuditEventFrom(ctx)
		admit = admission.WithAudit(admit, ae)
		audit.LogRequestObject(ae, obj, objGV, scope.Resource, scope.Subresource, scope.Serializer)

		userInfo, _ := request.UserFrom(ctx)

		trace.Step("About to store object in database")
		admissionAttributes := admission.NewAttributesRecord(obj, nil, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Create, options, dryrun.IsDryRun(options.DryRun), userInfo)
		requestFunc := func() (runtime.Object, error) {
			return r.Create(
				ctx,
				name,
				obj,
				rest.AdmissionToValidateObjectFunc(admit, admissionAttributes, scope),
				options,
			)
		}
		// Dedup owner references before updating managed fields
		dedupOwnerReferencesAndAddWarning(obj, req.Context(), false)
		result, err := finisher.FinishRequest(ctx, func() (runtime.Object, error) {
			if scope.FieldManager != nil {
				liveObj, err := scope.Creater.New(scope.Kind)
				if err != nil {
					return nil, fmt.Errorf("failed to create new object (Create for %v): %v", scope.Kind, err)
				}
				obj = scope.FieldManager.UpdateNoErrors(liveObj, obj, managerOrUserAgent(options.FieldManager, req.UserAgent()))
				admit = fieldmanager.NewManagedFieldsValidatingAdmissionController(admit)
			}
			if mutatingAdmission, ok := admit.(admission.MutationInterface); ok && mutatingAdmission.Handles(admission.Create) {
				if err := mutatingAdmission.Admit(ctx, admissionAttributes, scope); err != nil {
					return nil, err
				}
			}
			// Dedup owner references again after mutating admission happens
			dedupOwnerReferencesAndAddWarning(obj, req.Context(), true)
			result, err := requestFunc()
			// If the object wasn't committed to storage because it's serialized size was too large,
			// it is safe to remove managedFields (which can be large) and try again.
			if isTooLargeError(err) {
				if accessor, accessorErr := meta.Accessor(obj); accessorErr == nil {
					accessor.SetManagedFields(nil)
					result, err = requestFunc()
				}
			}
			return result, err
		})
		if err != nil {
			scope.err(err, w, req)
			return
		}
		trace.Step("Object stored in database")

		code := http.StatusCreated
		status, ok := result.(*metav1.Status)
		if ok && err == nil && status.Code == 0 {
			status.Code = int32(code)
		}

		transformResponseObject(ctx, scope, trace, req, w, code, outputMediaType, result)
	}
}

// CreateNamedResource returns a function that will handle a resource creation with name.
func CreateNamedResource(r rest.NamedCreater, scope *RequestScope, admission admission.Interface) http.HandlerFunc {
	return createHandler(r, scope, admission, true)
}

// CreateResource returns a function that will handle a resource creation.
func CreateResource(r rest.Creater, scope *RequestScope, admission admission.Interface) http.HandlerFunc {
	return createHandler(&namedCreaterAdapter{r}, scope, admission, false)
}

type namedCreaterAdapter struct {
	rest.Creater
}

func (c *namedCreaterAdapter) Create(ctx context.Context, name string, obj runtime.Object, createValidatingAdmission rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
	return c.Creater.Create(ctx, obj, createValidatingAdmission, options)
}

// manager is assumed to be already a valid value, we need to make
// userAgent into a valid value too.
func managerOrUserAgent(manager, userAgent string) string {
	if manager != "" {
		return manager
	}
	return prefixFromUserAgent(userAgent)
}

// prefixFromUserAgent takes the characters preceding the first /, quote
// unprintable character and then trim what's beyond the
// FieldManagerMaxLength limit.
func prefixFromUserAgent(u string) string {
	m := strings.Split(u, "/")[0]
	buf := bytes.NewBuffer(nil)
	for _, r := range m {
		// Ignore non-printable characters
		if !unicode.IsPrint(r) {
			continue
		}
		// Only append if we have room for it
		if buf.Len()+utf8.RuneLen(r) > validation.FieldManagerMaxLength {
			break
		}
		buf.WriteRune(r)
	}
	return buf.String()
}
<commit_msg>remove invalid nil check<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package handlers

import (
	"bytes"
	"context"
	"fmt"
	"net/http"
	"strings"
	"time"
	"unicode"
	"unicode/utf8"

	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metainternalversionscheme "k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/validation"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apiserver/pkg/admission"
	"k8s.io/apiserver/pkg/audit"
	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
	"k8s.io/apiserver/pkg/endpoints/handlers/finisher"
	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
	"k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/apiserver/pkg/features"
	"k8s.io/apiserver/pkg/registry/rest"
	"k8s.io/apiserver/pkg/util/dryrun"
	utilfeature "k8s.io/apiserver/pkg/util/feature"
	utiltrace "k8s.io/utils/trace"
)

var namespaceGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Namespace"}

func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Interface, includeName bool) http.HandlerFunc {
	return func(w http.ResponseWriter, req *http.Request) {
		// For performance tracking purposes.
		trace := utiltrace.New("Create", traceFields(req)...)
		defer trace.LogIfLong(500 * time.Millisecond)

		if isDryRun(req.URL) && !utilfeature.DefaultFeatureGate.Enabled(features.DryRun) {
			scope.err(errors.NewBadRequest("the dryRun feature is disabled"), w, req)
			return
		}

		namespace, name, err := scope.Namer.Name(req)
		if err != nil {
			if includeName {
				// name was required, return
				scope.err(err, w, req)
				return
			}

			// otherwise attempt to look up the namespace
			namespace, err = scope.Namer.Namespace(req)
			if err != nil {
				scope.err(err, w, req)
				return
			}
		}

		// enforce a timeout of at most requestTimeoutUpperBound (34s) or less if the user-provided
		// timeout inside the parent context is lower than requestTimeoutUpperBound.
		ctx, cancel := context.WithTimeout(req.Context(), requestTimeoutUpperBound)
		defer cancel()
		outputMediaType, _, err := negotiation.NegotiateOutputMediaType(req, scope.Serializer, scope)
		if err != nil {
			scope.err(err, w, req)
			return
		}

		gv := scope.Kind.GroupVersion()
		s, err := negotiation.NegotiateInputSerializer(req, false, scope.Serializer)
		if err != nil {
			scope.err(err, w, req)
			return
		}

		decoder := scope.Serializer.DecoderToVersion(s.Serializer, scope.HubGroupVersion)

		body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
		if err != nil {
			scope.err(err, w, req)
			return
		}

		options := &metav1.CreateOptions{}
		values := req.URL.Query()
		if err := metainternalversionscheme.ParameterCodec.DecodeParameters(values, scope.MetaGroupVersion, options); err != nil {
			err = errors.NewBadRequest(err.Error())
			scope.err(err, w, req)
			return
		}
		if errs := validation.ValidateCreateOptions(options); len(errs) > 0 {
			err := errors.NewInvalid(schema.GroupKind{Group: metav1.GroupName, Kind: "CreateOptions"}, "", errs)
			scope.err(err, w, req)
			return
		}
		options.TypeMeta.SetGroupVersionKind(metav1.SchemeGroupVersion.WithKind("CreateOptions"))

		defaultGVK := scope.Kind
		original := r.New()
		trace.Step("About to convert to expected version")
		obj, gvk, err := decoder.Decode(body, &defaultGVK, original)
		if err != nil {
			err = transformDecodeError(scope.Typer, err, original, gvk, body)
			scope.err(err, w, req)
			return
		}

		objGV := gvk.GroupVersion()
		if !scope.AcceptsGroupVersion(objGV) {
			err = errors.NewBadRequest(fmt.Sprintf("the API version in the data (%s) does not match the expected API version (%v)", objGV.String(), gv.String()))
			scope.err(err, w, req)
			return
		}
		trace.Step("Conversion done")

		// On create, get name from new object if unset
		if len(name) == 0 {
			_, name, _ = scope.Namer.ObjectName(obj)
		}
		if len(namespace) == 0 && *gvk == namespaceGVK {
			namespace = name
		}
		ctx = request.WithNamespace(ctx, namespace)

		ae := request.AuditEventFrom(ctx)
		admit = admission.WithAudit(admit, ae)
		audit.LogRequestObject(ae, obj, objGV, scope.Resource, scope.Subresource, scope.Serializer)

		userInfo, _ := request.UserFrom(ctx)

		trace.Step("About to store object in database")
		admissionAttributes := admission.NewAttributesRecord(obj, nil, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Create, options, dryrun.IsDryRun(options.DryRun), userInfo)
		requestFunc := func() (runtime.Object, error) {
			return r.Create(
				ctx,
				name,
				obj,
				rest.AdmissionToValidateObjectFunc(admit, admissionAttributes, scope),
				options,
			)
		}
		// Dedup owner references before updating managed fields
		dedupOwnerReferencesAndAddWarning(obj, req.Context(), false)
		result, err := finisher.FinishRequest(ctx, func() (runtime.Object, error) {
			if scope.FieldManager != nil {
				liveObj, err := scope.Creater.New(scope.Kind)
				if err != nil {
					return nil, fmt.Errorf("failed to create new object (Create for %v): %v", scope.Kind, err)
				}
				obj = scope.FieldManager.UpdateNoErrors(liveObj, obj, managerOrUserAgent(options.FieldManager, req.UserAgent()))
				admit = fieldmanager.NewManagedFieldsValidatingAdmissionController(admit)
			}
			if mutatingAdmission, ok := admit.(admission.MutationInterface); ok && mutatingAdmission.Handles(admission.Create) {
				if err := mutatingAdmission.Admit(ctx, admissionAttributes, scope); err != nil {
					return nil, err
				}
			}
			// Dedup owner references again after mutating admission happens
			dedupOwnerReferencesAndAddWarning(obj, req.Context(), true)
			result, err := requestFunc()
			// If the object wasn't committed to storage because it's serialized size was too large,
			// it is safe to remove managedFields (which can be large) and try again.
			if isTooLargeError(err) {
				if accessor, accessorErr := meta.Accessor(obj); accessorErr == nil {
					accessor.SetManagedFields(nil)
					result, err = requestFunc()
				}
			}
			return result, err
		})
		if err != nil {
			scope.err(err, w, req)
			return
		}
		trace.Step("Object stored in database")

		code := http.StatusCreated
		status, ok := result.(*metav1.Status)
		if ok && status.Code == 0 {
			status.Code = int32(code)
		}

		transformResponseObject(ctx, scope, trace, req, w, code, outputMediaType, result)
	}
}

// CreateNamedResource returns a function that will handle a resource creation with name.
func CreateNamedResource(r rest.NamedCreater, scope *RequestScope, admission admission.Interface) http.HandlerFunc {
	return createHandler(r, scope, admission, true)
}

// CreateResource returns a function that will handle a resource creation.
func CreateResource(r rest.Creater, scope *RequestScope, admission admission.Interface) http.HandlerFunc {
	return createHandler(&namedCreaterAdapter{r}, scope, admission, false)
}

type namedCreaterAdapter struct {
	rest.Creater
}

func (c *namedCreaterAdapter) Create(ctx context.Context, name string, obj runtime.Object, createValidatingAdmission rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
	return c.Creater.Create(ctx, obj, createValidatingAdmission, options)
}

// manager is assumed to be already a valid value, we need to make
// userAgent into a valid value too.
func managerOrUserAgent(manager, userAgent string) string {
	if manager != "" {
		return manager
	}
	return prefixFromUserAgent(userAgent)
}

// prefixFromUserAgent takes the characters preceding the first /, quote
// unprintable character and then trim what's beyond the
// FieldManagerMaxLength limit.
func prefixFromUserAgent(u string) string {
	m := strings.Split(u, "/")[0]
	buf := bytes.NewBuffer(nil)
	for _, r := range m {
		// Ignore non-printable characters
		if !unicode.IsPrint(r) {
			continue
		}
		// Only append if we have room for it
		if buf.Len()+utf8.RuneLen(r) > validation.FieldManagerMaxLength {
			break
		}
		buf.WriteRune(r)
	}
	return buf.String()
}
<|endoftext|>


<commit_before>package modules

import (
	"strings"
	"unicode"

	"github.com/pajlada/pajbot2/emotes"
	"github.com/pajlada/pajbot2/pkg"
	"github.com/pajlada/pajbot2/pkg/common"
)

type bttvEmoteParser struct {
	botChannel pkg.BotChannel

	globalEmotes *map[string]common.Emote
}

var bttvEmoteParserSpec = &moduleSpec{
	id:   "bttv_emote_parser",
	name: "BTTV emote parser",

	maker: newbttvEmoteParser,

	enabledByDefault: true,

	priority: -50000,
}

func newbttvEmoteParser() pkg.Module {
	return &bttvEmoteParser{
		globalEmotes: &emotes.GlobalEmotes.Bttv,
	}
}

func (m *bttvEmoteParser) Initialize(botChannel pkg.BotChannel, settings []byte) error {
	m.botChannel = botChannel
	return nil
}

func (m *bttvEmoteParser) Disable() error {
	return nil
}

func (m *bttvEmoteParser) Spec() pkg.ModuleSpec {
	return bttvEmoteParserSpec
}

func (m *bttvEmoteParser) BotChannel() pkg.BotChannel {
	return m.botChannel
}

func (m *bttvEmoteParser) OnWhisper(bot pkg.Sender, user pkg.User, message pkg.Message) error {
	return nil
}

func (m *bttvEmoteParser) OnMessage(bot pkg.Sender, channel pkg.Channel, user pkg.User, message pkg.Message, action pkg.Action) error {
	parts := strings.FieldsFunc(message.GetText(), func(r rune) bool {
		// TODO(pajlada): This needs better testing
		return r > 0xFF || unicode.IsSpace(r) || r == '!' || r == '.' || r == '$' || r == '^' || r == '#' || r == '*' || r == '@' || r == ')' || r == '%' || r == '&' || r > 0x7a || r < 0x30 || (r > 0x39 && r < 0x41) || (r > 0x5a && r < 0x61)
	})
	emoteCount := make(map[string]*common.Emote)
	for _, word := range parts {
		if emote, ok := emoteCount[word]; ok {
			emote.Count++
		} else if emote, ok := (*m.globalEmotes)[word]; ok {
			emoteCount[word] = &emote
		}
	}

	for _, emote := range emoteCount {
		message.AddBTTVEmote(emote)
	}

	return nil
}
<commit_msg>fix compilation<commit_after>package modules

import (
	"strings"
	"unicode"

	"github.com/pajlada/pajbot2/pkg"
	"github.com/pajlada/pajbot2/pkg/common"
	"github.com/pajlada/pajbot2/pkg/emotes"
)

type bttvEmoteParser struct {
	botChannel pkg.BotChannel

	globalEmotes *map[string]common.Emote
}

var bttvEmoteParserSpec = &moduleSpec{
	id:   "bttv_emote_parser",
	name: "BTTV emote parser",

	maker: newbttvEmoteParser,

	enabledByDefault: true,

	priority: -50000,
}

func newbttvEmoteParser() pkg.Module {
	return &bttvEmoteParser{
		globalEmotes: &emotes.GlobalEmotes.Bttv,
	}
}

func (m *bttvEmoteParser) Initialize(botChannel pkg.BotChannel, settings []byte) error {
	m.botChannel = botChannel
	return nil
}

func (m *bttvEmoteParser) Disable() error {
	return nil
}

func (m *bttvEmoteParser) Spec() pkg.ModuleSpec {
	return bttvEmoteParserSpec
}

func (m *bttvEmoteParser) BotChannel() pkg.BotChannel {
	return m.botChannel
}

func (m *bttvEmoteParser) OnWhisper(bot pkg.Sender, user pkg.User, message pkg.Message) error {
	return nil
}

func (m *bttvEmoteParser) OnMessage(bot pkg.Sender, channel pkg.Channel, user pkg.User, message pkg.Message, action pkg.Action) error {
	parts := strings.FieldsFunc(message.GetText(), func(r rune) bool {
		// TODO(pajlada): This needs better testing
		return r > 0xFF || unicode.IsSpace(r) || r == '!' || r == '.' || r == '$' || r == '^' || r == '#' || r == '*' || r == '@' || r == ')' || r == '%' || r == '&' || r > 0x7a || r < 0x30 || (r > 0x39 && r < 0x41) || (r > 0x5a && r < 0x61)
	})
	emoteCount := make(map[string]*common.Emote)
	for _, word := range parts {
		if emote, ok := emoteCount[word]; ok {
			emote.Count++
		} else if emote, ok := (*m.globalEmotes)[word]; ok {
			emoteCount[word] = &emote
		}
	}

	for _, emote := range emoteCount {
		message.AddBTTVEmote(emote)
	}

	return nil
}
<|endoftext|>


<commit_before>//
// Last.Backend LLC CONFIDENTIAL
// __________________
//
// [2014] - [2018] Last.Backend LLC
// All Rights Reserved.
//
// NOTICE:  All information contained herein is, and remains
// the property of Last.Backend LLC and its suppliers,
// if any.  The intellectual and technical concepts contained
// herein are proprietary to Last.Backend LLC
// and its suppliers and may be covered by Russian Federation and Foreign Patents,
// patents in process, and are protected by trade secret or copyright law.
// Dissemination of this information or reproduction of this material
// is strictly forbidden unless prior written permission is obtained
// from Last.Backend LLC.
//

package docker

import (
	"fmt"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/mount"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/api/types/strslice"
	"github.com/docker/go-connections/nat"
	"github.com/lastbackend/lastbackend/pkg/distribution/types"
	"strconv"
)

func GetConfig(manifest *types.ContainerManifest) *container.Config {

	var (
		volumes = make(map[string]struct{}, 0)
		ports   = make(nat.PortSet, 0)
	)

	for _, p := range manifest.Ports {
		port := nat.Port(strconv.Itoa(int(p.ContainerPort)))
		ports[port] = struct{}{}
	}

	return &container.Config{
		Hostname:     manifest.Network.Hostname,
		Domainname:   manifest.Network.Domain,
		Env:          manifest.Envs,
		ExposedPorts: ports,
		Volumes:      volumes,
		Labels:       manifest.Labels,
		Cmd:          strslice.StrSlice(manifest.Exec.Command),
		Entrypoint:   strslice.StrSlice(manifest.Exec.Entrypoint),
		Image:        manifest.Image,
		WorkingDir:   manifest.Exec.Workdir,
	}
}

func GetHostConfig(manifest *types.ContainerManifest) *container.HostConfig {

	rPolicy := container.RestartPolicy{
		Name:              manifest.RestartPolicy.Policy,
		MaximumRetryCount: manifest.RestartPolicy.Attempt,
	}

	resources := container.Resources{
		Memory:    manifest.Resources.Limits.RAM,
		CPUShares: manifest.Resources.Limits.CPU,
	}

	var (
		ports  = make(nat.PortMap, 0)
		mounts []mount.Mount
		links  []string
	)

	for _, l := range manifest.Links {
		links = append(links, fmt.Sprintf("%s:%s", l.Link, l.Alias))
	}

	logC := container.LogConfig{}
	ports = make(nat.PortMap)

	for _, p := range manifest.Ports {
		port := nat.Port(strconv.Itoa(int(p.ContainerPort)))
		ports[port] = make([]nat.PortBinding, 0)
		if p.HostPort != 0 {

			if p.HostIP == types.EmptyString {
				p.HostIP = "0.0.0.0"
			}

			ports[port] = append(ports[port], nat.PortBinding{
				HostIP:   p.HostIP,
				HostPort: fmt.Sprintf("%d", p.HostPort),
			})
		}

	}

	cfg := container.HostConfig{
		Binds:           manifest.Binds,
		LogConfig:       logC,
		NetworkMode:     container.NetworkMode(manifest.Network.Mode),
		PortBindings:    ports,
		DNS:             manifest.DNS.Server,
		DNSOptions:      manifest.DNS.Options,
		DNSSearch:       manifest.DNS.Search,
		RestartPolicy:   rPolicy,
		Resources:       resources,
		Mounts:          mounts,
		Links:           links,
		Privileged:      manifest.Security.Privileged,
		ExtraHosts:      manifest.ExtraHosts,
		PublishAllPorts: manifest.PublishAllPorts,
		AutoRemove:      manifest.AutoRemove,
	}

	if cfg.NetworkMode != types.EmptyString {
		cfg.DNS = make([]string, 0)
		cfg.DNSOptions = make([]string, 0)
		cfg.DNSSearch = make([]string, 0)
	}

	return &cfg
}

func GetNetworkConfig(manifest *types.ContainerManifest) *network.NetworkingConfig {

	cfg := &network.NetworkingConfig{}

	return cfg
}
<commit_msg>changed the limit cpu setting for host config<commit_after>//
// Last.Backend LLC CONFIDENTIAL
// __________________
//
// [2014] - [2018] Last.Backend LLC
// All Rights Reserved.
//
// NOTICE:  All information contained herein is, and remains
// the property of Last.Backend LLC and its suppliers,
// if any.  The intellectual and technical concepts contained
// herein are proprietary to Last.Backend LLC
// and its suppliers and may be covered by Russian Federation and Foreign Patents,
// patents in process, and are protected by trade secret or copyright law.
// Dissemination of this information or reproduction of this material
// is strictly forbidden unless prior written permission is obtained
// from Last.Backend LLC.
//

package docker

import (
	"fmt"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/mount"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/api/types/strslice"
	"github.com/docker/go-connections/nat"
	"github.com/lastbackend/lastbackend/pkg/distribution/types"
	"strconv"
)

func GetConfig(manifest *types.ContainerManifest) *container.Config {

	var (
		volumes = make(map[string]struct{}, 0)
		ports   = make(nat.PortSet, 0)
	)

	for _, p := range manifest.Ports {
		port := nat.Port(strconv.Itoa(int(p.ContainerPort)))
		ports[port] = struct{}{}
	}

	return &container.Config{
		Hostname:     manifest.Network.Hostname,
		Domainname:   manifest.Network.Domain,
		Env:          manifest.Envs,
		ExposedPorts: ports,
		Volumes:      volumes,
		Labels:       manifest.Labels,
		Cmd:          strslice.StrSlice(manifest.Exec.Command),
		Entrypoint:   strslice.StrSlice(manifest.Exec.Entrypoint),
		Image:        manifest.Image,
		WorkingDir:   manifest.Exec.Workdir,
	}
}

func GetHostConfig(manifest *types.ContainerManifest) *container.HostConfig {

	rPolicy := container.RestartPolicy{
		Name:              manifest.RestartPolicy.Policy,
		MaximumRetryCount: manifest.RestartPolicy.Attempt,
	}

	resources := container.Resources{
		Memory:   manifest.Resources.Limits.RAM,
		NanoCPUs: manifest.Resources.Limits.CPU,
	}

	var (
		ports  = make(nat.PortMap, 0)
		mounts []mount.Mount
		links  []string
	)

	for _, l := range manifest.Links {
		links = append(links, fmt.Sprintf("%s:%s", l.Link, l.Alias))
	}

	logC := container.LogConfig{}
	ports = make(nat.PortMap)

	for _, p := range manifest.Ports {
		port := nat.Port(strconv.Itoa(int(p.ContainerPort)))
		ports[port] = make([]nat.PortBinding, 0)
		if p.HostPort != 0 {

			if p.HostIP == types.EmptyString {
				p.HostIP = "0.0.0.0"
			}

			ports[port] = append(ports[port], nat.PortBinding{
				HostIP:   p.HostIP,
				HostPort: fmt.Sprintf("%d", p.HostPort),
			})
		}

	}

	cfg := container.HostConfig{
		Binds:           manifest.Binds,
		LogConfig:       logC,
		NetworkMode:     container.NetworkMode(manifest.Network.Mode),
		PortBindings:    ports,
		DNS:             manifest.DNS.Server,
		DNSOptions:      manifest.DNS.Options,
		DNSSearch:       manifest.DNS.Search,
		RestartPolicy:   rPolicy,
		Resources:       resources,
		Mounts:          mounts,
		Links:           links,
		Privileged:      manifest.Security.Privileged,
		ExtraHosts:      manifest.ExtraHosts,
		PublishAllPorts: manifest.PublishAllPorts,
		AutoRemove:      manifest.AutoRemove,
	}

	if cfg.NetworkMode != types.EmptyString {
		cfg.DNS = make([]string, 0)
		cfg.DNSOptions = make([]string, 0)
		cfg.DNSSearch = make([]string, 0)
	}

	return &cfg
}

func GetNetworkConfig(manifest *types.ContainerManifest) *network.NetworkingConfig {

	cfg := &network.NetworkingConfig{}

	return cfg
}
<|endoftext|>


<commit_before>package containerimage

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/leases"
	"github.com/containerd/containerd/platforms"
	"github.com/containerd/containerd/rootfs"
	"github.com/moby/buildkit/exporter"
	"github.com/moby/buildkit/session"
	"github.com/moby/buildkit/snapshot"
	"github.com/moby/buildkit/util/leaseutil"
	"github.com/moby/buildkit/util/push"
	"github.com/moby/buildkit/util/resolver"
	digest "github.com/opencontainers/go-digest"
	"github.com/opencontainers/image-spec/identity"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

const (
	keyImageName    = "name"
	keyPush         = "push"
	keyPushByDigest = "push-by-digest"
	keyInsecure     = "registry.insecure"
	keyUnpack       = "unpack"
	ociTypes        = "oci-mediatypes"
)

type Opt struct {
	SessionManager *session.Manager
	ImageWriter    *ImageWriter
	Images         images.Store
	ResolverOpt    resolver.ResolveOptionsFunc
	LeaseManager   leases.Manager
}

type imageExporter struct {
	opt Opt
}

// New returns a new containerimage exporter instance that supports exporting
// to an image store and pushing the image to registry.
// This exporter supports following values in returned kv map:
// - containerimage.digest - The digest of the root manifest for the image.
func New(opt Opt) (exporter.Exporter, error) {
	im := &imageExporter{opt: opt}
	return im, nil
}

func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exporter.ExporterInstance, error) {
	i := &imageExporterInstance{imageExporter: e}
	for k, v := range opt {
		switch k {
		case keyImageName:
			i.targetName = v
		case keyPush:
			if v == "" {
				i.push = true
				continue
			}
			b, err := strconv.ParseBool(v)
			if err != nil {
				return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
			}
			i.push = b
		case keyPushByDigest:
			if v == "" {
				i.pushByDigest = true
				continue
			}
			b, err := strconv.ParseBool(v)
			if err != nil {
				return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
			}
			i.pushByDigest = b
		case keyInsecure:
			if v == "" {
				i.insecure = true
				continue
			}
			b, err := strconv.ParseBool(v)
			if err != nil {
				return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
			}
			i.insecure = b
		case keyUnpack:
			if v == "" {
				i.unpack = true
				continue
			}
			b, err := strconv.ParseBool(v)
			if err != nil {
				return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
			}
			i.unpack = b
		case ociTypes:
			if v == "" {
				i.ociTypes = true
				continue
			}
			b, err := strconv.ParseBool(v)
			if err != nil {
				return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
			}
			i.ociTypes = b
		default:
			if i.meta == nil {
				i.meta = make(map[string][]byte)
			}
			i.meta[k] = []byte(v)
		}
	}
	return i, nil
}

type imageExporterInstance struct {
	*imageExporter
	targetName   string
	push         bool
	pushByDigest bool
	unpack       bool
	insecure     bool
	ociTypes     bool
	meta         map[string][]byte
}

func (e *imageExporterInstance) Name() string {
	return "exporting to image"
}

func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source) (map[string]string, error) {
	if src.Metadata == nil {
		src.Metadata = make(map[string][]byte)
	}
	for k, v := range e.meta {
		src.Metadata[k] = v
	}

	ctx, done, err := leaseutil.WithLease(ctx, e.opt.LeaseManager, leaseutil.MakeTemporary)
	if err != nil {
		return nil, err
	}
	defer done(context.TODO())

	desc, err := e.opt.ImageWriter.Commit(ctx, src, e.ociTypes)
	if err != nil {
		return nil, err
	}

	defer func() {
		e.opt.ImageWriter.ContentStore().Delete(context.TODO(), desc.Digest)
	}()

	resp := make(map[string]string)

	if n, ok := src.Metadata["image.name"]; e.targetName == "*" && ok {
		e.targetName = string(n)
	}

	if e.targetName != "" {
		targetNames := strings.Split(e.targetName, ",")
		for _, targetName := range targetNames {
			if e.opt.Images != nil {
				tagDone := oneOffProgress(ctx, "naming to "+targetName)
				img := images.Image{
					Name:      targetName,
					Target:    *desc,
					CreatedAt: time.Now(),
				}

				if _, err := e.opt.Images.Update(ctx, img); err != nil {
					if !errdefs.IsNotFound(err) {
						return nil, tagDone(err)
					}

					if _, err := e.opt.Images.Create(ctx, img); err != nil {
						return nil, tagDone(err)
					}
				}
				tagDone(nil)

				if e.unpack {
					if err := e.unpackImage(ctx, img); err != nil {
						return nil, err
					}
				}
			}
			if e.push {
				if err := push.Push(ctx, e.opt.SessionManager, e.opt.ImageWriter.ContentStore(), desc.Digest, targetName, e.insecure, e.opt.ResolverOpt, e.pushByDigest); err != nil {
					return nil, err
				}
			}
		}
		resp["image.name"] = e.targetName
	}

	resp["containerimage.digest"] = desc.Digest.String()
	return resp, nil
}

func (e *imageExporterInstance) unpackImage(ctx context.Context, img images.Image) (err0 error) {
	unpackDone := oneOffProgress(ctx, "unpacking to "+img.Name)
	defer func() {
		unpackDone(err0)
	}()

	var (
		contentStore = e.opt.ImageWriter.ContentStore()
		applier      = e.opt.ImageWriter.Applier()
		snapshotter  = e.opt.ImageWriter.Snapshotter()
	)

	// fetch manifest by default platform
	manifest, err := images.Manifest(ctx, contentStore, img.Target, platforms.Default())
	if err != nil {
		return err
	}

	layers, err := getLayers(ctx, contentStore, manifest)
	if err != nil {
		return err
	}

	// get containerd snapshotter
	ctrdSnapshotter, release := snapshot.NewContainerdSnapshotter(snapshotter)
	defer release()

	var chain []digest.Digest
	for _, layer := range layers {
		if _, err := rootfs.ApplyLayer(ctx, layer, chain, ctrdSnapshotter, applier); err != nil {
			return err
		}
		chain = append(chain, layer.Diff.Digest)
	}

	var (
		keyGCLabel   = fmt.Sprintf("containerd.io/gc.ref.snapshot.%s", snapshotter.Name())
		valueGCLabel = identity.ChainID(chain).String()
	)

	cinfo := content.Info{
		Digest: manifest.Config.Digest,
		Labels: map[string]string{keyGCLabel: valueGCLabel},
	}
	_, err = contentStore.Update(ctx, cinfo, fmt.Sprintf("labels.%s", keyGCLabel))
	return err
}

func getLayers(ctx context.Context, contentStore content.Store, manifest ocispec.Manifest) ([]rootfs.Layer, error) {
	diffIDs, err := images.RootFS(ctx, contentStore, manifest.Config)
	if err != nil {
		return nil, errors.Wrap(err, "failed to resolve rootfs")
	}

	if len(diffIDs) != len(manifest.Layers) {
		return nil, errors.Errorf("mismatched image rootfs and manifest layers")
	}

	layers := make([]rootfs.Layer, len(diffIDs))
	for i := range diffIDs {
		layers[i].Diff = ocispec.Descriptor{
			MediaType: ocispec.MediaTypeImageLayer,
			Digest:    diffIDs[i],
		}
		layers[i].Blob = manifest.Layers[i]
	}
	return layers, nil
}
<commit_msg>exporter: add canonical and dangling image naming<commit_after>package containerimage

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/leases"
	"github.com/containerd/containerd/platforms"
	"github.com/containerd/containerd/rootfs"
	"github.com/moby/buildkit/exporter"
	"github.com/moby/buildkit/session"
	"github.com/moby/buildkit/snapshot"
	"github.com/moby/buildkit/util/leaseutil"
	"github.com/moby/buildkit/util/push"
	"github.com/moby/buildkit/util/resolver"
	digest "github.com/opencontainers/go-digest"
	"github.com/opencontainers/image-spec/identity"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

const (
	keyImageName      = "name"
	keyPush           = "push"
	keyPushByDigest   = "push-by-digest"
	keyInsecure       = "registry.insecure"
	keyUnpack         = "unpack"
	keyDanglingPrefix = "dangling-name-prefix"
	keyNameCanonical  = "name-canonical"
	ociTypes          = "oci-mediatypes"
)

type Opt struct {
	SessionManager *session.Manager
	ImageWriter    *ImageWriter
	Images         images.Store
	ResolverOpt    resolver.ResolveOptionsFunc
	LeaseManager   leases.Manager
}

type imageExporter struct {
	opt Opt
}

// New returns a new containerimage exporter instance that supports exporting
// to an image store and pushing the image to registry.
// This exporter supports following values in returned kv map:
// - containerimage.digest - The digest of the root manifest for the image.
func New(opt Opt) (exporter.Exporter, error) {
	im := &imageExporter{opt: opt}
	return im, nil
}

func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exporter.ExporterInstance, error) {
	i := &imageExporterInstance{imageExporter: e}
	for k, v := range opt {
		switch k {
		case keyImageName:
			i.targetName = v
		case keyPush:
			if v == "" {
				i.push = true
				continue
			}
			b, err := strconv.ParseBool(v)
			if err != nil {
				return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
			}
			i.push = b
		case keyPushByDigest:
			if v == "" {
				i.pushByDigest = true
				continue
			}
			b, err := strconv.ParseBool(v)
			if err != nil {
				return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
			}
			i.pushByDigest = b
		case keyInsecure:
			if v == "" {
				i.insecure = true
				continue
			}
			b, err := strconv.ParseBool(v)
			if err != nil {
				return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
			}
			i.insecure = b
		case keyUnpack:
			if v == "" {
				i.unpack = true
				continue
			}
			b, err := strconv.ParseBool(v)
			if err != nil {
				return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
			}
			i.unpack = b
		case ociTypes:
			if v == "" {
				i.ociTypes = true
				continue
			}
			b, err := strconv.ParseBool(v)
			if err != nil {
				return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
			}
			i.ociTypes = b
		case keyDanglingPrefix:
			i.danglingPrefix = v
		case keyNameCanonical:
			if v == "" {
				i.nameCanonical = true
				continue
			}
			b, err := strconv.ParseBool(v)
			if err != nil {
				return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
			}
			i.nameCanonical = b
		default:
			if i.meta == nil {
				i.meta = make(map[string][]byte)
			}
			i.meta[k] = []byte(v)
		}
	}
	return i, nil
}

type imageExporterInstance struct {
	*imageExporter
	targetName     string
	push           bool
	pushByDigest   bool
	unpack         bool
	insecure       bool
	ociTypes       bool
	nameCanonical  bool
	danglingPrefix string
	meta           map[string][]byte
}

func (e *imageExporterInstance) Name() string {
	return "exporting to image"
}

func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source) (map[string]string, error) {
	if src.Metadata == nil {
		src.Metadata = make(map[string][]byte)
	}
	for k, v := range e.meta {
		src.Metadata[k] = v
	}

	ctx, done, err := leaseutil.WithLease(ctx, e.opt.LeaseManager, leaseutil.MakeTemporary)
	if err != nil {
		return nil, err
	}
	defer done(context.TODO())

	desc, err := e.opt.ImageWriter.Commit(ctx, src, e.ociTypes)
	if err != nil {
		return nil, err
	}

	defer func() {
		e.opt.ImageWriter.ContentStore().Delete(context.TODO(), desc.Digest)
	}()

	resp := make(map[string]string)

	if n, ok := src.Metadata["image.name"]; e.targetName == "*" && ok {
		e.targetName = string(n)
	}

	nameCanonical := e.nameCanonical
	if e.targetName == "" && e.danglingPrefix != "" {
		e.targetName = e.danglingPrefix + "@" + desc.Digest.String()
		nameCanonical = false
	}

	if e.targetName != "" {
		targetNames := strings.Split(e.targetName, ",")
		for _, targetName := range targetNames {
			if e.opt.Images != nil {
				tagDone := oneOffProgress(ctx, "naming to "+targetName)
				img := images.Image{
					Target:    *desc,
					CreatedAt: time.Now(),
				}
				sfx := []string{""}
				if nameCanonical {
					sfx = append(sfx, "@"+desc.Digest.String())
				}
				for _, sfx := range sfx {
					img.Name = targetName + sfx
					if _, err := e.opt.Images.Update(ctx, img); err != nil {
						if !errdefs.IsNotFound(err) {
							return nil, tagDone(err)
						}

						if _, err := e.opt.Images.Create(ctx, img); err != nil {
							return nil, tagDone(err)
						}
					}
				}
				tagDone(nil)

				if e.unpack {
					if err := e.unpackImage(ctx, img); err != nil {
						return nil, err
					}
				}
			}
			if e.push {
				if err := push.Push(ctx, e.opt.SessionManager, e.opt.ImageWriter.ContentStore(), desc.Digest, targetName, e.insecure, e.opt.ResolverOpt, e.pushByDigest); err != nil {
					return nil, err
				}
			}
		}
		resp["image.name"] = e.targetName
	}

	resp["containerimage.digest"] = desc.Digest.String()
	return resp, nil
}

func (e *imageExporterInstance) unpackImage(ctx context.Context, img images.Image) (err0 error) {
	unpackDone := oneOffProgress(ctx, "unpacking to "+img.Name)
	defer func() {
		unpackDone(err0)
	}()

	var (
		contentStore = e.opt.ImageWriter.ContentStore()
		applier      = e.opt.ImageWriter.Applier()
		snapshotter  = e.opt.ImageWriter.Snapshotter()
	)

	// fetch manifest by default platform
	manifest, err := images.Manifest(ctx, contentStore, img.Target, platforms.Default())
	if err != nil {
		return err
	}

	layers, err := getLayers(ctx, contentStore, manifest)
	if err != nil {
		return err
	}

	// get containerd snapshotter
	ctrdSnapshotter, release := snapshot.NewContainerdSnapshotter(snapshotter)
	defer release()

	var chain []digest.Digest
	for _, layer := range layers {
		if _, err := rootfs.ApplyLayer(ctx, layer, chain, ctrdSnapshotter, applier); err != nil {
			return err
		}
		chain = append(chain, layer.Diff.Digest)
	}

	var (
		keyGCLabel   = fmt.Sprintf("containerd.io/gc.ref.snapshot.%s", snapshotter.Name())
		valueGCLabel = identity.ChainID(chain).String()
	)

	cinfo := content.Info{
		Digest: manifest.Config.Digest,
		Labels: map[string]string{keyGCLabel: valueGCLabel},
	}
	_, err = contentStore.Update(ctx, cinfo, fmt.Sprintf("labels.%s", keyGCLabel))
	return err
}

func getLayers(ctx context.Context, contentStore content.Store, manifest ocispec.Manifest) ([]rootfs.Layer, error) {
	diffIDs, err := images.RootFS(ctx, contentStore, manifest.Config)
	if err != nil {
		return nil, errors.Wrap(err, "failed to resolve rootfs")
	}

	if len(diffIDs) != len(manifest.Layers) {
		return nil, errors.Errorf("mismatched image rootfs and manifest layers")
	}

	layers := make([]rootfs.Layer, len(diffIDs))
	for i := range diffIDs {
		layers[i].Diff = ocispec.Descriptor{
			MediaType: ocispec.MediaTypeImageLayer,
			Digest:    diffIDs[i],
		}
		layers[i].Blob = manifest.Layers[i]
	}
	return layers, nil
}
<|endoftext|>


<commit_before>package crawler

import (
	"github.com/ipfs-search/ipfs-search/indexer"
	"github.com/ipfs-search/ipfs-search/queue"
	"github.com/ipfs/go-ipfs-api"
	"log"
	"net"
	"net/url"
	// "path"
	"strings"
	"syscall"
	"time"
)

// Args describe a resource to be crawled
type Args struct {
	Hash       string
	Name       string
	Size       uint64
	ParentHash string
	ParentName string // This is legacy, should be removed
}

// Crawler consumes file and hash queues and indexes them
type Crawler struct {
	Config *Config

	Shell     *shell.Shell
	Indexer   *indexer.Indexer
	FileQueue *queue.TaskQueue
	HashQueue *queue.TaskQueue
}

// skipItem determines whether a particular item should not be indexed
// This holds particularly to partial content.
func (c *Crawler) skipItem(args *Args) bool {
	if args.Size == c.Config.PartialSize && args.ParentHash == "" {
		log.Printf("Skipping unreferenced partial content for file %s", args.Hash)
		return true
	}

	return false
}

// Handle errors graceously, returns try again bool and original error
func (c *Crawler) handleError(err error, hash string) (bool, error) {
	if _, ok := err.(*shell.Error); ok && strings.Contains(err.Error(), "proto") {
		// We're not recovering from protocol errors, so panic

		// Attempt to index panic to prevent re-indexing
		m := metadata{
			"error": err.Error(),
		}

		c.Indexer.IndexItem("invalid", hash, m)

		panic(err)
	}

	if uerr, ok := err.(*url.Error); ok {
		// URL errors

		log.Printf("URL error: %v", uerr)

		if uerr.Timeout() {
			// Fail on timeouts
			return false, err
		}

		if uerr.Temporary() {
			// Retry on other temp errors
			return true, nil
		}

		// Somehow, the errors below are not temp errors !?
		switch t := uerr.Err.(type) {
		case *net.OpError:
			if t.Op == "dial" {
				log.Printf("Unknown host %v", t)
				return true, nil

			} else if t.Op == "read" {
				log.Printf("Connection refused %v", t)
				return true, nil
			}

		case syscall.Errno:
			if t == syscall.ECONNREFUSED {
				log.Printf("Connection refused %v", t)
				return true, nil
			}
		}
	}

	return false, err
}

// getFileList return list of files and/or type of item (directory/file)
func (c *Crawler) getFileList(args *Args) (list *shell.UnixLsObject, err error) {
	url := hashURL(args.Hash)

	tryAgain := true
	for tryAgain {
		list, err = c.Shell.FileList(url)

		tryAgain, err = c.handleError(err, args.Hash)

		if tryAgain {
			log.Printf("Retrying in %s", c.Config.RetryWait)
			time.Sleep(c.Config.RetryWait)
		}
	}

	return
}

// queueList queues any items in a given list/directory
func (c *Crawler) queueList(args *Args, list *shell.UnixLsObject) (err error) {
	for _, link := range list.Links {
		dirArgs := &Args{
			Hash:       link.Hash,
			Name:       link.Name,
			Size:       link.Size,
			ParentHash: args.Hash,
		}

		switch link.Type {
		case "File":
			// Add file to crawl queue
			err = c.FileQueue.AddTask(dirArgs)
		case "Directory":
			// Add directory to crawl queue
			err = c.HashQueue.AddTask(dirArgs)
		default:
			log.Printf("Type '%s' skipped for '%s'", link.Type, args.Hash)
		}
	}

	return
}

// CrawlHash crawls a particular hash (file or directory)
func (c *Crawler) CrawlHash(args *Args) error {
	references, alreadyIndexed, err := c.indexReferences(args.Hash, args.Name, args.ParentHash)
	if err != nil {
		return err
	}

	if alreadyIndexed {
		return nil
	}

	log.Printf("Crawling hash '%s' (%s)", args.Hash, args.Name)

	list, err := c.getFileList(args)
	if err != nil {
		return err
	}

	switch list.Type {
	case "File":
		// Add to file crawl queue
		fileArgs := Args{
			Hash:       args.Hash,
			Name:       args.Name,
			Size:       list.Size,
			ParentHash: args.ParentHash,
		}

		if c.skipItem(args) {
			return nil
		}

		err = c.FileQueue.AddTask(fileArgs)
		if err != nil {
			// failed to send the task
			return err
		}
	case "Directory":
		// Queue indexing of linked items
		err = c.queueList(args, list)
		if err != nil {
			return err
		}

		// Index name and size for directory and directory items
		properties := map[string]interface{}{
			"links":      list.Links,
			"size":       list.Size,
			"references": references,
		}

		if c.skipItem(args) {
			return nil
		}

		err := c.Indexer.IndexItem("directory", args.Hash, properties)
		if err != nil {
			return err
		}

	default:
		log.Printf("Type '%s' skipped for '%s'", list.Type, args.Hash)
	}

	log.Printf("Finished hash %s", args.Hash)

	return nil
}

// CrawlFile crawls a single object, known to be a file
func (c *Crawler) CrawlFile(args *Args) error {
	if c.skipItem(args) {
		return nil
	}

	references, alreadyIndexed, err := c.indexReferences(args.Hash, args.Name, args.ParentHash)

	if err != nil {
		return err
	}

	if alreadyIndexed {
		return nil
	}

	log.Printf("Crawling file %s (%s)\n", args.Hash, args.Name)

	m := make(metadata)
	c.getMetadata(args, &m)
	if err != nil {
		return err
	}

	// Add previously found references now
	m["size"] = args.Size
	m["references"] = references

	err = c.Indexer.IndexItem("file", args.Hash, m)
	if err != nil {
		return err
	}

	log.Printf("Finished file %s", args.Hash)

	return nil
}
<commit_msg>Add nice TODO.<commit_after>package crawler

import (
	"github.com/ipfs-search/ipfs-search/indexer"
	"github.com/ipfs-search/ipfs-search/queue"
	"github.com/ipfs/go-ipfs-api"
	"log"
	"net"
	"net/url"
	// "path"
	"strings"
	"syscall"
	"time"
)

// Args describe a resource to be crawled
type Args struct {
	Hash       string
	Name       string
	Size       uint64
	ParentHash string
	ParentName string // This is legacy, should be removed
}

// Crawler consumes file and hash queues and indexes them
type Crawler struct {
	Config *Config

	Shell     *shell.Shell
	Indexer   *indexer.Indexer
	FileQueue *queue.TaskQueue
	HashQueue *queue.TaskQueue
}

// skipItem determines whether a particular item should not be indexed
// This holds particularly to partial content.
func (c *Crawler) skipItem(args *Args) bool {
	if args.Size == c.Config.PartialSize && args.ParentHash == "" {
		log.Printf("Skipping unreferenced partial content for file %s", args.Hash)
		return true
	}

	return false
}

// Handle errors graceously, returns try again bool and original error
// TODO: this handles both errors for listing as well as metadata errors,
// which seems a very bad idea and makes this function unnecessarily complex.
// We should figure out which code handles which and split it up.
func (c *Crawler) handleError(err error, hash string) (bool, error) {
	if _, ok := err.(*shell.Error); ok && strings.Contains(err.Error(), "proto") {
		// We're not recovering from protocol errors, so panic

		// Attempt to index panic to prevent re-indexing
		m := metadata{
			"error": err.Error(),
		}

		c.Indexer.IndexItem("invalid", hash, m)

		panic(err)
	}

	if uerr, ok := err.(*url.Error); ok {
		// URL errors

		log.Printf("URL error: %v", uerr)

		if uerr.Timeout() {
			// Fail on timeouts
			return false, err
		}

		if uerr.Temporary() {
			// Retry on other temp errors
			return true, nil
		}

		// Somehow, the errors below are not temp errors !?
		switch t := uerr.Err.(type) {
		case *net.OpError:
			if t.Op == "dial" {
				log.Printf("Unknown host %v", t)
				return true, nil

			} else if t.Op == "read" {
				log.Printf("Connection refused %v", t)
				return true, nil
			}

		case syscall.Errno:
			if t == syscall.ECONNREFUSED {
				log.Printf("Connection refused %v", t)
				return true, nil
			}
		}
	}

	return false, err
}

// getFileList return list of files and/or type of item (directory/file)
func (c *Crawler) getFileList(args *Args) (list *shell.UnixLsObject, err error) {
	url := hashURL(args.Hash)

	tryAgain := true
	for tryAgain {
		list, err = c.Shell.FileList(url)

		tryAgain, err = c.handleError(err, args.Hash)

		if tryAgain {
			log.Printf("Retrying in %s", c.Config.RetryWait)
			time.Sleep(c.Config.RetryWait)
		}
	}

	return
}

// queueList queues any items in a given list/directory
func (c *Crawler) queueList(args *Args, list *shell.UnixLsObject) (err error) {
	for _, link := range list.Links {
		dirArgs := &Args{
			Hash:       link.Hash,
			Name:       link.Name,
			Size:       link.Size,
			ParentHash: args.Hash,
		}

		switch link.Type {
		case "File":
			// Add file to crawl queue
			err = c.FileQueue.AddTask(dirArgs)
		case "Directory":
			// Add directory to crawl queue
			err = c.HashQueue.AddTask(dirArgs)
		default:
			log.Printf("Type '%s' skipped for '%s'", link.Type, args.Hash)
		}
	}

	return
}

// CrawlHash crawls a particular hash (file or directory)
func (c *Crawler) CrawlHash(args *Args) error {
	references, alreadyIndexed, err := c.indexReferences(args.Hash, args.Name, args.ParentHash)
	if err != nil {
		return err
	}

	if alreadyIndexed {
		return nil
	}

	log.Printf("Crawling hash '%s' (%s)", args.Hash, args.Name)

	list, err := c.getFileList(args)
	if err != nil {
		return err
	}

	switch list.Type {
	case "File":
		// Add to file crawl queue
		fileArgs := Args{
			Hash:       args.Hash,
			Name:       args.Name,
			Size:       list.Size,
			ParentHash: args.ParentHash,
		}

		if c.skipItem(args) {
			return nil
		}

		err = c.FileQueue.AddTask(fileArgs)
		if err != nil {
			// failed to send the task
			return err
		}
	case "Directory":
		// Queue indexing of linked items
		err = c.queueList(args, list)
		if err != nil {
			return err
		}

		// Index name and size for directory and directory items
		properties := map[string]interface{}{
			"links":      list.Links,
			"size":       list.Size,
			"references": references,
		}

		if c.skipItem(args) {
			return nil
		}

		err := c.Indexer.IndexItem("directory", args.Hash, properties)
		if err != nil {
			return err
		}

	default:
		log.Printf("Type '%s' skipped for '%s'", list.Type, args.Hash)
	}

	log.Printf("Finished hash %s", args.Hash)

	return nil
}

// CrawlFile crawls a single object, known to be a file
func (c *Crawler) CrawlFile(args *Args) error {
	if c.skipItem(args) {
		return nil
	}

	references, alreadyIndexed, err := c.indexReferences(args.Hash, args.Name, args.ParentHash)

	if err != nil {
		return err
	}

	if alreadyIndexed {
		return nil
	}

	log.Printf("Crawling file %s (%s)\n", args.Hash, args.Name)

	m := make(metadata)
	c.getMetadata(args, &m)
	if err != nil {
		return err
	}

	// Add previously found references now
	m["size"] = args.Size
	m["references"] = references

	err = c.Indexer.IndexItem("file", args.Hash, m)
	if err != nil {
		return err
	}

	log.Printf("Finished file %s", args.Hash)

	return nil
}
<|endoftext|>


<commit_before>// aiffinfo is a command line tool to gather information about aiff/aifc files.
package main

import (
	"encoding/hex"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/mattetti/audio/aiff"
)

var pathToParse = flag.String("path", ".", "Where to find aiff files")
var fileToParse = flag.String("file", "", "The wav file to analyze (instead of a path)")
var logChunks = flag.Bool("v", false, "Should the parser log chunks (not SSND)")

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: \n")
		flag.PrintDefaults()
	}

	flag.Parse()

	if *fileToParse != "" {
		analyze(*fileToParse)
		return
	}
	if err := filepath.Walk(*pathToParse, walkFn); err != nil {
		log.Fatal(err)
	}
}

func walkFn(path string, fi os.FileInfo, err error) (e error) {
	if err != nil {
		log.Fatal(err)
	}
	if fi.IsDir() {
		filepath.Walk(path, walkFolder)
		return
	}
	if (!strings.HasSuffix(fi.Name(), ".aif") && !strings.HasSuffix(fi.Name(), ".aiff")) || fi.IsDir() {
		return
	}
	analyze(path)
	return nil
}

func walkFolder(path string, fi os.FileInfo, err error) (e error) {
	if (!strings.HasSuffix(fi.Name(), ".aif") && !strings.HasSuffix(fi.Name(), ".aiff")) || fi.IsDir() {
		return
	}
	analyze(path)
	return nil
}

func analyze(path string) {
	fmt.Println(path)
	f, err := os.Open(path)
	if err != nil {
		panic(err)
	}
	defer f.Close()
	if *logChunks {
		ch := make(chan *aiff.Chunk)
		c := aiff.NewParser(f, ch)
		go func() {
			if err := c.Parse(); err != nil {
				panic(err)
			}
		}()

		for chunk := range ch {
			id := string(chunk.ID[:])
			fmt.Println(id, chunk.Size)
			if id != "SSND" {
				buf := make([]byte, chunk.Size)
				chunk.ReadBE(buf)
				fmt.Print(hex.Dump(buf))
			}
			chunk.Done()
		}
		return
	}
	c := aiff.New(f)
	if err := c.Parse(); err != nil {
		log.Fatalf("Can't parse the headers of %s - %s\n", path, err)
	}
	f.Seek(0, 0)
	sampleRate, data := aiff.ReadFrames(f)
	fmt.Println("sampleRate", sampleRate)
	fmt.Printf("data: %#v\n", data)
	fmt.Println(c)
}
<commit_msg>fix the aiffinfo tool (wasn't compiling due to API change)<commit_after>// aiffinfo is a command line tool to gather information about aiff/aifc files.
package main

import (
	"encoding/hex"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/mattetti/audio/aiff"
)

var pathToParse = flag.String("path", ".", "Where to find aiff files")
var fileToParse = flag.String("file", "", "The wav file to analyze (instead of a path)")
var logChunks = flag.Bool("v", false, "Should the parser log chunks (not SSND)")

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: \n")
		flag.PrintDefaults()
	}

	flag.Parse()

	if *fileToParse != "" {
		analyze(*fileToParse)
		return
	}
	if err := filepath.Walk(*pathToParse, walkFn); err != nil {
		log.Fatal(err)
	}
}

func walkFn(path string, fi os.FileInfo, err error) (e error) {
	if err != nil {
		log.Fatal(err)
	}
	if fi.IsDir() {
		filepath.Walk(path, walkFolder)
		return
	}
	if (!strings.HasSuffix(fi.Name(), ".aif") && !strings.HasSuffix(fi.Name(), ".aiff")) || fi.IsDir() {
		return
	}
	analyze(path)
	return nil
}

func walkFolder(path string, fi os.FileInfo, err error) (e error) {
	if (!strings.HasSuffix(fi.Name(), ".aif") && !strings.HasSuffix(fi.Name(), ".aiff")) || fi.IsDir() {
		return
	}
	analyze(path)
	return nil
}

func analyze(path string) {
	fmt.Println(path)
	f, err := os.Open(path)
	if err != nil {
		panic(err)
	}
	defer f.Close()
	if *logChunks {
		ch := make(chan *aiff.Chunk)
		c := aiff.NewParser(f, ch)
		go func() {
			if err := c.Parse(); err != nil {
				panic(err)
			}
		}()

		for chunk := range ch {
			id := string(chunk.ID[:])
			fmt.Println(id, chunk.Size)
			if id != "SSND" {
				buf := make([]byte, chunk.Size)
				chunk.ReadBE(buf)
				fmt.Print(hex.Dump(buf))
			}
			chunk.Done()
		}
		return
	}
	c := aiff.New(f)
	if err := c.Parse(); err != nil {
		log.Fatalf("Can't parse the headers of %s - %s\n", path, err)
	}
	f.Seek(0, 0)
	sampleRate, sampleSize, numChans, data := aiff.ReadFrames(f)
	fmt.Println("sampleRate", sampleRate)
	fmt.Println("sampleSize", sampleSize)
	fmt.Println("numChans", numChans)
	fmt.Printf("frames: %+v\n", data)
	fmt.Println(c)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"testing"
)

func fakeReader(size int) io.Reader {
	var i int
	var buf bytes.Buffer
	for i < size {
		i++
		s := fmt.Sprintf("%d,name%d,%d\n", i, i, size-i)
		_, _ = buf.WriteString(s)
	}
	return bytes.NewBuffer(buf.Bytes())
}

func TestScanString(t *testing.T) {
	f := fakeReader(10)
	ps := scanString(f)
	fmt.Println(ps)
}

func TestScanBytes(t *testing.T) {
	f := fakeReader(10)
	ps := scanBytes(f)
	fmt.Println(ps)
}

var resultPersons Persons

func BenchmarkScanString(b *testing.B) {
	f := fakeReader(10000)
	b.ResetTimer()
	var r Persons
	for n := 0; n < b.N; n++ {
		r = scanString(f)
	}
	resultPersons = r
}

func BenchmarkScanBytes(b *testing.B) {
	f := fakeReader(10000)
	b.ResetTimer()
	var r Persons
	for n := 0; n < b.N; n++ {
		r = scanBytes(f)
	}
	resultPersons = r
}
<commit_msg>file_scanner: clean TestScanString and TestScanBytes<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"testing"
)

func fakeReader(size int) io.Reader {
	var i int
	var buf bytes.Buffer
	for i < size {
		i++
		s := fmt.Sprintf("%d,name%d,%d\n", i, i, size-i)
		_, _ = buf.WriteString(s)
	}
	return bytes.NewBuffer(buf.Bytes())
}

func testScan(t *testing.T, fn func(io.Reader) Persons) {
	const fakeRows = 10
	f := fakeReader(fakeRows)
	ps := fn(f)
	fmt.Println(ps)
}

func TestScanString(t *testing.T) {
	testScan(t, scanString)
}

func TestScanBytes(t *testing.T) {
	testScan(t, scanBytes)
}

var resultPersons Persons

func BenchmarkScanString(b *testing.B) {
	f := fakeReader(10000)
	b.ResetTimer()
	var r Persons
	for n := 0; n < b.N; n++ {
		r = scanString(f)
	}
	resultPersons = r
}

func BenchmarkScanBytes(b *testing.B) {
	f := fakeReader(10000)
	b.ResetTimer()
	var r Persons
	for n := 0; n < b.N; n++ {
		r = scanBytes(f)
	}
	resultPersons = r
}
<|endoftext|>


<commit_before>package nexmo

import (
	"bytes"
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
)

// SMS represents the SMS API functions for sending text messages.
type SMS struct {
	client *Client
}

// SMS message types.
const (
	Text    = "text"
	Binary  = "binary"
	WAPPush = "wappush"
	Unicode = "unicode"
	VCal    = "vcal"
	VCard   = "vcard"
)

type MessageClass int

// SMS message classes.
const (
	// This type of SMS message is displayed on the mobile screen without being
	// saved in the message store or on the SIM card; unless explicitly saved
	// by the mobile user.
	Flash MessageClass = iota

	// This message is to be stored in the device memory or the SIM card
	// (depending on memory availability).
	Standard

	// This message class carries SIM card data. The SIM card data must be
	// successfully transferred prior to sending acknowledgment to the service
	// center. An error message is sent to the service center if this
	// transmission is not possible.
	SIMData

	// This message is forwarded from the receiving entity to an external
	// device. The delivery acknowledgment is sent to the service center
	// regardless of whether or not the message was forwarded to the external
	// device.
	Forward
)

var messageClassMap = map[MessageClass]string{
	Flash:    "flash",
	Standard: "standard",
	SIMData:  "SIM data",
	Forward:  "forward",
}

func (m MessageClass) String() string {
	return messageClassMap[m]
}

// Type SMSMessage defines a single SMS message.
type SMSMessage struct {
	ApiKey               string       `json:"api_key"`
	ApiSecret            string       `json:"api_secret"`
	From                 string       `json:"from"`
	To                   string       `json:"to"`
	Type                 string       `json:"type"`
	Text                 string       `json:"text,omitempty"`              // Optional.
	StatusReportRequired int          `json:"status-report-req,omitempty"` // Optional.
	ClientReference      string       `json:"client-ref,omitempty"`        // Optional.
	NetworkCode          string       `json:"network-code,omitempty"`      // Optional.
	VCard                string       `json:"vcrad,omitempty"`             // Optional.
	VCal                 string       `json:"vcal,omitempty"`              // Optional.
	TTL                  int          `json:"ttl,omitempty"`               // Optional.
	Class                MessageClass `json:"message-class,omitempty"`     // Optional.
	Body                 []byte       `json:"body,omitempty"`              // Required for Binary message.
	UDH                  []byte       `json:"udh,omitempty"`               // Required for Binary message.

	// The following is only for type=wappush

	Title    string `json:"title,omitempty"`    // Title shown to recipient
	URL      string `json:"url,omitempty"`      // WAP Push URL
	Validity int    `json:"validity,omitempty"` // Duration WAP Push is available in milliseconds
}

type ResponseCode int

func (c ResponseCode) String() string {
	return responseCodeMap[c]
}

const (
	ResponseSuccess = iota
	ResponseThrottled
	ResponseMissingParams
	ResponseInvalidParams
	ResponseInvalidCredentials
	ResponseInternalError
	ResponseInvalidMessage
	ResponseNumberBarred
	ResponsePartnerAcctBarred
	ResponsePartnerQuotaExceeded
	ResponseRESTNotEnabled
	ResponseMessageTooLong
	ResponseCommunicationFailed
	ResponseInvalidSignature
	ResponseInvalidSenderAddress
	ResponseInvalidTTL
	ResponseFacilityNotAllowed
	ResponseInvalidMessageClass
)

var responseCodeMap = map[ResponseCode]string{
	ResponseSuccess:              "Success",
	ResponseThrottled:            "Throttled",
	ResponseMissingParams:        "Missing params",
	ResponseInvalidParams:        "Invalid params",
	ResponseInvalidCredentials:   "Invalid credentials",
	ResponseInternalError:        "Internal error",
	ResponseInvalidMessage:       "Invalid message",
	ResponseNumberBarred:         "Number barred",
	ResponsePartnerAcctBarred:    "Partner account barred",
	ResponsePartnerQuotaExceeded: "Partner quota exceeded",
	ResponseRESTNotEnabled:       "Account not enabled for REST",
	ResponseMessageTooLong:       "Message too long",
	ResponseCommunicationFailed:  "Communication failed",
	ResponseInvalidSignature:     "Invalid signature",
	ResponseInvalidSenderAddress: "Invalid sender address",
	ResponseInvalidTTL:           "Invalid TTL",
	ResponseFacilityNotAllowed:   "Facility not allowed",
	ResponseInvalidMessageClass:  "Invalid message class",
}

// MessageReport is the "status report" for a single SMS sent via the Nexmo API
type MessageReport struct {
	Status           ResponseCode `json:"status,string"`
	MessageID        string       `json:"message-id"`
	To               string       `json:"to"`
	ClientReference  string       `json:"client-ref"`
	RemainingBalance string       `json:"remaining-balance"`
	MessagePrice     string       `json:"message-price"`
	Network          string       `json:"network"`
	ErrorText        string       `json:"error-text"`
}

// MessageResponse contains the response from Nexmo's API after we attempt to
// send any kind of message.
// It will contain one MessageReport for every 160 chars sent.
type MessageResponse struct {
	MessageCount int             `json:"message-count,string"`
	Messages     []MessageReport `json:"messages"`
}

// Send the message using the specified SMS client.
func (c *SMS) Send(msg *SMSMessage) (*MessageResponse, error) {
	if len(msg.From) <= 0 {
		return nil, errors.New("Invalid From field specified")
	}

	if len(msg.To) <= 0 {
		return nil, errors.New("Invalid To field specified")
	}

	if len(msg.ClientReference) > 40 {
		return nil, errors.New("Client reference too long")
	}

	var messageResponse *MessageResponse

	switch msg.Type {
	case Text:
	case Unicode:
		if len(msg.Text) <= 0 {
			return nil, errors.New("Invalid message text")
		}
	case Binary:
		if len(msg.UDH) == 0 || len(msg.Body) == 0 {
			return nil, errors.New("Invalid binary message")
		}

	case WAPPush:
		if len(msg.URL) == 0 || len(msg.Title) == 0 {
			return nil, errors.New("Invalid WAP Push parameters")
		}
	}

	if !c.client.useOauth {
		msg.ApiKey = c.client.apiKey
		msg.ApiSecret = c.client.apiSecret
	}

	client := &http.Client{}

	var r *http.Request
	buf, err := json.Marshal(msg)
	if err != nil {
		return nil, errors.New("Invalid message struct. Cannot convert to json.")
	}
	b := bytes.NewBuffer(buf)
	r, _ = http.NewRequest("POST", apiRoot+"/sms/json", b)

	r.Header.Add("Accept", "application/json")
	r.Header.Add("Content-Type", "application/json")

	resp, err := client.Do(r)

	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, _ := ioutil.ReadAll(resp.Body)

	err = json.Unmarshal(body, &messageResponse)
	if err != nil {
		return nil, err
	}
	return messageResponse, nil
}
<commit_msg>Hide API key and secret fields<commit_after>package nexmo

import (
	"bytes"
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
)

// SMS represents the SMS API functions for sending text messages.
type SMS struct {
	client *Client
}

// SMS message types.
const (
	Text    = "text"
	Binary  = "binary"
	WAPPush = "wappush"
	Unicode = "unicode"
	VCal    = "vcal"
	VCard   = "vcard"
)

type MessageClass int

// SMS message classes.
const (
	// This type of SMS message is displayed on the mobile screen without being
	// saved in the message store or on the SIM card; unless explicitly saved
	// by the mobile user.
	Flash MessageClass = iota

	// This message is to be stored in the device memory or the SIM card
	// (depending on memory availability).
	Standard

	// This message class carries SIM card data. The SIM card data must be
	// successfully transferred prior to sending acknowledgment to the service
	// center. An error message is sent to the service center if this
	// transmission is not possible.
	SIMData

	// This message is forwarded from the receiving entity to an external
	// device. The delivery acknowledgment is sent to the service center
	// regardless of whether or not the message was forwarded to the external
	// device.
	Forward
)

var messageClassMap = map[MessageClass]string{
	Flash:    "flash",
	Standard: "standard",
	SIMData:  "SIM data",
	Forward:  "forward",
}

func (m MessageClass) String() string {
	return messageClassMap[m]
}
func (m *SMSMessage) MarshalJSON() ([]byte, error) {
	return json.Marshal(struct {
		ApiKey    string `json:"api_key"`
		ApiSecret string `json:"api_secret"`
		SMSMessage
	}{
		ApiKey:     m.apiKey,
		ApiSecret:  m.apiSecret,
		SMSMessage: *m,
	})
}

// Type SMSMessage defines a single SMS message.
type SMSMessage struct {
	apiKey               string
	apiSecret            string
	From                 string       `json:"from"`
	To                   string       `json:"to"`
	Type                 string       `json:"type"`
	Text                 string       `json:"text,omitempty"`              // Optional.
	StatusReportRequired int          `json:"status-report-req,omitempty"` // Optional.
	ClientReference      string       `json:"client-ref,omitempty"`        // Optional.
	NetworkCode          string       `json:"network-code,omitempty"`      // Optional.
	VCard                string       `json:"vcrad,omitempty"`             // Optional.
	VCal                 string       `json:"vcal,omitempty"`              // Optional.
	TTL                  int          `json:"ttl,omitempty"`               // Optional.
	Class                MessageClass `json:"message-class,omitempty"`     // Optional.
	Body                 []byte       `json:"body,omitempty"`              // Required for Binary message.
	UDH                  []byte       `json:"udh,omitempty"`               // Required for Binary message.

	// The following is only for type=wappush

	Title    string `json:"title,omitempty"`    // Title shown to recipient
	URL      string `json:"url,omitempty"`      // WAP Push URL
	Validity int    `json:"validity,omitempty"` // Duration WAP Push is available in milliseconds
}

type ResponseCode int

func (c ResponseCode) String() string {
	return responseCodeMap[c]
}

const (
	ResponseSuccess = iota
	ResponseThrottled
	ResponseMissingParams
	ResponseInvalidParams
	ResponseInvalidCredentials
	ResponseInternalError
	ResponseInvalidMessage
	ResponseNumberBarred
	ResponsePartnerAcctBarred
	ResponsePartnerQuotaExceeded
	ResponseRESTNotEnabled
	ResponseMessageTooLong
	ResponseCommunicationFailed
	ResponseInvalidSignature
	ResponseInvalidSenderAddress
	ResponseInvalidTTL
	ResponseFacilityNotAllowed
	ResponseInvalidMessageClass
)

var responseCodeMap = map[ResponseCode]string{
	ResponseSuccess:              "Success",
	ResponseThrottled:            "Throttled",
	ResponseMissingParams:        "Missing params",
	ResponseInvalidParams:        "Invalid params",
	ResponseInvalidCredentials:   "Invalid credentials",
	ResponseInternalError:        "Internal error",
	ResponseInvalidMessage:       "Invalid message",
	ResponseNumberBarred:         "Number barred",
	ResponsePartnerAcctBarred:    "Partner account barred",
	ResponsePartnerQuotaExceeded: "Partner quota exceeded",
	ResponseRESTNotEnabled:       "Account not enabled for REST",
	ResponseMessageTooLong:       "Message too long",
	ResponseCommunicationFailed:  "Communication failed",
	ResponseInvalidSignature:     "Invalid signature",
	ResponseInvalidSenderAddress: "Invalid sender address",
	ResponseInvalidTTL:           "Invalid TTL",
	ResponseFacilityNotAllowed:   "Facility not allowed",
	ResponseInvalidMessageClass:  "Invalid message class",
}

// MessageReport is the "status report" for a single SMS sent via the Nexmo API
type MessageReport struct {
	Status           ResponseCode `json:"status,string"`
	MessageID        string       `json:"message-id"`
	To               string       `json:"to"`
	ClientReference  string       `json:"client-ref"`
	RemainingBalance string       `json:"remaining-balance"`
	MessagePrice     string       `json:"message-price"`
	Network          string       `json:"network"`
	ErrorText        string       `json:"error-text"`
}

// MessageResponse contains the response from Nexmo's API after we attempt to
// send any kind of message.
// It will contain one MessageReport for every 160 chars sent.
type MessageResponse struct {
	MessageCount int             `json:"message-count,string"`
	Messages     []MessageReport `json:"messages"`
}

// Send the message using the specified SMS client.
func (c *SMS) Send(msg *SMSMessage) (*MessageResponse, error) {
	if len(msg.From) <= 0 {
		return nil, errors.New("Invalid From field specified")
	}

	if len(msg.To) <= 0 {
		return nil, errors.New("Invalid To field specified")
	}

	if len(msg.ClientReference) > 40 {
		return nil, errors.New("Client reference too long")
	}

	var messageResponse *MessageResponse

	switch msg.Type {
	case Text:
	case Unicode:
		if len(msg.Text) <= 0 {
			return nil, errors.New("Invalid message text")
		}
	case Binary:
		if len(msg.UDH) == 0 || len(msg.Body) == 0 {
			return nil, errors.New("Invalid binary message")
		}

	case WAPPush:
		if len(msg.URL) == 0 || len(msg.Title) == 0 {
			return nil, errors.New("Invalid WAP Push parameters")
		}
	}
	if !c.client.useOauth {
		msg.apiKey = c.client.apiKey
		msg.apiSecret = c.client.apiSecret
	}

	client := &http.Client{}

	var r *http.Request
	buf, err := json.Marshal(msg)
	if err != nil {
		return nil, errors.New("Invalid message struct. Cannot convert to json.")
	}
	b := bytes.NewBuffer(buf)
	r, _ = http.NewRequest("POST", apiRoot+"/sms/json", b)

	r.Header.Add("Accept", "application/json")
	r.Header.Add("Content-Type", "application/json")

	resp, err := client.Do(r)

	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, _ := ioutil.ReadAll(resp.Body)

	err = json.Unmarshal(body, &messageResponse)
	if err != nil {
		return nil, err
	}
	return messageResponse, nil
}
<|endoftext|>


<commit_before>package config

import (
	"github.com/materials-commons/config/cfg"
	"github.com/materials-commons/config/handler"
	"time"
)

// Store configuration in environment as specified for 12 Factor Applications:
// http://12factor.net/config
var TwelveFactor = handler.Env()

// Store configuration in environment, but allow overrides, either by the
// application setting them internally as defaults, or setting them from
// the command line. See http://12factor.net/config. Overrides are an
// extension to the specification.
var TwelveFactorOverride = handler.Multi(handler.Map(), handler.Env())

var std Configer

// Init initializes the standard Configer using the specified handler. The
// standard configer is a global config that can be conveniently accessed
// from the config package.
func Init(handler cfg.Handler) error {
	std = New(handler)
	return std.Init()
}

// Get gets a key from the standard Configer.
func Get(key string, args ...interface{}) (interface{}, error) {
	return std.Get(key, args...)
}

// GetInt gets an integer key from the standard Configer.
func GetInt(key string, args ...interface{}) (int, error) {
	return std.GetInt(key, args...)
}

// GetString gets an string key from the standard Configer.
func GetString(key string, args ...interface{}) (string, error) {
	return std.GetString(key, args...)
}

// GetTime gets an time key from the standard Configer.
func GetTime(key string, args ...interface{}) (time.Time, error) {
	return std.GetTime(key, args...)
}

// GetBool gets an bool key from the standard Configer.
func GetBool(key string, args ...interface{}) (bool, error) {
	return std.GetBool(key, args...)
}

// Set sets key to value in the standard Configer.
func Set(key string, value interface{}, args ...interface{}) error {
	return std.Set(key, value, args...)
}
<commit_msg>Rename TwelveFactorOverride to TwelveFactorWithOverride. Make the override handler thread safe by wrapping it with handler.Sync.<commit_after>package config

import (
	"github.com/materials-commons/config/cfg"
	"github.com/materials-commons/config/handler"
	"time"
)

// Store configuration in environment as specified for 12 Factor Applications:
// http://12factor.net/config
var TwelveFactor = handler.Env()

// Store configuration in environment, but allow overrides, either by the
// application setting them internally as defaults, or setting them from
// the command line. See http://12factor.net/config. Overrides are an
// extension to the specification. This handler is thread safe and can
// safely be used across multiple go routines.
var TwelveFactorWithOverride = handler.Multi(handler.Sync(handler.Map()), handler.Env())

var std Configer

// Init initializes the standard Configer using the specified handler. The
// standard configer is a global config that can be conveniently accessed
// from the config package.
func Init(handler cfg.Handler) error {
	std = New(handler)
	return std.Init()
}

// Get gets a key from the standard Configer.
func Get(key string, args ...interface{}) (interface{}, error) {
	return std.Get(key, args...)
}

// GetInt gets an integer key from the standard Configer.
func GetInt(key string, args ...interface{}) (int, error) {
	return std.GetInt(key, args...)
}

// GetString gets an string key from the standard Configer.
func GetString(key string, args ...interface{}) (string, error) {
	return std.GetString(key, args...)
}

// GetTime gets an time key from the standard Configer.
func GetTime(key string, args ...interface{}) (time.Time, error) {
	return std.GetTime(key, args...)
}

// GetBool gets an bool key from the standard Configer.
func GetBool(key string, args ...interface{}) (bool, error) {
	return std.GetBool(key, args...)
}

// Set sets key to value in the standard Configer.
func Set(key string, value interface{}, args ...interface{}) error {
	return std.Set(key, value, args...)
}
<|endoftext|>


<commit_before>package svg

import (
	"encoding/xml"
	"fmt"
	"io"
	"strconv"

	mt "github.com/rustyoz/Mtransform"
)

// DrawingInstructionParser allow getting segments and drawing
// instructions from them. All SVG elements should implement this
// interface.
type DrawingInstructionParser interface {
	ParseDrawingInstructions() (chan *DrawingInstruction, chan error)
}

// Tuple is an X,Y coordinate
type Tuple [2]float64

// Svg represents an SVG file containing at least a top level group or a
// number of Paths
type Svg struct {
	Title        string  `xml:"title"`
	Groups       []Group `xml:"g"`
	Elements     []DrawingInstructionParser
	Name         string
	Transform    *mt.Transform
	scale        float64
	instructions chan *DrawingInstruction
	errors       chan error
	segments     chan Segment
}

// Group represents an SVG group (usually located in a 'g' XML element)
type Group struct {
	ID              string
	Stroke          string
	StrokeWidth     int32
	Fill            string
	FillRule        string
	Elements        []DrawingInstructionParser
	TransformString string
	Transform       *mt.Transform // row, column
	Parent          *Group
	Owner           *Svg
	instructions    chan *DrawingInstruction
	errors          chan error
	segments        chan Segment
}

// ParseDrawingInstructions implements the DrawingInstructionParser interface
//
// This method makes it easier to get all the drawing instructions.
func (g *Group) ParseDrawingInstructions() (chan *DrawingInstruction, chan error) {
	g.instructions = make(chan *DrawingInstruction, 100)
	g.errors = make(chan error, 100)

	go func() {
		defer close(g.instructions)
		defer close(g.errors)
		for _, e := range g.Elements {
			instrs, errs := e.ParseDrawingInstructions()
			go func() {
				for er := range errs {
					g.errors <- er
				}
			}()
			for is := range instrs {
				g.instructions <- is
			}
		}
	}()

	return g.instructions, g.errors
}

// UnmarshalXML implements the encoding.xml.Unmarshaler interface
func (g *Group) UnmarshalXML(decoder *xml.Decoder, start xml.StartElement) error {
	for _, attr := range start.Attr {
		switch attr.Name.Local {
		case "id":
			g.ID = attr.Value
		case "stroke":
			g.Stroke = attr.Value
		case "stroke-width":
			intValue, err := strconv.ParseInt(attr.Value, 10, 32)
			if err != nil {
				return err
			}
			g.StrokeWidth = int32(intValue)
		case "fill":
			g.Fill = attr.Value
		case "fill-rule":
			g.FillRule = attr.Value
		case "transform":
			g.TransformString = attr.Value
			t, err := parseTransform(g.TransformString)
			if err != nil {
				fmt.Println(err)
			}
			g.Transform = &t
		}
	}

	for {
		token, err := decoder.Token()
		if err != nil {
			return err
		}

		switch tok := token.(type) {
		case xml.StartElement:
			var elementStruct DrawingInstructionParser

			switch tok.Name.Local {
			case "g":
				elementStruct = &Group{Parent: g, Owner: g.Owner, Transform: mt.NewTransform()}
			case "rect":
				elementStruct = &Rect{group: g}
			case "circle":
				elementStruct = &Circle{group: g}
			case "path":
				elementStruct = &Path{group: g, StrokeWidth: float64(g.StrokeWidth), Stroke: &g.Stroke, Fill: &g.Fill}

			}

			if err = decoder.DecodeElement(elementStruct, &tok); err != nil {
				return fmt.Errorf("error decoding element of Group: %s", err)
			}
			g.Elements = append(g.Elements, elementStruct)

		case xml.EndElement:
			return nil
		}
	}
}

// ParseDrawingInstructions implements the DrawingInstructionParser interface
//
// This method makes it easier to get all the drawing instructions.
func (s *Svg) ParseDrawingInstructions() (chan *DrawingInstruction, chan error) {
	s.instructions = make(chan *DrawingInstruction, 100)
	s.errors = make(chan error, 100)

	go func() {
		var elecount int
		defer close(s.instructions)
		defer close(s.errors)
		for _, e := range s.Elements {
			elecount++
			instrs, errs := e.ParseDrawingInstructions()
			go func(count int) {
				for er := range errs {
					s.errors <- fmt.Errorf("error when parsing element nr. %d: %s", count, er)
				}
			}(elecount)

			for is := range instrs {
				s.instructions <- is
			}
		}

		for _, g := range s.Groups {
			instrs, errs := g.ParseDrawingInstructions()
			go func() {
				for er := range errs {
					g.errors <- er
				}
			}()
			for is := range instrs {
				s.instructions <- is
			}
		}
	}()

	return s.instructions, s.errors
}

// UnmarshalXML implements the encoding.xml.Unmarshaler interface
func (s *Svg) UnmarshalXML(decoder *xml.Decoder, start xml.StartElement) error {
	for {
		token, err := decoder.Token()
		if err != nil {
			return err
		}

		switch tok := token.(type) {
		case xml.StartElement:
			var dip DrawingInstructionParser

			switch tok.Name.Local {
			case "g":
				g := &Group{Owner: s, Transform: mt.NewTransform()}
				if err = decoder.DecodeElement(g, &tok); err != nil {
					return fmt.Errorf("error decoding group element within SVG struct: %s", err)
				}
				s.Groups = append(s.Groups, *g)
				continue
			case "rect":
				dip = &Rect{}
			case "circle":
				dip = &Circle{}
			case "path":
				dip = &Path{}

			default:
				continue
			}

			if err = decoder.DecodeElement(dip, &tok); err != nil {
				return fmt.Errorf("error decoding element of SVG struct: %s", err)
			}

			s.Elements = append(s.Elements, dip)

		case xml.EndElement:
			if tok.Name.Local == "svg" {
				return nil
			}
		}
	}
}

// ParseSvg parses an SVG string into an SVG struct
func ParseSvg(str string, name string, scale float64) (*Svg, error) {
	var svg Svg
	svg.Name = name
	svg.Transform = mt.NewTransform()
	if scale > 0 {
		svg.Transform.Scale(scale, scale)
		svg.scale = scale
	}
	if scale < 0 {
		svg.Transform.Scale(1.0/-scale, 1.0/-scale)
		svg.scale = 1.0 / -scale
	}

	err := xml.Unmarshal([]byte(str), &svg)
	if err != nil {
		return nil, fmt.Errorf("ParseSvg Error: %v", err)
	}
	fmt.Println(len(svg.Groups))
	for i := range svg.Groups {
		svg.Groups[i].SetOwner(&svg)
		if svg.Groups[i].Transform == nil {
			svg.Groups[i].Transform = mt.NewTransform()
		}
	}
	return &svg, nil
}

// ParseSvgFromReader parses an SVG struct from an io.Reader
func ParseSvgFromReader(r io.Reader, name string, scale float64) (*Svg, error) {
	var svg Svg
	svg.Name = name
	svg.Transform = mt.NewTransform()
	if scale > 0 {
		svg.Transform.Scale(scale, scale)
		svg.scale = scale
	}
	if scale < 0 {
		svg.Transform.Scale(1.0/-scale, 1.0/-scale)
		svg.scale = 1.0 / -scale
	}

	if err := xml.NewDecoder(r).Decode(&svg); err != nil {
		return nil, fmt.Errorf("ParseSvg Error: %v", err)
	}

	fmt.Println(len(svg.Groups))

	for i := range svg.Groups {
		svg.Groups[i].SetOwner(&svg)
		if svg.Groups[i].Transform == nil {
			svg.Groups[i].Transform = mt.NewTransform()
		}
	}
	return &svg, nil
}

// SetOwner sets the owner of a SVG Group
func (g *Group) SetOwner(svg *Svg) {
	g.Owner = svg
	for _, gn := range g.Elements {
		switch gn.(type) {
		case *Group:
			gn.(*Group).Owner = g.Owner
			gn.(*Group).SetOwner(svg)
		case *Path:
			gn.(*Path).group = g
		}
	}
}
<commit_msg>svg: make viewBox values accessible<commit_after>package svg

import (
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"strconv"
	"strings"

	mt "github.com/rustyoz/Mtransform"
)

// DrawingInstructionParser allow getting segments and drawing
// instructions from them. All SVG elements should implement this
// interface.
type DrawingInstructionParser interface {
	ParseDrawingInstructions() (chan *DrawingInstruction, chan error)
}

// Tuple is an X,Y coordinate
type Tuple [2]float64

// Svg represents an SVG file containing at least a top level group or a
// number of Paths
type Svg struct {
	Title        string  `xml:"title"`
	Groups       []Group `xml:"g"`
	ViewBox      string  `xml:"viewBox,attr"`
	Elements     []DrawingInstructionParser
	Name         string
	Transform    *mt.Transform
	scale        float64
	instructions chan *DrawingInstruction
	errors       chan error
	segments     chan Segment
}

// Group represents an SVG group (usually located in a 'g' XML element)
type Group struct {
	ID              string
	Stroke          string
	StrokeWidth     int32
	Fill            string
	FillRule        string
	Elements        []DrawingInstructionParser
	TransformString string
	Transform       *mt.Transform // row, column
	Parent          *Group
	Owner           *Svg
	instructions    chan *DrawingInstruction
	errors          chan error
	segments        chan Segment
}

// ParseDrawingInstructions implements the DrawingInstructionParser interface
//
// This method makes it easier to get all the drawing instructions.
func (g *Group) ParseDrawingInstructions() (chan *DrawingInstruction, chan error) {
	g.instructions = make(chan *DrawingInstruction, 100)
	g.errors = make(chan error, 100)

	go func() {
		defer close(g.instructions)
		defer close(g.errors)
		for _, e := range g.Elements {
			instrs, errs := e.ParseDrawingInstructions()
			go func() {
				for er := range errs {
					g.errors <- er
				}
			}()
			for is := range instrs {
				g.instructions <- is
			}
		}
	}()

	return g.instructions, g.errors
}

// UnmarshalXML implements the encoding.xml.Unmarshaler interface
func (g *Group) UnmarshalXML(decoder *xml.Decoder, start xml.StartElement) error {
	for _, attr := range start.Attr {
		switch attr.Name.Local {
		case "id":
			g.ID = attr.Value
		case "stroke":
			g.Stroke = attr.Value
		case "stroke-width":
			intValue, err := strconv.ParseInt(attr.Value, 10, 32)
			if err != nil {
				return err
			}
			g.StrokeWidth = int32(intValue)
		case "fill":
			g.Fill = attr.Value
		case "fill-rule":
			g.FillRule = attr.Value
		case "transform":
			g.TransformString = attr.Value
			t, err := parseTransform(g.TransformString)
			if err != nil {
				fmt.Println(err)
			}
			g.Transform = &t
		}
	}

	for {
		token, err := decoder.Token()
		if err != nil {
			return err
		}

		switch tok := token.(type) {
		case xml.StartElement:
			var elementStruct DrawingInstructionParser

			switch tok.Name.Local {
			case "g":
				elementStruct = &Group{Parent: g, Owner: g.Owner, Transform: mt.NewTransform()}
			case "rect":
				elementStruct = &Rect{group: g}
			case "circle":
				elementStruct = &Circle{group: g}
			case "path":
				elementStruct = &Path{group: g, StrokeWidth: float64(g.StrokeWidth), Stroke: &g.Stroke, Fill: &g.Fill}

			}

			if err = decoder.DecodeElement(elementStruct, &tok); err != nil {
				return fmt.Errorf("error decoding element of Group: %s", err)
			}
			g.Elements = append(g.Elements, elementStruct)

		case xml.EndElement:
			return nil
		}
	}
}

// ParseDrawingInstructions implements the DrawingInstructionParser interface
//
// This method makes it easier to get all the drawing instructions.
func (s *Svg) ParseDrawingInstructions() (chan *DrawingInstruction, chan error) {
	s.instructions = make(chan *DrawingInstruction, 100)
	s.errors = make(chan error, 100)

	go func() {
		var elecount int
		defer close(s.instructions)
		defer close(s.errors)
		for _, e := range s.Elements {
			elecount++
			instrs, errs := e.ParseDrawingInstructions()
			go func(count int) {
				for er := range errs {
					s.errors <- fmt.Errorf("error when parsing element nr. %d: %s", count, er)
				}
			}(elecount)

			for is := range instrs {
				s.instructions <- is
			}
		}

		for _, g := range s.Groups {
			instrs, errs := g.ParseDrawingInstructions()
			go func() {
				for er := range errs {
					g.errors <- er
				}
			}()
			for is := range instrs {
				s.instructions <- is
			}
		}
	}()

	return s.instructions, s.errors
}

// UnmarshalXML implements the encoding.xml.Unmarshaler interface
func (s *Svg) UnmarshalXML(decoder *xml.Decoder, start xml.StartElement) error {
	for {
		for _, attr := range start.Attr {
			if attr.Name.Local == "viewBox" {
				s.ViewBox = attr.Value
				break
			}
		}

		token, err := decoder.Token()
		if err != nil {
			return err
		}

		switch tok := token.(type) {
		case xml.StartElement:
			var dip DrawingInstructionParser

			switch tok.Name.Local {
			case "g":
				g := &Group{Owner: s, Transform: mt.NewTransform()}
				if err = decoder.DecodeElement(g, &tok); err != nil {
					return fmt.Errorf("error decoding group element within SVG struct: %s", err)
				}
				s.Groups = append(s.Groups, *g)
				continue
			case "rect":
				dip = &Rect{}
			case "circle":
				dip = &Circle{}
			case "path":
				dip = &Path{}

			default:
				continue
			}

			if err = decoder.DecodeElement(dip, &tok); err != nil {
				return fmt.Errorf("error decoding element of SVG struct: %s", err)
			}

			s.Elements = append(s.Elements, dip)

		case xml.EndElement:
			if tok.Name.Local == "svg" {
				return nil
			}
		}
	}
}

// ParseSvg parses an SVG string into an SVG struct
func ParseSvg(str string, name string, scale float64) (*Svg, error) {
	var svg Svg
	svg.Name = name
	svg.Transform = mt.NewTransform()
	if scale > 0 {
		svg.Transform.Scale(scale, scale)
		svg.scale = scale
	}
	if scale < 0 {
		svg.Transform.Scale(1.0/-scale, 1.0/-scale)
		svg.scale = 1.0 / -scale
	}

	err := xml.Unmarshal([]byte(str), &svg)
	if err != nil {
		return nil, fmt.Errorf("ParseSvg Error: %v", err)
	}
	fmt.Println(len(svg.Groups))
	for i := range svg.Groups {
		svg.Groups[i].SetOwner(&svg)
		if svg.Groups[i].Transform == nil {
			svg.Groups[i].Transform = mt.NewTransform()
		}
	}
	return &svg, nil
}

// ParseSvgFromReader parses an SVG struct from an io.Reader
func ParseSvgFromReader(r io.Reader, name string, scale float64) (*Svg, error) {
	var svg Svg
	svg.Name = name
	svg.Transform = mt.NewTransform()
	if scale > 0 {
		svg.Transform.Scale(scale, scale)
		svg.scale = scale
	}
	if scale < 0 {
		svg.Transform.Scale(1.0/-scale, 1.0/-scale)
		svg.scale = 1.0 / -scale
	}

	if err := xml.NewDecoder(r).Decode(&svg); err != nil {
		return nil, fmt.Errorf("ParseSvg Error: %v", err)
	}

	fmt.Println(len(svg.Groups))

	for i := range svg.Groups {
		svg.Groups[i].SetOwner(&svg)
		if svg.Groups[i].Transform == nil {
			svg.Groups[i].Transform = mt.NewTransform()
		}
	}
	return &svg, nil
}

// ViewBoxValues returns all the numerical values in the viewBox
// attribute.
func (s *Svg) ViewBoxValues() ([]float64, error) {
	var vals []float64

	if s.ViewBox == "" {
		return vals, errors.New("viewBox attribute is empty")
	}

	split := strings.Split(s.ViewBox, " ")

	for _, val := range split {
		ival, err := strconv.ParseFloat(val, 64)
		if err != nil {
			return vals, err
		}
		vals = append(vals, ival)
	}

	return vals, nil
}

// SetOwner sets the owner of a SVG Group
func (g *Group) SetOwner(svg *Svg) {
	g.Owner = svg
	for _, gn := range g.Elements {
		switch gn.(type) {
		case *Group:
			gn.(*Group).Owner = g.Owner
			gn.(*Group).SetOwner(svg)
		case *Path:
			gn.(*Path).group = g
		}
	}
}
<|endoftext|>


<commit_before>package kubernetes

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

// FilesFor returns the resource definition files in path (or any subdirectory)
// that are responsible for driving the given namespace/service. It presumes
// kubeservice is available in the PWD or PATH.
func FilesFor(path, namespace, service string) (filenames []string, err error) {
	bin, err := func() (string, error) {
		cwd, err := os.Getwd()
		if err != nil {
			return "", err
		}
		localBin := filepath.Join(cwd, "kubeservice")
		if _, err := os.Stat(localBin); err == nil {
			return localBin, nil
		}
		if pathBin, err := exec.LookPath("kubeservice"); err == nil {
			return pathBin, nil
		}
		return "", errors.New("kubeservice not found")
	}()
	if err != nil {
		return nil, err
	}

	var candidates []string
	filepath.Walk(path, func(target string, fi os.FileInfo, err error) error {
		if fi.IsDir() {
			return nil
		}
		if ext := filepath.Ext(target); ext == ".yaml" || ext == ".yml" {
			candidates = append(candidates, target)
		}
		return nil
	})

	tgt := fmt.Sprintf("%s/%s", namespace, service)
	var winners []string
	for _, file := range candidates {
		var stdout, stderr bytes.Buffer
		cmd := exec.Command(bin, "./"+filepath.Base(file)) // due to bug (?) in kubeservice
		cmd.Dir = filepath.Dir(file)
		cmd.Stdout = &stdout
		cmd.Stderr = &stderr
		fmt.Fprintf(os.Stderr, "Running: %#+v\n", cmd)
		fmt.Fprintf(os.Stderr, "Dir: %s\n", cmd.Dir)
		fmt.Fprintf(os.Stderr, "Args: %v\n", cmd.Args)
		if err := cmd.Run(); err != nil {
			fmt.Fprintf(os.Stderr, stdout.String()+"\n")
			fmt.Fprintf(os.Stderr, stderr.String()+"\n")
			fmt.Fprintf(os.Stderr, "kubeservice FAILED: %v\n", err)
			continue
		}
		out := strings.TrimSpace(stdout.String())
		fmt.Fprintf(os.Stderr, "kubeservice SUCCESS: %s -> %s\n", file, out)
		if out == tgt { // kubeservice output is "namespace/service", same as ServiceID
			winners = append(winners, file)
		}
	}

	if len(winners) <= 0 {
		return nil, fmt.Errorf("no file found for namespace %s service %s", namespace, service)
	}
	return winners, nil
}
<commit_msg>Remove unnecessary debug output<commit_after>package kubernetes

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

// FilesFor returns the resource definition files in path (or any subdirectory)
// that are responsible for driving the given namespace/service. It presumes
// kubeservice is available in the PWD or PATH.
func FilesFor(path, namespace, service string) (filenames []string, err error) {
	bin, err := func() (string, error) {
		cwd, err := os.Getwd()
		if err != nil {
			return "", err
		}
		localBin := filepath.Join(cwd, "kubeservice")
		if _, err := os.Stat(localBin); err == nil {
			return localBin, nil
		}
		if pathBin, err := exec.LookPath("kubeservice"); err == nil {
			return pathBin, nil
		}
		return "", errors.New("kubeservice not found")
	}()
	if err != nil {
		return nil, err
	}

	var candidates []string
	filepath.Walk(path, func(target string, fi os.FileInfo, err error) error {
		if fi.IsDir() {
			return nil
		}
		if ext := filepath.Ext(target); ext == ".yaml" || ext == ".yml" {
			candidates = append(candidates, target)
		}
		return nil
	})

	tgt := fmt.Sprintf("%s/%s", namespace, service)
	var winners []string
	for _, file := range candidates {
		var stdout, stderr bytes.Buffer
		cmd := exec.Command(bin, "./"+filepath.Base(file)) // due to bug (?) in kubeservice
		cmd.Dir = filepath.Dir(file)
		cmd.Stdout = &stdout
		cmd.Stderr = &stderr
		fmt.Fprintf(os.Stderr, "Args: %v\n", cmd.Args)
		fmt.Fprintf(os.Stderr, "Dir: %s\n", cmd.Dir)
		if err := cmd.Run(); err != nil {
			fmt.Fprintf(os.Stderr, stdout.String()+"\n")
			fmt.Fprintf(os.Stderr, stderr.String()+"\n")
			fmt.Fprintf(os.Stderr, "kubeservice FAILED: %v\n", err)
			continue
		}
		out := strings.TrimSpace(stdout.String())
		fmt.Fprintf(os.Stderr, "kubeservice SUCCESS: %s -> %s\n", file, out)
		if out == tgt { // kubeservice output is "namespace/service", same as ServiceID
			winners = append(winners, file)
		}
	}

	if len(winners) <= 0 {
		return nil, fmt.Errorf("no file found for namespace %s service %s", namespace, service)
	}
	return winners, nil
}
<|endoftext|>