<commit_before>// Package aws collects AWS-specific configuration.
package aws

import (
	"github.com/aws/aws-sdk-go/aws/session"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	ccaws "github.com/openshift/cloud-credential-operator/pkg/aws"
	credvalidator "github.com/openshift/cloud-credential-operator/pkg/controller/utils"
)

var installPermissions = []string{
	// EC2 related perms
	"ec2:AllocateAddress",
	"ec2:AssociateAddress",
	"ec2:AssociateDhcpOptions",
	"ec2:AssociateRouteTable",
	"ec2:AttachInternetGateway",
	"ec2:AuthorizeSecurityGroupEgress",
	"ec2:AuthorizeSecurityGroupIngress",
	"ec2:CreateDhcpOptions",
	"ec2:CreateInternetGateway",
	"ec2:CreateNatGateway",
	"ec2:CreateRoute",
	"ec2:CreateRouteTable",
	"ec2:CreateSecurityGroup",
	"ec2:CreateSubnet",
	"ec2:CreateTags",
	"ec2:CreateVpc",
	"ec2:CreateVpcEndpoint",
	"ec2:CreateVolume",
	"ec2:DescribeAccountAttributes",
	"ec2:DescribeAddresses",
	"ec2:DescribeAvailabilityZones",
	"ec2:DescribeDhcpOptions",
	"ec2:DescribeImages",
	"ec2:DescribeInstanceAttribute",
	"ec2:DescribeInstanceCreditSpecifications",
	"ec2:DescribeInstances",
	"ec2:DescribeInternetGateways",
	"ec2:DescribeKeyPairs",
	"ec2:DescribeNatGateways",
	"ec2:DescribeNetworkAcls",
	"ec2:DescribePrefixLists",
	"ec2:DescribeRegions",
	"ec2:DescribeRouteTables",
	"ec2:DescribeSecurityGroups",
	"ec2:DescribeSubnets",
	"ec2:DescribeTags",
	"ec2:DescribeVpcEndpoints",
	"ec2:DescribeVpcs",
	"ec2:DescribeVpcAttribute",
	"ec2:DescribeVolumes",
	"ec2:DescribeVpcClassicLink",
	"ec2:DescribeVpcClassicLinkDnsSupport",
	"ec2:ModifyInstanceAttribute",
	"ec2:ModifySubnetAttribute",
	"ec2:ModifyVpcAttribute",
	"ec2:RevokeSecurityGroupEgress",
	"ec2:RunInstances",
	"ec2:TerminateInstances",
	"ec2:DeleteDhcpOptions",
	"ec2:DeleteRoute",
	"ec2:RevokeSecurityGroupIngress",
	"ec2:DisassociateRouteTable",
	"ec2:ReplaceRouteTableAssociation",
	"ec2:DeleteRouteTable",
	"ec2:DeleteSubnet",
	"ec2:DescribeNetworkInterfaces",
	"ec2:ModifyNetworkInterfaceAttribute",
	"ec2:DeleteNatGateway",
	"ec2:DeleteSecurityGroup",
	"ec2:DetachInternetGateway",
	"ec2:DeleteInternetGateway",
	"ec2:ReleaseAddress",
	"ec2:DeleteVpc",

	// ELB related perms
	"elasticloadbalancing:AddTags",
	"elasticloadbalancing:ApplySecurityGroupsToLoadBalancer",
	"elasticloadbalancing:AttachLoadBalancerToSubnets",
	"elasticloadbalancing:CreateListener",
	"elasticloadbalancing:CreateLoadBalancer",
	"elasticloadbalancing:CreateLoadBalancerListeners",
	"elasticloadbalancing:CreateTargetGroup",
	"elasticloadbalancing:ConfigureHealthCheck",
	"elasticloadbalancing:DeleteLoadBalancer",
	"elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
	"elasticloadbalancing:DeregisterTargets",
	"elasticloadbalancing:DescribeInstanceHealth",
	"elasticloadbalancing:DescribeListeners",
	"elasticloadbalancing:DescribeLoadBalancers",
	"elasticloadbalancing:DescribeLoadBalancerAttributes",
	"elasticloadbalancing:DescribeTags",
	"elasticloadbalancing:DescribeTargetGroupAttributes",
	"elasticloadbalancing:DescribeTargetHealth",
	"elasticloadbalancing:ModifyLoadBalancerAttributes",
	"elasticloadbalancing:ModifyTargetGroup",
	"elasticloadbalancing:ModifyTargetGroupAttributes",
	"elasticloadbalancing:RegisterTargets",
	"elasticloadbalancing:RegisterInstancesWithLoadBalancer",
	"elasticloadbalancing:SetLoadBalancerPoliciesOfListener",

	// IAM related perms
	"iam:AddRoleToInstanceProfile",
	"iam:CreateInstanceProfile",
	"iam:CreateRole",
	"iam:DeleteInstanceProfile",
	"iam:DeleteRole",
	"iam:DeleteRolePolicy",
	"iam:GetInstanceProfile",
	"iam:GetRole",
	"iam:GetRolePolicy",
	"iam:GetUser",
	"iam:ListInstanceProfilesForRole",
	"iam:ListRoles",
	"iam:ListUsers",
	"iam:PassRole",
	"iam:PutRolePolicy",
	"iam:RemoveRoleFromInstanceProfile",
	"iam:SimulatePrincipalPolicy",
	"iam:TagRole",

	// Route53 related perms
	"route53:ChangeResourceRecordSets",
	"route53:ChangeTagsForResource",
	"route53:GetChange",
	"route53:GetHostedZone",
	"route53:CreateHostedZone",
	"route53:DeleteHostedZone",
	"route53:ListHostedZones",
	"route53:ListHostedZonesByName",
	"route53:ListResourceRecordSets",
	"route53:ListTagsForResource",
	"route53:UpdateHostedZoneComment",

	// S3 related perms
	"s3:CreateBucket",
	"s3:ListBucket",
	"s3:GetBucketCors",
	"s3:GetBucketWebsite",
	"s3:GetBucketVersioning",
	"s3:GetAccelerateConfiguration",
	"s3:GetEncryptionConfiguration",
	"s3:GetBucketRequestPayment",
	"s3:GetBucketLogging",
	"s3:GetLifecycleConfiguration",
	"s3:GetBucketReplication",
	"s3:GetReplicationConfiguration",
	"s3:GetBucketLocation",
	"s3:GetBucketTagging",
	"s3:DeleteBucket",
	"s3:PutBucketAcl",
	"s3:PutBucketTagging",
	"s3:PutEncryptionConfiguration",

	// More S3 (would be nice to limit 'Resource' to just the bucket we actualy interact with...)
	"s3:PutObject",
	"s3:PutObjectAcl",
	"s3:PutObjectTagging",
	"s3:GetObject",
	"s3:GetObjectAcl",
	"s3:GetObjectTagging",
	"s3:GetObjectVersion",
	"s3:DeleteObject",

	// Uninstall-specific perms
	"autoscaling:DescribeAutoScalingGroups",
	"ec2:DeleteNetworkInterface",
	"ec2:DeleteVolume",
	"ec2:DeleteVpcEndpoints",
	"elasticloadbalancing:DescribeTargetGroups",
	"elasticloadbalancing:DeleteTargetGroup",
	"iam:ListInstanceProfiles",
	"iam:ListRolePolicies",
	"iam:ListUserPolicies",
	"tag:GetResources",
}

// ValidateCreds will try to create an AWS session, and also verify that the current credentials
// are sufficient to perform an installation, and that they can be used for cluster runtime
// as either capable of creating new credentials for components that interact with the cloud or
// being able to be passed through as-is to the components that need cloud credentials
func ValidateCreds(ssn *session.Session) error {
	creds, err := ssn.Config.Credentials.Get()
	if err != nil {
		return errors.Wrap(err, "getting creds from session")
	}

	client, err := ccaws.NewClient([]byte(creds.AccessKeyID), []byte(creds.SecretAccessKey))
	if err != nil {
		return errors.Wrap(err, "initialize cloud-credentials client")
	}

	// Check whether we can do an installation
	logger := logrus.New()
	canInstall, err := credvalidator.CheckPermissionsAgainstActions(client, installPermissions, logger)
	if err != nil {
		return errors.Wrap(err, "checking install permissions")
	}
	if !canInstall {
		return errors.New("current credentials insufficient for performing cluster installation")
	}

	// Check whether we can mint new creds for cluster services needing to interact with the cloud
	canMint, err := credvalidator.CheckCloudCredCreation(client, logger)
	if err != nil {
		return errors.Wrap(err, "mint credentials check")
	}

	// Check whether we can use the current credentials in passthrough mode to satisfy
	// cluster services needing to interact with the cloud
	canPassthrough, err := credvalidator.CheckCloudCredPassthrough(client, logger)
	if err != nil {
		return errors.Wrap(err, "passthrough credentials check")
	}

	if !canMint && !canPassthrough {
		return errors.New("AWS credentials cannot be used to either create new creds or use as-is")
	}

	return nil
}
<commit_msg>aws: use the standard logger for creds checking<commit_after>// Package aws collects AWS-specific configuration.
package aws

import (
	"github.com/aws/aws-sdk-go/aws/session"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	ccaws "github.com/openshift/cloud-credential-operator/pkg/aws"
	credvalidator "github.com/openshift/cloud-credential-operator/pkg/controller/utils"
)

var installPermissions = []string{
	// EC2 related perms
	"ec2:AllocateAddress",
	"ec2:AssociateAddress",
	"ec2:AssociateDhcpOptions",
	"ec2:AssociateRouteTable",
	"ec2:AttachInternetGateway",
	"ec2:AuthorizeSecurityGroupEgress",
	"ec2:AuthorizeSecurityGroupIngress",
	"ec2:CreateDhcpOptions",
	"ec2:CreateInternetGateway",
	"ec2:CreateNatGateway",
	"ec2:CreateRoute",
	"ec2:CreateRouteTable",
	"ec2:CreateSecurityGroup",
	"ec2:CreateSubnet",
	"ec2:CreateTags",
	"ec2:CreateVpc",
	"ec2:CreateVpcEndpoint",
	"ec2:CreateVolume",
	"ec2:DescribeAccountAttributes",
	"ec2:DescribeAddresses",
	"ec2:DescribeAvailabilityZones",
	"ec2:DescribeDhcpOptions",
	"ec2:DescribeImages",
	"ec2:DescribeInstanceAttribute",
	"ec2:DescribeInstanceCreditSpecifications",
	"ec2:DescribeInstances",
	"ec2:DescribeInternetGateways",
	"ec2:DescribeKeyPairs",
	"ec2:DescribeNatGateways",
	"ec2:DescribeNetworkAcls",
	"ec2:DescribePrefixLists",
	"ec2:DescribeRegions",
	"ec2:DescribeRouteTables",
	"ec2:DescribeSecurityGroups",
	"ec2:DescribeSubnets",
	"ec2:DescribeTags",
	"ec2:DescribeVpcEndpoints",
	"ec2:DescribeVpcs",
	"ec2:DescribeVpcAttribute",
	"ec2:DescribeVolumes",
	"ec2:DescribeVpcClassicLink",
	"ec2:DescribeVpcClassicLinkDnsSupport",
	"ec2:ModifyInstanceAttribute",
	"ec2:ModifySubnetAttribute",
	"ec2:ModifyVpcAttribute",
	"ec2:RevokeSecurityGroupEgress",
	"ec2:RunInstances",
	"ec2:TerminateInstances",
	"ec2:DeleteDhcpOptions",
	"ec2:DeleteRoute",
	"ec2:RevokeSecurityGroupIngress",
	"ec2:DisassociateRouteTable",
	"ec2:ReplaceRouteTableAssociation",
	"ec2:DeleteRouteTable",
	"ec2:DeleteSubnet",
	"ec2:DescribeNetworkInterfaces",
	"ec2:ModifyNetworkInterfaceAttribute",
	"ec2:DeleteNatGateway",
	"ec2:DeleteSecurityGroup",
	"ec2:DetachInternetGateway",
	"ec2:DeleteInternetGateway",
	"ec2:ReleaseAddress",
	"ec2:DeleteVpc",

	// ELB related perms
	"elasticloadbalancing:AddTags",
	"elasticloadbalancing:ApplySecurityGroupsToLoadBalancer",
	"elasticloadbalancing:AttachLoadBalancerToSubnets",
	"elasticloadbalancing:CreateListener",
	"elasticloadbalancing:CreateLoadBalancer",
	"elasticloadbalancing:CreateLoadBalancerListeners",
	"elasticloadbalancing:CreateTargetGroup",
	"elasticloadbalancing:ConfigureHealthCheck",
	"elasticloadbalancing:DeleteLoadBalancer",
	"elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
	"elasticloadbalancing:DeregisterTargets",
	"elasticloadbalancing:DescribeInstanceHealth",
	"elasticloadbalancing:DescribeListeners",
	"elasticloadbalancing:DescribeLoadBalancers",
	"elasticloadbalancing:DescribeLoadBalancerAttributes",
	"elasticloadbalancing:DescribeTags",
	"elasticloadbalancing:DescribeTargetGroupAttributes",
	"elasticloadbalancing:DescribeTargetHealth",
	"elasticloadbalancing:ModifyLoadBalancerAttributes",
	"elasticloadbalancing:ModifyTargetGroup",
	"elasticloadbalancing:ModifyTargetGroupAttributes",
	"elasticloadbalancing:RegisterTargets",
	"elasticloadbalancing:RegisterInstancesWithLoadBalancer",
	"elasticloadbalancing:SetLoadBalancerPoliciesOfListener",

	// IAM related perms
	"iam:AddRoleToInstanceProfile",
	"iam:CreateInstanceProfile",
	"iam:CreateRole",
	"iam:DeleteInstanceProfile",
	"iam:DeleteRole",
	"iam:DeleteRolePolicy",
	"iam:GetInstanceProfile",
	"iam:GetRole",
	"iam:GetRolePolicy",
	"iam:GetUser",
	"iam:ListInstanceProfilesForRole",
	"iam:ListRoles",
	"iam:ListUsers",
	"iam:PassRole",
	"iam:PutRolePolicy",
	"iam:RemoveRoleFromInstanceProfile",
	"iam:SimulatePrincipalPolicy",
	"iam:TagRole",

	// Route53 related perms
	"route53:ChangeResourceRecordSets",
	"route53:ChangeTagsForResource",
	"route53:GetChange",
	"route53:GetHostedZone",
	"route53:CreateHostedZone",
	"route53:DeleteHostedZone",
	"route53:ListHostedZones",
	"route53:ListHostedZonesByName",
	"route53:ListResourceRecordSets",
	"route53:ListTagsForResource",
	"route53:UpdateHostedZoneComment",

	// S3 related perms
	"s3:CreateBucket",
	"s3:ListBucket",
	"s3:GetBucketCors",
	"s3:GetBucketWebsite",
	"s3:GetBucketVersioning",
	"s3:GetAccelerateConfiguration",
	"s3:GetEncryptionConfiguration",
	"s3:GetBucketRequestPayment",
	"s3:GetBucketLogging",
	"s3:GetLifecycleConfiguration",
	"s3:GetBucketReplication",
	"s3:GetReplicationConfiguration",
	"s3:GetBucketLocation",
	"s3:GetBucketTagging",
	"s3:DeleteBucket",
	"s3:PutBucketAcl",
	"s3:PutBucketTagging",
	"s3:PutEncryptionConfiguration",

	// More S3 (would be nice to limit 'Resource' to just the bucket we actualy interact with...)
	"s3:PutObject",
	"s3:PutObjectAcl",
	"s3:PutObjectTagging",
	"s3:GetObject",
	"s3:GetObjectAcl",
	"s3:GetObjectTagging",
	"s3:GetObjectVersion",
	"s3:DeleteObject",

	// Uninstall-specific perms
	"autoscaling:DescribeAutoScalingGroups",
	"ec2:DeleteNetworkInterface",
	"ec2:DeleteVolume",
	"ec2:DeleteVpcEndpoints",
	"elasticloadbalancing:DescribeTargetGroups",
	"elasticloadbalancing:DeleteTargetGroup",
	"iam:ListInstanceProfiles",
	"iam:ListRolePolicies",
	"iam:ListUserPolicies",
	"tag:GetResources",
}

// ValidateCreds will try to create an AWS session, and also verify that the current credentials
// are sufficient to perform an installation, and that they can be used for cluster runtime
// as either capable of creating new credentials for components that interact with the cloud or
// being able to be passed through as-is to the components that need cloud credentials
func ValidateCreds(ssn *session.Session) error {
	creds, err := ssn.Config.Credentials.Get()
	if err != nil {
		return errors.Wrap(err, "getting creds from session")
	}

	client, err := ccaws.NewClient([]byte(creds.AccessKeyID), []byte(creds.SecretAccessKey))
	if err != nil {
		return errors.Wrap(err, "initialize cloud-credentials client")
	}

	// Check whether we can do an installation
	logger := logrus.StandardLogger()
	canInstall, err := credvalidator.CheckPermissionsAgainstActions(client, installPermissions, logger)
	if err != nil {
		return errors.Wrap(err, "checking install permissions")
	}
	if !canInstall {
		return errors.New("current credentials insufficient for performing cluster installation")
	}

	// Check whether we can mint new creds for cluster services needing to interact with the cloud
	canMint, err := credvalidator.CheckCloudCredCreation(client, logger)
	if err != nil {
		return errors.Wrap(err, "mint credentials check")
	}

	// Check whether we can use the current credentials in passthrough mode to satisfy
	// cluster services needing to interact with the cloud
	canPassthrough, err := credvalidator.CheckCloudCredPassthrough(client, logger)
	if err != nil {
		return errors.Wrap(err, "passthrough credentials check")
	}

	if !canMint && !canPassthrough {
		return errors.New("AWS credentials cannot be used to either create new creds or use as-is")
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package controller

import (
	"fmt"

	"github.com/go-kit/kit/log/level"
	habv1beta1 "github.com/kinvolk/habitat-operator/pkg/apis/habitat/v1beta1"
	apiv1 "k8s.io/api/core/v1"
	"k8s.io/client-go/tools/cache"
)

func (hc *HabitatController) cachePersistentVolumeClaims() {
	source := newListWatchFromClientWithLabels(
		hc.config.KubernetesClientset.CoreV1().RESTClient(),
		"persistentvolumeclaims",
		apiv1.NamespaceAll,
		labelListOptions())

	hc.cmInformer = cache.NewSharedIndexInformer(
		source,
		&apiv1.PersistentVolumeClaim{},
		resyncPeriod,
		cache.Indexers{},
	)

	hc.cmInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		UpdateFunc: hc.handlePVCUpdate,
		DeleteFunc: hc.handlePVCDelete,
	})

	hc.cmInformerSynced = hc.pvcInformer.HasSynced
}

func (hc *HabitatController) checkPVC(pvc *apiv1.PersistentVolumeClaim) {
	// Find out if there's a Habitat object around
	// If so, we have a problem.
	// TODO we might use an ownerref to find the STS here
	key := fmt.Sprintf("%s/%s", pvc.Namespace, pvc.Labels[habv1beta1.HabitatNameLabel])

	_, exists, err := hc.habInformer.GetStore().GetByKey(key)
	if err != nil {
		level.Error(hc.logger).Log("msg", "Failed to get key in Store", "obj", key)
		return
	} else if !exists {
		level.Debug(hc.logger).Log("msg", "No matching Habitat found for PVC", "name", pvc.Name)
		return
	} else {
		level.Error(hc.logger).Log("msg", "A PVC has lost its PersistentVolume", "name", pvc.Name)
	}
}

func (hc *HabitatController) handlePVCUpdate(oldObj, newObj interface{}) {
	pvc, ok := newObj.(*apiv1.PersistentVolumeClaim)
	if !ok {
		level.Error(hc.logger).Log("msg", "Failed to type assert PersistentVolumeClaim", "obj", newObj)
		return
	}

	if pvc.Status.Phase == apiv1.ClaimLost {
		hc.checkPVC(pvc)
	}
}

func (hc *HabitatController) handlePVCDelete(obj interface{}) {
	pvc, ok := obj.(*apiv1.PersistentVolumeClaim)
	if !ok {
		level.Error(hc.logger).Log("msg", "Failed to type assert PersistentVolumeClaim", "obj", obj)
		return
	}

	hc.checkPVC(pvc)
}
<commit_msg>Rename checkPVC method<commit_after>// Copyright (c) 2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package controller

import (
	"fmt"

	"github.com/go-kit/kit/log/level"
	habv1beta1 "github.com/kinvolk/habitat-operator/pkg/apis/habitat/v1beta1"
	apiv1 "k8s.io/api/core/v1"
	"k8s.io/client-go/tools/cache"
)

func (hc *HabitatController) cachePersistentVolumeClaims() {
	source := newListWatchFromClientWithLabels(
		hc.config.KubernetesClientset.CoreV1().RESTClient(),
		"persistentvolumeclaims",
		apiv1.NamespaceAll,
		labelListOptions())

	hc.cmInformer = cache.NewSharedIndexInformer(
		source,
		&apiv1.PersistentVolumeClaim{},
		resyncPeriod,
		cache.Indexers{},
	)

	hc.cmInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		UpdateFunc: hc.handlePVCUpdate,
		DeleteFunc: hc.handlePVCDelete,
	})

	hc.cmInformerSynced = hc.pvcInformer.HasSynced
}

// findHabForPVC looks for a matching Habitat object for a PVC.
// This method is called when the PVC has been deleted or its status is Lost,
// so finding a matching Habitat means that this object has lost its storage.
func (hc *HabitatController) findHabForPVC(pvc *apiv1.PersistentVolumeClaim) {
	// TODO we might use an ownerref to find the STS here
	key := fmt.Sprintf("%s/%s", pvc.Namespace, pvc.Labels[habv1beta1.HabitatNameLabel])

	_, exists, err := hc.habInformer.GetStore().GetByKey(key)
	if err != nil {
		level.Error(hc.logger).Log("msg", "Failed to get key in Store", "obj", key)
		return
	} else if !exists {
		level.Debug(hc.logger).Log("msg", "No matching Habitat found for PVC", "name", pvc.Name)
		return
	} else {
		level.Error(hc.logger).Log("msg", "A PVC has lost its PersistentVolume", "name", pvc.Name)
	}
}

func (hc *HabitatController) handlePVCUpdate(oldObj, newObj interface{}) {
	pvc, ok := newObj.(*apiv1.PersistentVolumeClaim)
	if !ok {
		level.Error(hc.logger).Log("msg", "Failed to type assert PersistentVolumeClaim", "obj", newObj)
		return
	}

	if pvc.Status.Phase == apiv1.ClaimLost {
		hc.findHabForPVC(pvc)
	}
}

func (hc *HabitatController) handlePVCDelete(obj interface{}) {
	pvc, ok := obj.(*apiv1.PersistentVolumeClaim)
	if !ok {
		level.Error(hc.logger).Log("msg", "Failed to type assert PersistentVolumeClaim", "obj", obj)
		return
	}

	hc.findHabForPVC(pvc)
}
<|endoftext|>


<commit_before>package v5

import (
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/jetstack-experimental/navigator/pkg/apis/navigator/v1alpha1"
)

const (
	elasticsearchConfigSubDir = "elasticsearch/config"
)

func (p *Pilot) WriteConfig(pilot *v1alpha1.Pilot) error {
	esConfigPath := fmt.Sprintf("%s/%s", p.Options.ConfigDir, elasticsearchConfigSubDir)
	err := filepath.Walk(esConfigPath, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}
		if err != nil {
			return err
		}
		relPath, err := filepath.Rel(esConfigPath, path)
		if err != nil {
			return err
		}
		dstPath := fmt.Sprintf("%s/%s", p.Options.ElasticsearchOptions.ConfigDir, relPath)
		relDir := filepath.Dir(relPath)
		stat, err := os.Stat(relDir)
		if os.IsNotExist(err) {
			err = os.MkdirAll(relDir, 0644)
			if err != nil {
				return err
			}
			stat, err = os.Stat(relDir)
		}
		if err != nil {
			return err
		}
		if !stat.IsDir() {
			return fmt.Errorf("path '%s' exists and is not a directory", relDir)
		}
		if err = copyFileContents(path, dstPath); err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return fmt.Errorf("error writing config file: %s", err.Error())
	}

	return nil
}

// copyFileContents copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
// From: https://stackoverflow.com/a/21067803
func copyFileContents(src, dst string) (err error) {
	in, err := os.Open(src)
	if err != nil {
		return
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		cerr := out.Close()
		if err == nil {
			err = cerr
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		return
	}
	err = out.Sync()
	return
}
<commit_msg>Skip config directories beginning with '..'<commit_after>package v5

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/jetstack-experimental/navigator/pkg/apis/navigator/v1alpha1"
)

const (
	elasticsearchConfigSubDir = "elasticsearch/config"
)

func (p *Pilot) WriteConfig(pilot *v1alpha1.Pilot) error {
	esConfigPath := fmt.Sprintf("%s/%s", p.Options.ConfigDir, elasticsearchConfigSubDir)
	err := filepath.Walk(esConfigPath, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			if strings.HasPrefix(info.Name(), "..") {
				return filepath.SkipDir
			}
			return nil
		}
		if err != nil {
			return err
		}
		relPath, err := filepath.Rel(esConfigPath, path)
		if err != nil {
			return err
		}
		dstPath := fmt.Sprintf("%s/%s", p.Options.ElasticsearchOptions.ConfigDir, relPath)
		relDir := filepath.Dir(relPath)
		stat, err := os.Stat(relDir)
		if os.IsNotExist(err) {
			err = os.MkdirAll(relDir, 0644)
			if err != nil {
				return err
			}
			stat, err = os.Stat(relDir)
		}
		if err != nil {
			return err
		}
		if !stat.IsDir() {
			return fmt.Errorf("path '%s' exists and is not a directory", relDir)
		}
		if err = copyFileContents(path, dstPath); err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return fmt.Errorf("error writing config file: %s", err.Error())
	}

	return nil
}

// copyFileContents copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
// From: https://stackoverflow.com/a/21067803
func copyFileContents(src, dst string) (err error) {
	in, err := os.Open(src)
	if err != nil {
		return
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		cerr := out.Close()
		if err == nil {
			err = cerr
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		return
	}
	err = out.Sync()
	return
}
<|endoftext|>


<commit_before>package sqlstore

import (
	"fmt"

	"github.com/grafana/grafana/pkg/bus"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/setting"
)

func init() {
	bus.AddHandler("sql", GetDashboardVersion)
	bus.AddHandler("sql", GetDashboardVersions)
	bus.AddHandler("sql", DeleteExpiredVersions)
}

// GetDashboardVersion gets the dashboard version for the given dashboard ID and version number.
func GetDashboardVersion(query *m.GetDashboardVersionQuery) error {
	version := m.DashboardVersion{}
	has, err := x.Where("dashboard_version.dashboard_id=? AND dashboard_version.version=? AND dashboard.org_id=?", query.DashboardId, query.Version, query.OrgId).
		Join("LEFT", "dashboard", `dashboard.id = dashboard_version.dashboard_id`).
		Get(&version)

	if err != nil {
		return err
	}

	if !has {
		return m.ErrDashboardVersionNotFound
	}

	version.Data.Set("id", version.DashboardId)
	query.Result = &version
	return nil
}

// GetDashboardVersions gets all dashboard versions for the given dashboard ID.
func GetDashboardVersions(query *m.GetDashboardVersionsQuery) error {
	if query.Limit == 0 {
		query.Limit = 1000
	}

	err := x.Table("dashboard_version").
		Select(`dashboard_version.id,
				dashboard_version.dashboard_id,
				dashboard_version.parent_version,
				dashboard_version.restored_from,
				dashboard_version.version,
				dashboard_version.created,
				dashboard_version.created_by as created_by_id,
				dashboard_version.message,
				dashboard_version.data,`+
			dialect.Quote("user")+`.login as created_by`).
		Join("LEFT", "user", `dashboard_version.created_by = `+dialect.Quote("user")+`.id`).
		Join("LEFT", "dashboard", `dashboard.id = dashboard_version.dashboard_id`).
		Where("dashboard_version.dashboard_id=? AND dashboard.org_id=?", query.DashboardId, query.OrgId).
		OrderBy("dashboard_version.version DESC").
		Limit(query.Limit, query.Start).
		Find(&query.Result)
	if err != nil {
		return err
	}

	if len(query.Result) < 1 {
		return m.ErrNoVersionsForDashboardId
	}
	return nil
}

func DeleteExpiredVersions(cmd *m.DeleteExpiredVersionsCommand) error {
	return inTransaction(func(sess *DBSession) error {
		versionsToKeep := setting.DashboardVersionsToKeep
		if versionsToKeep < 1 {
			versionsToKeep = 1
		}

		// Idea of this query is finding version IDs to delete based on formula:
		// min_version_to_keep = min_version + (versions_count - versions_to_keep)
		// where version stats is processed for each dashboard. This guarantees that we keep at least versions_to_keep
		// versions, but in some cases (when versions are sparse) this number may be more.
		versionIdsToDeleteSybqueryTemplate := `SELECT id
			FROM dashboard_version, (
				SELECT dashboard_id, count(version) as count, min(version) as min
				FROM dashboard_version
				GROUP BY dashboard_id
			) AS vtd
			WHERE dashboard_version.dashboard_id=vtd.dashboard_id
				AND version < vtd.min + vtd.count - %v`

		versionIdsToDeleteSubquery := fmt.Sprintf(versionIdsToDeleteSybqueryTemplate, versionsToKeep)
		deleteExpiredSql := fmt.Sprintf(`DELETE FROM dashboard_version WHERE id IN (%s)`, versionIdsToDeleteSubquery)
		expiredResponse, err := sess.Exec(deleteExpiredSql)
		if err != nil {
			return err
		}
		cmd.DeletedRows, _ = expiredResponse.RowsAffected()

		return nil
	})
}

// Short version of DashboardVersion for getting expired versions
type DashboardVersionExp struct {
	Id          int64 `json:"id"`
	DashboardId int64 `json:"dashboardId"`
	Version     int   `json:"version"`
}

func getVersionIDsToDelete(versions []DashboardVersionExp, versionsToKeep int) []interface{} {
	versionIds := make([]interface{}, 0)

	if len(versions) == 0 {
		return versionIds
	}

	currentDashboard := versions[0].DashboardId
	count := 0
	for _, v := range versions {
		if v.DashboardId == currentDashboard {
			count++
		} else {
			count = 1
			currentDashboard = v.DashboardId
		}
		if count > versionsToKeep {
			versionIds = append(versionIds, v.Id)
		}
	}

	return versionIds
}
<commit_msg>limit number of rows deleted by dashboard version cleanup<commit_after>package sqlstore

import (
	"fmt"
	"strings"

	"github.com/grafana/grafana/pkg/bus"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/setting"
)

func init() {
	bus.AddHandler("sql", GetDashboardVersion)
	bus.AddHandler("sql", GetDashboardVersions)
	bus.AddHandler("sql", DeleteExpiredVersions)
}

// GetDashboardVersion gets the dashboard version for the given dashboard ID and version number.
func GetDashboardVersion(query *m.GetDashboardVersionQuery) error {
	version := m.DashboardVersion{}
	has, err := x.Where("dashboard_version.dashboard_id=? AND dashboard_version.version=? AND dashboard.org_id=?", query.DashboardId, query.Version, query.OrgId).
		Join("LEFT", "dashboard", `dashboard.id = dashboard_version.dashboard_id`).
		Get(&version)

	if err != nil {
		return err
	}

	if !has {
		return m.ErrDashboardVersionNotFound
	}

	version.Data.Set("id", version.DashboardId)
	query.Result = &version
	return nil
}

// GetDashboardVersions gets all dashboard versions for the given dashboard ID.
func GetDashboardVersions(query *m.GetDashboardVersionsQuery) error {
	if query.Limit == 0 {
		query.Limit = 1000
	}

	err := x.Table("dashboard_version").
		Select(`dashboard_version.id,
				dashboard_version.dashboard_id,
				dashboard_version.parent_version,
				dashboard_version.restored_from,
				dashboard_version.version,
				dashboard_version.created,
				dashboard_version.created_by as created_by_id,
				dashboard_version.message,
				dashboard_version.data,`+
			dialect.Quote("user")+`.login as created_by`).
		Join("LEFT", "user", `dashboard_version.created_by = `+dialect.Quote("user")+`.id`).
		Join("LEFT", "dashboard", `dashboard.id = dashboard_version.dashboard_id`).
		Where("dashboard_version.dashboard_id=? AND dashboard.org_id=?", query.DashboardId, query.OrgId).
		OrderBy("dashboard_version.version DESC").
		Limit(query.Limit, query.Start).
		Find(&query.Result)
	if err != nil {
		return err
	}

	if len(query.Result) < 1 {
		return m.ErrNoVersionsForDashboardId
	}
	return nil
}

func DeleteExpiredVersions(cmd *m.DeleteExpiredVersionsCommand) error {
	return inTransaction(func(sess *DBSession) error {
		const MAX_VERSIONS_TO_DELETE = 100

		versionsToKeep := setting.DashboardVersionsToKeep
		if versionsToKeep < 1 {
			versionsToKeep = 1
		}

		// Idea of this query is finding version IDs to delete based on formula:
		// min_version_to_keep = min_version + (versions_count - versions_to_keep)
		// where version stats is processed for each dashboard. This guarantees that we keep at least versions_to_keep
		// versions, but in some cases (when versions are sparse) this number may be more.
		versionIdsToDeleteSybqueryTemplate := `SELECT id
			FROM dashboard_version, (
				SELECT dashboard_id, count(version) as count, min(version) as min
				FROM dashboard_version
				GROUP BY dashboard_id
				) AS vtd
				WHERE dashboard_version.dashboard_id=vtd.dashboard_id
				AND version < vtd.min + vtd.count - %v`

		versionIdsToDeleteSubquery := fmt.Sprintf(versionIdsToDeleteSybqueryTemplate, versionsToKeep)
		versions := []string{}
		err := sess.SQL(versionIdsToDeleteSubquery).Find(&versions)
		if err != nil {
			return err
		}

		// Don't delete more than MAX_VERSIONS_TO_DELETE version per time
		limit := MAX_VERSIONS_TO_DELETE
		if len(versions) < MAX_VERSIONS_TO_DELETE {
			limit = len(versions)
		}
		versions = versions[:limit]

		deleteExpiredSql := fmt.Sprintf(`DELETE FROM dashboard_version WHERE id IN (%s)`, strings.Join(versions, `,`))
		expiredResponse, err := sess.Exec(deleteExpiredSql)
		if err != nil {
			return err
		}
		cmd.DeletedRows, _ = expiredResponse.RowsAffected()

		return nil
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package resource

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/sirupsen/logrus"

	"github.com/GoogleContainerTools/skaffold/pkg/diag"
	"github.com/GoogleContainerTools/skaffold/pkg/diag/validator"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/event"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubectl"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/runner/runcontext"
	"github.com/GoogleContainerTools/skaffold/proto"
)

const (
	deploymentType          = "deployment"
	rollOutSuccess          = "successfully rolled out"
	connectionErrMsg        = "Unable to connect to the server"
	killedErrMsg            = "signal: killed"
	defaultPodCheckDeadline = 30 * time.Second
	tabHeader               = " -"
	tab                     = "  "
)

var (
	errKubectlKilled     = errors.New("kubectl rollout status command interrupted")
	ErrKubectlConnection = errors.New("kubectl connection error")
)

type Deployment struct {
	name         string
	namespace    string
	rType        string
	status       Status
	done         bool
	deadline     time.Duration
	pods         map[string]validator.Resource
	podDiagnoser diag.Diagnose
}

func (d *Deployment) Deadline() time.Duration {
	return d.deadline
}

func (d *Deployment) UpdateStatus(details string, err error) {
	errCode := proto.StatusCode_STATUSCHECK_SUCCESS
	if err != nil {
		errCode = proto.StatusCode_STATUSCHECK_UNKNOWN
	}
	updated := newStatus(details, errCode, err)
	if d.status.Equal(updated) {
		d.status.changed = false
		return
	}
	d.status.changed = true
	d.status = updated
	if strings.Contains(details, rollOutSuccess) || isErrAndNotRetryAble(err) {
		d.done = true
	}
}

func NewDeployment(name string, ns string, deadline time.Duration) *Deployment {
	return &Deployment{
		name:         name,
		namespace:    ns,
		rType:        deploymentType,
		status:       newStatus("", proto.StatusCode_STATUSCHECK_UNKNOWN, nil),
		deadline:     deadline,
		podDiagnoser: diag.New(nil),
	}
}

func (d *Deployment) WithDiagnoser(pd diag.Diagnose) *Deployment {
	d.podDiagnoser = pd
	return d
}

func (d *Deployment) CheckStatus(ctx context.Context, runCtx *runcontext.RunContext) {
	kubeCtl := kubectl.NewFromRunContext(runCtx)

	b, err := kubeCtl.RunOut(ctx, "rollout", "status", "deployment", d.name, "--namespace", d.namespace, "--watch=false")
	if ctx.Err() != nil {
		return
	}

	details := d.cleanupStatus(string(b))

	err = parseKubectlRolloutError(err)
	if err == errKubectlKilled {
		err = fmt.Errorf("received Ctrl-C or deployments could not stabilize within %v: %w", d.deadline, err)
	} else if err := d.fetchPods(ctx); err != nil {
		logrus.Debugf("pod statuses could be fetched this time due to %s", err)
	}
	d.UpdateStatus(details, err)
}

func (d *Deployment) String() string {
	if d.namespace == "default" {
		return fmt.Sprintf("%s/%s", d.rType, d.name)
	}

	return fmt.Sprintf("%s:%s/%s", d.namespace, d.rType, d.name)
}

func (d *Deployment) Name() string {
	return d.name
}

func (d *Deployment) Status() Status {
	return d.status
}

func (d *Deployment) IsStatusCheckComplete() bool {
	return d.done
}

// This returns a string representing deployment status along with tab header
// e.g.
//  - testNs/leeroy-app: waiting for rollout to complete. (1/2) pending
//      - testNs/leeroy-app-xvbg : error pulling container image
func (d *Deployment) ReportSinceLastUpdated() string {
	if !d.status.changed {
		return ""
	}
	if d.status.String() == "" {
		return ""
	}
	var result strings.Builder
	result.WriteString(fmt.Sprintf("%s %s: %s", tabHeader, d, d.status))
	for _, p := range d.pods {
		if p.Error() != nil {
			result.WriteString(fmt.Sprintf("%s %s %s: %s\n", tab, tabHeader, p, p.Error()))
		}
	}
	return result.String()
}

func (d *Deployment) cleanupStatus(msg string) string {
	clean := strings.ReplaceAll(msg, `deployment "`+d.Name()+`" `, "")
	if len(clean) > 0 {
		clean = strings.ToLower(clean[0:1]) + clean[1:]
	}
	return clean
}

func parseKubectlRolloutError(err error) error {
	if err == nil {
		return err
	}
	if strings.Contains(err.Error(), connectionErrMsg) {
		return ErrKubectlConnection
	}
	if strings.Contains(err.Error(), killedErrMsg) {
		return errKubectlKilled
	}
	return err
}

func isErrAndNotRetryAble(err error) bool {
	if err == nil {
		return false
	}
	return err != ErrKubectlConnection
}

func (d *Deployment) fetchPods(ctx context.Context) error {
	timeoutContext, cancel := context.WithTimeout(ctx, defaultPodCheckDeadline)
	defer cancel()
	pods, err := d.podDiagnoser.Run(timeoutContext)
	if err != nil {
		return err
	}

	newPods := map[string]validator.Resource{}
	d.status.changed = false
	for _, p := range pods {
		originalPod, ok := d.pods[p.String()]
		if !ok {
			d.status.changed = true
			event.ResourceStatusCheckEventCompleted(p.String(), p.Error())
		}
		if originalPod.StatusCode != p.StatusCode {
			d.status.changed = true
			event.ResourceStatusCheckEventCompleted(p.String(), p.Error())
		}
		newPods[p.String()] = p
	}
	d.pods = newPods
	return nil
}
<commit_msg>Update status before fetching pods<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package resource

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/sirupsen/logrus"

	"github.com/GoogleContainerTools/skaffold/pkg/diag"
	"github.com/GoogleContainerTools/skaffold/pkg/diag/validator"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/event"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubectl"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/runner/runcontext"
	"github.com/GoogleContainerTools/skaffold/proto"
)

const (
	deploymentType          = "deployment"
	rollOutSuccess          = "successfully rolled out"
	connectionErrMsg        = "Unable to connect to the server"
	killedErrMsg            = "signal: killed"
	defaultPodCheckDeadline = 30 * time.Second
	tabHeader               = " -"
	tab                     = "  "
)

var (
	errKubectlKilled     = errors.New("kubectl rollout status command interrupted")
	ErrKubectlConnection = errors.New("kubectl connection error")
)

type Deployment struct {
	name         string
	namespace    string
	rType        string
	status       Status
	done         bool
	deadline     time.Duration
	pods         map[string]validator.Resource
	podDiagnoser diag.Diagnose
}

func (d *Deployment) Deadline() time.Duration {
	return d.deadline
}

func (d *Deployment) UpdateStatus(details string, err error) {
	errCode := proto.StatusCode_STATUSCHECK_SUCCESS
	if err != nil {
		errCode = proto.StatusCode_STATUSCHECK_UNKNOWN
	}
	updated := newStatus(details, errCode, err)
	if d.status.Equal(updated) {
		d.status.changed = false
		return
	}
	d.status.changed = true
	d.status = updated
	if strings.Contains(details, rollOutSuccess) || isErrAndNotRetryAble(err) {
		d.done = true
	}
}

func NewDeployment(name string, ns string, deadline time.Duration) *Deployment {
	return &Deployment{
		name:         name,
		namespace:    ns,
		rType:        deploymentType,
		status:       newStatus("", proto.StatusCode_STATUSCHECK_UNKNOWN, nil),
		deadline:     deadline,
		podDiagnoser: diag.New(nil),
	}
}

func (d *Deployment) WithDiagnoser(pd diag.Diagnose) *Deployment {
	d.podDiagnoser = pd
	return d
}

func (d *Deployment) CheckStatus(ctx context.Context, runCtx *runcontext.RunContext) {
	kubeCtl := kubectl.NewFromRunContext(runCtx)

	b, err := kubeCtl.RunOut(ctx, "rollout", "status", "deployment", d.name, "--namespace", d.namespace, "--watch=false")
	if ctx.Err() != nil {
		return
	}

	details := d.cleanupStatus(string(b))
	d.UpdateStatus(details, err)

	err = parseKubectlRolloutError(err)
	if err == errKubectlKilled {
		err = fmt.Errorf("received Ctrl-C or deployments could not stabilize within %v: %w", d.deadline, err)
	} else if err := d.fetchPods(ctx); err != nil {
		logrus.Debugf("pod statuses could be fetched this time due to %s", err)
	}

}

func (d *Deployment) String() string {
	if d.namespace == "default" {
		return fmt.Sprintf("%s/%s", d.rType, d.name)
	}

	return fmt.Sprintf("%s:%s/%s", d.namespace, d.rType, d.name)
}

func (d *Deployment) Name() string {
	return d.name
}

func (d *Deployment) Status() Status {
	return d.status
}

func (d *Deployment) IsStatusCheckComplete() bool {
	return d.done
}

// This returns a string representing deployment status along with tab header
// e.g.
//  - testNs/leeroy-app: waiting for rollout to complete. (1/2) pending
//      - testNs/leeroy-app-xvbg : error pulling container image
func (d *Deployment) ReportSinceLastUpdated() string {
	if !d.status.changed {
		return ""
	}
	if d.status.String() == "" {
		return ""
	}
	var result strings.Builder
	result.WriteString(fmt.Sprintf("%s %s: %s", tabHeader, d, d.status))
	for _, p := range d.pods {
		if p.Error() != nil {
			result.WriteString(fmt.Sprintf("%s %s %s: %s\n", tab, tabHeader, p, p.Error()))
		}
	}
	return result.String()
}

func (d *Deployment) cleanupStatus(msg string) string {
	clean := strings.ReplaceAll(msg, `deployment "`+d.Name()+`" `, "")
	if len(clean) > 0 {
		clean = strings.ToLower(clean[0:1]) + clean[1:]
	}
	return clean
}

func parseKubectlRolloutError(err error) error {
	if err == nil {
		return err
	}
	if strings.Contains(err.Error(), connectionErrMsg) {
		return ErrKubectlConnection
	}
	if strings.Contains(err.Error(), killedErrMsg) {
		return errKubectlKilled
	}
	return err
}

func isErrAndNotRetryAble(err error) bool {
	if err == nil {
		return false
	}
	return err != ErrKubectlConnection
}

func (d *Deployment) fetchPods(ctx context.Context) error {
	timeoutContext, cancel := context.WithTimeout(ctx, defaultPodCheckDeadline)
	defer cancel()
	pods, err := d.podDiagnoser.Run(timeoutContext)
	if err != nil {
		return err
	}

	newPods := map[string]validator.Resource{}
	d.status.changed = false
	for _, p := range pods {
		originalPod, ok := d.pods[p.String()]
		if !ok {
			d.status.changed = true
			event.ResourceStatusCheckEventCompleted(p.String(), p.Error())
		} else if originalPod.StatusCode != p.StatusCode {
			d.status.changed = true
			event.ResourceStatusCheckEventCompleted(p.String(), p.Error())
		}
		newPods[p.String()] = p
	}
	d.pods = newPods
	return nil
}
<|endoftext|>


<commit_before>package metadata

type MetaData struct {
  Version         string              `json:"metadata_version"`
  Title           LanguageList        `json:"title"`
  Description     LanguageList        `json:"description"`
  Year            int                 `json:"production_year"`
  Type            string              `json:"type"`
  Series          Series              `json:"series,omitempty"`
  Genres          []string            `json:"genres"`
  Credits         []Credit            `json:"credits"`
  Regional        bool                `json:"regional_content"`
  Rating          string              `json:"parental_rating"`
  Ratio           string              `json:"aspect_ratio,omitempty"`
  Duration        string              `json:"expected_duration,omitempty"`
  Countries       []string            `json:"country_of_origin"`
}

type Series struct {
  Episode         int                 `json:"episode_number,omitempty"`
  Season          int                 `json:"season,omitempty"`
  ExternalId      string              `json:"external_id,omitempty"`
  InternalId      string              `json:"internal_id,omitempty"`
  EpisodeCount    int                 `json:"episodes_in_season,omitempty"`
}

type Credit struct {
  Name            string              `json:"name"`
  Function        string              `json:"role"`
}

type Language     map[string]string
type LanguageList map[string]Language<commit_msg>Adding in all outputs to Akka from mizuchi (#88)<commit_after>package metadata

type MetaData struct {
  Version         string              `json:"metadata_version"`
  Title           LanguageList        `json:"title"`
  Description     LanguageList        `json:"description"`
  Year            int                 `json:"production_year"`
  Type            string              `json:"type"`
  Series          Series              `json:"series,omitempty"`
  Genres          []string            `json:"genres"`
  Credits         []Credit            `json:"credits"`
  Regional        bool                `json:"regional_content"`
  Rating          string              `json:"parental_rating"`
  Ratio           string              `json:"aspect_ratio,omitempty"`
  Duration        string              `json:"expected_duration,omitempty"`
  Countries       []string            `json:"country_of_origin"`
}

type Series struct {
  Episode         int                 `json:"episode_number,omitempty"`
  Season          int                 `json:"season,omitempty"`
  ExternalId      string              `json:"external_id,omitempty"`
  InternalId      string              `json:"internal_id,omitempty"`
  EpisodeCount    int                 `json:"episodes_in_season,omitempty"`
}

type Credit struct {
  Name            string              `json:"name"`
  Function        string              `json:"role"`
}

type Language     map[string]string
type LanguageList map[string]Language

type ImageData struct {
  Type          string              `json:"type"`
  Orientation   string              `json:"orientation"`
  Language      string              `json:"language,omitempty"`
  File          string              `json:"org_file"`
}

type AkkaXMLAsset struct {
  MetaData
  Images        []ImageData         `json:"images"`
}<|endoftext|>


<commit_before>// +build freebsd

package freebsd

import (
	"fmt"
	"os/exec"
	"regexp"
	"strconv"
	"strings"

	"github.com/mackerelio/mackerel-agent/logging"
	"github.com/mackerelio/mackerel-agent/metrics"
)

// MemoryGenerator XXX
type MemoryGenerator struct {
}

var memoryLogger = logging.GetLogger("metrics.memory")

// Generate generate metrics values
func (g *MemoryGenerator) Generate() (metrics.Values, error) {
	var errRet error
	outBytes, err := exec.Command("top", "-bn", "1").Output()
	if err != nil {
		logger.Warningf("'top -bn 1' command exited with a non-zero status: '%s'", err)
		errRet = err
	}
	ret := make(map[string]float64)

	out := string(outBytes)
	lines := strings.Split(out, "\n")

	for _, line := range lines {
		fields := strings.Fields(line)
		if len(fields) < 1 {
			continue
		}

		if fields[0] == "Mem:" {

			if v, err := getValue(fields[1]); err == nil {
				ret["memory.active"] = v
			} else {
				errRet = err
			}
			if v, err := getValue(fields[3]); err == nil {
				ret["memory.inactive"] = v
			} else {
				errRet = err
			}
			/*
			         if v, err := getValue(fields[5]); err == nil {
			           ret["memory.wired"] = v
			         } else {
			   	errRet = err
			         }
			*/
			if v, err := getValue(fields[7]); err == nil {
				ret["memory.cached"] = v
			} else {
				errRet = err
			}
			if v, err := getValue(fields[9]); err == nil {
				ret["memory.buffers"] = v
			} else {
				errRet = err
			}
			if v, err := getValue(fields[11]); err == nil {
				ret["memory.free"] = v
			} else {
				errRet = err
			}
		}
		if fields[0] == "Swap:" {
			if v, err := getValue(fields[1]); err == nil {
				ret["memory.swap_total"] = v
			} else {
				errRet = err
			}
			swapFreeIndex := 5
			if len(fields) == 5 {
			   swapFreeIndex =  3
			}
			if v, err := getValue(fields[swapFreeIndex]); err == nil {
				ret["memory.swap_free"] = v
			} else {
				errRet = err
			}
		}
	}

	v, err := getTotalMem()
	if err != nil {
		return nil, err
	}
	ret["memory.total"] = v

	if errRet == nil {
		return metrics.Values(ret), nil
	}
	return nil, errRet
}

func getValue(strValue string) (float64, error) {
	parseRegexp := regexp.MustCompile(`^(\d+)(.?)$`)
	match := parseRegexp.FindStringSubmatch(strValue)
	var unit float64 = 1
	switch {
	case match[2] == "G":
		unit = 1024 * 1024 * 1024
	case match[2] == "M":
		unit = 1024 * 1024
	case match[2] == "K":
		unit = 1024
	}
	value, err := strconv.ParseFloat(strings.TrimSpace(match[1]), 64)
	if err != nil {
		return 0, fmt.Errorf("while parsing %q: %s", match[1], err)
	}
	return value * unit, nil
}

func getTotalMem() (float64, error) {
	cmd := exec.Command("sysctl", "-n", "hw.physmem")
	outputBytes, err := cmd.Output()
	if err != nil {
		return 0, fmt.Errorf("sysctl -n hw.physmem: %s", err)
	}

	output := string(outputBytes)

	memsizeInBytes, err := strconv.ParseFloat(strings.TrimSpace(output), 64)
	if err != nil {
		return 0, fmt.Errorf("while parsing %q: %s", output, err)
	}

	return memsizeInBytes, nil
}
<commit_msg>Fix 'panic: runtime error: index out of range'.<commit_after>// +build freebsd

package freebsd

import (
	"fmt"
	"os/exec"
	"regexp"
	"strconv"
	"strings"

	"github.com/mackerelio/mackerel-agent/logging"
	"github.com/mackerelio/mackerel-agent/metrics"
)

// MemoryGenerator XXX
type MemoryGenerator struct {
}

var memoryLogger = logging.GetLogger("metrics.memory")

// Generate generate metrics values
func (g *MemoryGenerator) Generate() (metrics.Values, error) {
	var errRet error
	outBytes, err := exec.Command("top", "-bn", "1").Output()
	if err != nil {
		logger.Warningf("'top -bn 1' command exited with a non-zero status: '%s'", err)
		errRet = err
	}
	ret := make(map[string]float64)

	out := string(outBytes)
	lines := strings.Split(out, "\n")

	for _, line := range lines {
		fields := strings.Fields(line)
		if len(fields) < 1 {
			continue
		}

		if fields[0] == "Mem:" {
			for i := 1; i < len(fields); i += 2 {
				v, err := getValue(fields[i])
				if err == nil {
					k := fields[i + 1]
					if strings.HasSuffix(k, ",") {
						k = k[0 : len(k) - 1]
					}
					switch k {
					case "Active":
						ret["memory.active"] = v
					case "Inact":
						ret["memory.inactive"] = v
					/*
					case "Wired":
						ret["memory.wired"] = v
					*/
					case "Cache":
						ret["memory.cached"] = v
					case "Buf":
						ret["memory.buffers"] = v
					case "Free":
						ret["memory.free"] = v
					}
				} else {
					errRet = err;
				}
			}
		}
		if fields[0] == "Swap:" {
			if v, err := getValue(fields[1]); err == nil {
				ret["memory.swap_total"] = v
			} else {
				errRet = err
			}
			swapFreeIndex := 5
			if len(fields) == 5 {
			   swapFreeIndex =  3
			}
			if v, err := getValue(fields[swapFreeIndex]); err == nil {
				ret["memory.swap_free"] = v
			} else {
				errRet = err
			}
		}
	}

	v, err := getTotalMem()
	if err != nil {
		return nil, err
	}
	ret["memory.total"] = v

	if errRet == nil {
		return metrics.Values(ret), nil
	}
	return nil, errRet
}

func getValue(strValue string) (float64, error) {
	parseRegexp := regexp.MustCompile(`^(\d+)(.?)$`)
	match := parseRegexp.FindStringSubmatch(strValue)
	var unit float64 = 1
	switch {
	case match[2] == "G":
		unit = 1024 * 1024 * 1024
	case match[2] == "M":
		unit = 1024 * 1024
	case match[2] == "K":
		unit = 1024
	}
	value, err := strconv.ParseFloat(strings.TrimSpace(match[1]), 64)
	if err != nil {
		return 0, fmt.Errorf("while parsing %q: %s", match[1], err)
	}
	return value * unit, nil
}

func getTotalMem() (float64, error) {
	cmd := exec.Command("sysctl", "-n", "hw.physmem")
	outputBytes, err := cmd.Output()
	if err != nil {
		return 0, fmt.Errorf("sysctl -n hw.physmem: %s", err)
	}

	output := string(outputBytes)

	memsizeInBytes, err := strconv.ParseFloat(strings.TrimSpace(output), 64)
	if err != nil {
		return 0, fmt.Errorf("while parsing %q: %s", output, err)
	}

	return memsizeInBytes, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/mmitton/asn1-ber"
	"github.com/mmitton/ldap"
	"net"
	"os"
)

const (
	ListenPort = "3333"
	ListenType = "tcp"
)

func main() {
	// Listen for incoming connections.
	l, err := net.Listen(ListenType, ":"+ListenPort)
	if err != nil {
		fmt.Println("Error listening:", err.Error())
		os.Exit(1)
	}
	// Close the listener when the application closes.
	defer l.Close()
	fmt.Println("Listening on :" + ListenPort)
	for {
		// Listen for an incoming connection.
		conn, err := l.Accept()
		if err != nil {
			fmt.Println("Error accepting: ", err.Error())
			os.Exit(1)
		}

		//logs an incoming message
		fmt.Printf("Received message %s -> %s \n", conn.RemoteAddr(), conn.LocalAddr())

		// Handle connections in a new goroutine.
		go handleRequest(conn)
	}
}

// Handles incoming requests.
func handleRequest(conn net.Conn) {
	defer conn.Close()

	buf := make([]byte, 1024)
	_, err := conn.Read(buf)
	if err != nil {
		fmt.Println("Error reading:", err.Error())
	}

	packet := ber.DecodePacket(buf)

	if len(packet.Children) == 0 {
		//handles TLS requests over non-TLS and vice-versa
		log.Println("Error decoding asn1-ber packet: wrong port?")
		return
	}

	messageID := packet.Children[0].Value.(uint64)
	response := packet.Children[1]

	if response.ClassType == ber.ClassApplication &&
		response.TagType == ber.TypeConstructed {

		if response.Tag == ldap.ApplicationBindRequest {
			version := response.Children[0].Value.(uint64)
			name := response.Children[1].Value.(string)
			auth := response.Children[2]
			pass := ""

			//tag == 0 (I think) indicates simple auth
			//tag == 3 should be SASL once we support SSL/TLS
			if auth.Tag == 0 {
				pass = auth.Data.String()
				fmt.Println("ApplicationBindRequest:",
					"messageID:", messageID,
					//"response", response,
					"LDAP version:", version,
					"username:", name,
					//"auth", auth,
					"password:", pass)
			}
		}

	}
}
<commit_msg>Use log rather than fmt where appropriate<commit_after>package main

import (
	"fmt"
	"github.com/mmitton/asn1-ber"
	"github.com/mmitton/ldap"
	"net"
	"os"
)

const (
	ListenPort = "3333"
	ListenType = "tcp"
)

func main() {
	// Listen for incoming connections.
	l, err := net.Listen(ListenType, ":"+ListenPort)
	if err != nil {
		log.Fatal(err)
	}
	// Close the listener when the application closes.
	defer l.Close()
	fmt.Println("Listening on :" + ListenPort)
	for {
		// Listen for an incoming connection.
		conn, err := l.Accept()
		if err != nil {
			log.Fatal(err)
		}

		//logs an incoming message
		log.Printf("Received message %s -> %s \n", conn.RemoteAddr(), conn.LocalAddr())

		// Handle connections in a new goroutine.
		go handleRequest(conn)
	}
}

// Handles incoming requests.
func handleRequest(conn net.Conn) {
	defer conn.Close()

	buf := make([]byte, 1024)
	_, err := conn.Read(buf)
	if err != nil {
		log.Println("Error reading:", err.Error())
	}

	packet := ber.DecodePacket(buf)

	if len(packet.Children) == 0 {
		//handles TLS requests over non-TLS and vice-versa
		log.Println("Error decoding asn1-ber packet: wrong port?")
		return
	}

	messageID := packet.Children[0].Value.(uint64)
	response := packet.Children[1]

	if response.ClassType == ber.ClassApplication &&
		response.TagType == ber.TypeConstructed {

		if response.Tag == ldap.ApplicationBindRequest {
			version := response.Children[0].Value.(uint64)
			name := response.Children[1].Value.(string)
			auth := response.Children[2]
			pass := ""

			//tag == 0 (I think) indicates simple auth
			//tag == 3 should be SASL once we support SSL/TLS
			if auth.Tag == 0 {
				pass = auth.Data.String()
				fmt.Println("ApplicationBindRequest:",
					"messageID:", messageID,
					//"response", response,
					"LDAP version:", version,
					"username:", name,
					//"auth", auth,
					"password:", pass)
			}
		}

	}
}
<|endoftext|>


<commit_before>package http

import (
	"bytes"
	"compress/zlib"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptrace"
	"net/url"
	"sync"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/sirupsen/logrus"
	"github.com/stripe/veneur/trace"
)

var tracer = trace.GlobalTracer

type HTTPClientTracer struct {
	prefix      string
	stats       *statsd.Client
	mutex       *sync.Mutex
	ctx         context.Context
	currentSpan *trace.Span
}

func NewHTTPClientTrace(ctx context.Context, stats *statsd.Client, prefix string) *HTTPClientTracer {
	// TODO Should be a child of PostHelper but isn't
	span, _ := trace.StartSpanFromContext(ctx, "http.start")
	return &HTTPClientTracer{
		prefix:      prefix,
		stats:       stats,
		mutex:       &sync.Mutex{},
		ctx:         ctx,
		currentSpan: span,
	}
}

func (hct *HTTPClientTracer) GetClientTrace() *httptrace.ClientTrace {
	return &httptrace.ClientTrace{
		GotConn:              hct.GotConn,
		DNSStart:             hct.DNSStart,
		GotFirstResponseByte: hct.GotFirstResponseByte,
		ConnectStart:         hct.ConnectStart,
		WroteRequest:         hct.WroteRequest,
	}
}

func (hct *HTTPClientTracer) StartSpan(name string) *trace.Span {
	hct.mutex.Lock()
	defer hct.mutex.Unlock()
	newSpan, _ := trace.StartSpanFromContext(hct.ctx, name)
	hct.currentSpan.ClientFinish(trace.DefaultClient) // TODO Fix client

	hct.currentSpan = newSpan
	return newSpan
}

func (hct *HTTPClientTracer) GotConn(info httptrace.GotConnInfo) {
	state := "new"
	if info.Reused {
		state = "reused"
	}
	hct.StartSpan(fmt.Sprintf("http.gotConnection.%s", state))

	hct.stats.Count(hct.prefix+".connections_used_total", 1, []string{fmt.Sprintf("state:%s", state)}, 1.0)
}

func (hct *HTTPClientTracer) DNSStart(info httptrace.DNSStartInfo) {
	hct.StartSpan("http.resolvingDNS")
}

func (hct *HTTPClientTracer) GotFirstResponseByte() {
	// TODO Is this working?
	hct.StartSpan("http.gotFirstByte")
}

func (hct *HTTPClientTracer) ConnectStart(network, addr string) {
	hct.StartSpan("http.connecting")
}

func (hct *HTTPClientTracer) WroteRequest(info httptrace.WroteRequestInfo) {
	hct.StartSpan("http.finishedWrite")
}

// shared code for POSTing to an endpoint, that consumes JSON, is zlib-
// compressed, that returns 202 on success, that has a small response
// action as a string used for statsd metric names and log messages emitted from
// this function - probably a static string for each callsite
// you can disable compression with compress=false for endpoints that don't
// support it
func PostHelper(ctx context.Context, httpClient *http.Client, stats *statsd.Client, tc *trace.Client, endpoint string, bodyObject interface{}, action string, compress bool, log *logrus.Logger) error {
	span, _ := trace.StartSpanFromContext(ctx, "")
	span.SetTag("action", action)
	defer span.ClientFinish(tc)

	// attach this field to all the logs we generate
	innerLogger := log.WithField("action", action)

	marshalStart := time.Now()
	var (
		bodyBuffer bytes.Buffer
		encoder    *json.Encoder
		compressor *zlib.Writer
	)
	if compress {
		compressor = zlib.NewWriter(&bodyBuffer)
		encoder = json.NewEncoder(compressor)
	} else {
		encoder = json.NewEncoder(&bodyBuffer)
	}
	if err := encoder.Encode(bodyObject); err != nil {
		stats.Count(action+".error_total", 1, []string{"cause:json"}, 1.0)
		innerLogger.WithError(err).Error("Could not render JSON")
		return err
	}
	if compress {
		// don't forget to flush leftover compressed bytes to the buffer
		if err := compressor.Close(); err != nil {
			stats.Count(action+".error_total", 1, []string{"cause:compress"}, 1.0)
			innerLogger.WithError(err).Error("Could not finalize compression")
			return err
		}
	}
	stats.TimeInMilliseconds(action+".duration_ns", float64(time.Since(marshalStart).Nanoseconds()), []string{"part:json"}, 1.0)

	// Len reports the unread length, so we have to record this before the
	// http client consumes it
	bodyLength := bodyBuffer.Len()
	stats.Histogram(action+".content_length_bytes", float64(bodyLength), nil, 1.0)

	req, err := http.NewRequest(http.MethodPost, endpoint, &bodyBuffer)

	if err != nil {
		stats.Count(action+".error_total", 1, []string{"cause:construct"}, 1.0)
		innerLogger.WithError(err).Error("Could not construct request")
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	if compress {
		req.Header.Set("Content-Encoding", "deflate")
	}

	// TODO: Use the HCT span to set the trace here.
	err = tracer.InjectRequest(span.Trace, req)
	if err != nil {
		stats.Count("veneur.opentracing.flush.inject.errors", 1, nil, 1.0)
		innerLogger.WithError(err).Error("Error injecting header")
	}

	// Attach a ClientTrace that emits metrics for all our HTTP bits. n.b. this
	// clienttrace can only measure _reused_ connections as the final `PutIdleConn`
	// callback is only invoked for connections returned to the idle pool.
	hct := NewHTTPClientTrace(span.Attach(ctx), stats, action)
	req = req.WithContext(httptrace.WithClientTrace(req.Context(), hct.GetClientTrace()))

	resp, err := httpClient.Do(req)
	if err != nil {
		if urlErr, ok := err.(*url.Error); ok {
			// if the error has the url in it, then retrieve the inner error
			// and ditch the url (which might contain secrets)
			err = urlErr.Err
		}
		stats.Count(action+".error_total", 1, []string{"cause:io"}, 1.0)
		// Log at Warn level instead of Error, because we don't want to create
		// Sentry events for these (they're only important in large numbers, and
		// we already have Datadog metrics for them)
		innerLogger.WithError(err).Warn("Could not execute request")
		return err
	}
	defer resp.Body.Close()

	responseBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		// this error is not fatal, since we only need the body for reporting
		// purposes
		stats.Count(action+".error_total", 1, []string{"cause:readresponse"}, 1.0)
		innerLogger.WithError(err).Error("Could not read response body")
	}
	resultLogger := innerLogger.WithFields(logrus.Fields{
		"endpoint":         endpoint,
		"request_length":   bodyLength,
		"request_headers":  req.Header,
		"status":           resp.Status,
		"response_headers": resp.Header,
		"response":         string(responseBody),
	})

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
		stats.Count(action+".error_total", 1, []string{fmt.Sprintf("cause:%d", resp.StatusCode)}, 1.0)
		resultLogger.WithError(err).Warn("Could not POST")
		return err
	}

	// make sure the error metric isn't sparse
	stats.Count(action+".error_total", 0, nil, 1.0)
	resultLogger.Debug("POSTed successfully")
	return nil
}
<commit_msg>Polish and docs<commit_after>package http

import (
	"bytes"
	"compress/zlib"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptrace"
	"net/url"
	"sync"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/sirupsen/logrus"
	"github.com/stripe/veneur/trace"
)

var tracer = trace.GlobalTracer

// HTTPClientTracer is a wrapper around a `net/http/httptrace` that handles
// proper tracing and metric emission.
type HTTPClientTracer struct {
	prefix      string
	stats       *statsd.Client
	traceClient *trace.Client
	mutex       *sync.Mutex
	ctx         context.Context
	currentSpan *trace.Span
}

// NewHTTPClientTrace makes a new HTTPClientTracer with new span created from
// the context.
func NewHTTPClientTrace(ctx context.Context, stats *statsd.Client, tc *trace.Client, prefix string) *HTTPClientTracer {
	span, _ := trace.StartSpanFromContext(ctx, "http.start")
	return &HTTPClientTracer{
		prefix:      prefix,
		stats:       stats,
		traceClient: tc,
		mutex:       &sync.Mutex{},
		ctx:         ctx,
		currentSpan: span,
	}
}

// GetClientTrace is a convenience to return a filled-in `ClientTrace`.
func (hct *HTTPClientTracer) GetClientTrace() *httptrace.ClientTrace {
	return &httptrace.ClientTrace{
		GotConn:              hct.GotConn,
		DNSStart:             hct.DNSStart,
		GotFirstResponseByte: hct.GotFirstResponseByte,
		ConnectStart:         hct.ConnectStart,
		WroteRequest:         hct.WroteRequest,
	}
}

// StartSpan is a convenience that replaces the current span in our
// tracer and flushes the outgoing one.
func (hct *HTTPClientTracer) StartSpan(name string) *trace.Span {
	hct.mutex.Lock()
	defer hct.mutex.Unlock()
	newSpan, _ := trace.StartSpanFromContext(hct.ctx, name)
	hct.currentSpan.ClientFinish(hct.traceClient)

	hct.currentSpan = newSpan
	return newSpan
}

//  GotConn signals that the connection — be it new or reused — was fetched
// from the pool.
func (hct *HTTPClientTracer) GotConn(info httptrace.GotConnInfo) {
	state := "new"
	if info.Reused {
		state = "reused"
	}
	hct.StartSpan(fmt.Sprintf("http.gotConnection.%s", state))

	hct.stats.Count(hct.prefix+".connections_used_total", 1, []string{fmt.Sprintf("state:%s", state)}, 1.0)
}

// DNSStart marks the beginning of the DNS lookup
func (hct *HTTPClientTracer) DNSStart(info httptrace.DNSStartInfo) {
	hct.StartSpan("http.resolvingDNS")
}

// GotFirstResponseByte marks us receiving our first byte
func (hct *HTTPClientTracer) GotFirstResponseByte() {
	hct.StartSpan("http.gotFirstByte")
}

// ConnectStart marks beginning of the connect process
func (hct *HTTPClientTracer) ConnectStart(network, addr string) {
	hct.StartSpan("http.connecting")
}

// WroteRequest marks the write being completed
func (hct *HTTPClientTracer) WroteRequest(info httptrace.WroteRequestInfo) {
	hct.StartSpan("http.finishedWrite")
}

// PostHelper is shared code for POSTing to an endpoint, that consumes JSON, is zlib-
// compressed, that returns 202 on success, that has a small response
// action as a string used for statsd metric names and log messages emitted from
// this function - probably a static string for each callsite
// you can disable compression with compress=false for endpoints that don't
// support it
func PostHelper(ctx context.Context, httpClient *http.Client, stats *statsd.Client, tc *trace.Client, endpoint string, bodyObject interface{}, action string, compress bool, log *logrus.Logger) error {
	span, _ := trace.StartSpanFromContext(ctx, "")
	span.SetTag("action", action)
	defer span.ClientFinish(tc)

	// attach this field to all the logs we generate
	innerLogger := log.WithField("action", action)

	marshalStart := time.Now()
	var (
		bodyBuffer bytes.Buffer
		encoder    *json.Encoder
		compressor *zlib.Writer
	)
	if compress {
		compressor = zlib.NewWriter(&bodyBuffer)
		encoder = json.NewEncoder(compressor)
	} else {
		encoder = json.NewEncoder(&bodyBuffer)
	}
	if err := encoder.Encode(bodyObject); err != nil {
		stats.Count(action+".error_total", 1, []string{"cause:json"}, 1.0)
		innerLogger.WithError(err).Error("Could not render JSON")
		return err
	}
	if compress {
		// don't forget to flush leftover compressed bytes to the buffer
		if err := compressor.Close(); err != nil {
			stats.Count(action+".error_total", 1, []string{"cause:compress"}, 1.0)
			innerLogger.WithError(err).Error("Could not finalize compression")
			return err
		}
	}
	stats.TimeInMilliseconds(action+".duration_ns", float64(time.Since(marshalStart).Nanoseconds()), []string{"part:json"}, 1.0)

	// Len reports the unread length, so we have to record this before the
	// http client consumes it
	bodyLength := bodyBuffer.Len()
	stats.Histogram(action+".content_length_bytes", float64(bodyLength), nil, 1.0)

	req, err := http.NewRequest(http.MethodPost, endpoint, &bodyBuffer)

	if err != nil {
		stats.Count(action+".error_total", 1, []string{"cause:construct"}, 1.0)
		innerLogger.WithError(err).Error("Could not construct request")
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	if compress {
		req.Header.Set("Content-Encoding", "deflate")
	}

	err = tracer.InjectRequest(span.Trace, req)
	if err != nil {
		stats.Count("veneur.opentracing.flush.inject.errors", 1, nil, 1.0)
		innerLogger.WithError(err).Error("Error injecting header")
	}

	// Attach a ClientTrace that emits metrics for all our HTTP bits. n.b. this
	// clienttrace can only measure _reused_ connections as the final `PutIdleConn`
	// callback is only invoked for connections returned to the idle pool.
	hct := NewHTTPClientTrace(span.Attach(ctx), stats, tc, action)
	req = req.WithContext(httptrace.WithClientTrace(req.Context(), hct.GetClientTrace()))

	resp, err := httpClient.Do(req)
	if err != nil {
		if urlErr, ok := err.(*url.Error); ok {
			// if the error has the url in it, then retrieve the inner error
			// and ditch the url (which might contain secrets)
			err = urlErr.Err
		}
		stats.Count(action+".error_total", 1, []string{"cause:io"}, 1.0)
		// Log at Warn level instead of Error, because we don't want to create
		// Sentry events for these (they're only important in large numbers, and
		// we already have Datadog metrics for them)
		innerLogger.WithError(err).Warn("Could not execute request")
		return err
	}
	defer resp.Body.Close()

	responseBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		// this error is not fatal, since we only need the body for reporting
		// purposes
		stats.Count(action+".error_total", 1, []string{"cause:readresponse"}, 1.0)
		innerLogger.WithError(err).Error("Could not read response body")
	}
	resultLogger := innerLogger.WithFields(logrus.Fields{
		"endpoint":         endpoint,
		"request_length":   bodyLength,
		"request_headers":  req.Header,
		"status":           resp.Status,
		"response_headers": resp.Header,
		"response":         string(responseBody),
	})

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
		stats.Count(action+".error_total", 1, []string{fmt.Sprintf("cause:%d", resp.StatusCode)}, 1.0)
		resultLogger.WithError(err).Warn("Could not POST")
		return err
	}

	// make sure the error metric isn't sparse
	stats.Count(action+".error_total", 0, nil, 1.0)
	resultLogger.Debug("POSTed successfully")
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

package brotli

import "io"
import "bufio"

// TODO(dsnet): Most of this logic is identical to compress/flate.
// Centralize common logic to compress/internal/prefix.

type byteReader interface {
	io.Reader
	io.ByteReader
}

type bitReader struct {
	rd      byteReader
	prefix  prefixDecoder // Local prefix decoder
	bufBits uint32        // Buffer to hold some bits
	numBits uint          // Number of valid bits in bufBits
	offset  int64         // Number of bytes read from the underlying reader
}

func (br *bitReader) Init(r io.Reader) {
	if rr, ok := r.(byteReader); ok {
		*br = bitReader{rd: rr}
	} else {
		*br = bitReader{rd: bufio.NewReader(r)}
	}
}

// Read reads up to len(buf) bytes into buf.
func (br *bitReader) Read(buf []byte) (int, error) {
	if br.numBits%8 > 0 {
		return 0, Error("unaligned byte read")
	}

	bufBase := buf
	for len(buf) > 0 {
		if br.numBits > 0 {
			buf[0] = byte(br.bufBits)
			br.bufBits >>= 8
			br.numBits -= 8
			buf = buf[1:]
		} else {
			cnt, err := br.rd.Read(buf)
			buf = buf[cnt:]
			br.offset += int64(cnt)
			if err != nil {
				return len(bufBase) - len(buf), err
			}
		}
	}
	return len(bufBase) - len(buf), nil
}

// ReadBits reads nb bits in LSB order from the underlying reader.
// If an IO error occurs, then it panics.
func (br *bitReader) ReadBits(nb uint) uint {
	for br.numBits < nb {
		c, err := br.rd.ReadByte()
		if err != nil {
			if err == io.EOF {
				err = io.ErrUnexpectedEOF
			}
			panic(err)
		}
		br.offset++
		br.bufBits |= uint32(c) << br.numBits
		br.numBits += 8
	}
	val := uint(br.bufBits & uint32(1<<nb-1))
	br.bufBits >>= nb
	br.numBits -= nb
	return val
}

// ReadPads reads 0-7 bits from the bit buffer to achieve byte-alignment.
func (br *bitReader) ReadPads() uint {
	nb := br.numBits % 8
	val := uint(br.bufBits & uint32(1<<nb-1))
	br.bufBits >>= nb
	br.numBits -= nb
	return val
}

// ReadSymbol reads the next prefix symbol using the provided prefixDecoder.
// If an IO error occurs, then it panics.
func (br *bitReader) ReadSymbol(pd *prefixDecoder) uint {
	if len(pd.chunks) == 0 {
		panic(ErrCorrupt) // Decode with empty tree
	}

	nb := uint(pd.minBits)
	for {
		for br.numBits < nb {
			// This section is an inlined version of the inner loop of ReadBits
			// for performance reasons.
			c, err := br.rd.ReadByte()
			if err != nil {
				if err == io.EOF {
					err = io.ErrUnexpectedEOF
				}
				panic(err)
			}
			br.offset++
			br.bufBits |= uint32(c) << br.numBits
			br.numBits += 8
		}
		chunk := pd.chunks[uint16(br.bufBits)&pd.chunkMask]
		nb = uint(chunk & prefixCountMask)
		if nb > uint(pd.chunkBits) {
			linkIdx := chunk >> prefixCountBits
			chunk = pd.links[linkIdx][uint16(br.bufBits>>pd.chunkBits)&pd.linkMask]
			nb = uint(chunk & prefixCountMask)
		}
		if nb <= br.numBits {
			br.bufBits >>= nb
			br.numBits -= nb
			return uint(chunk >> prefixCountBits)
		}
	}
}
<commit_msg>brotli: add support for parsing simple prefix codes<commit_after>// Copyright 2015, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

package brotli

import "io"
import "bufio"

// TODO(dsnet): Most of this logic is identical to compress/flate.
// Centralize common logic to compress/internal/prefix.

type byteReader interface {
	io.Reader
	io.ByteReader
}

type bitReader struct {
	rd      byteReader
	prefix  prefixDecoder // Local prefix decoder
	bufBits uint32        // Buffer to hold some bits
	numBits uint          // Number of valid bits in bufBits
	offset  int64         // Number of bytes read from the underlying reader
}

func (br *bitReader) Init(r io.Reader) {
	if rr, ok := r.(byteReader); ok {
		*br = bitReader{rd: rr}
	} else {
		*br = bitReader{rd: bufio.NewReader(r)}
	}
}

// Read reads up to len(buf) bytes into buf.
func (br *bitReader) Read(buf []byte) (int, error) {
	if br.numBits%8 > 0 {
		return 0, Error("unaligned byte read")
	}

	bufBase := buf
	for len(buf) > 0 {
		if br.numBits > 0 {
			buf[0] = byte(br.bufBits)
			br.bufBits >>= 8
			br.numBits -= 8
			buf = buf[1:]
		} else {
			cnt, err := br.rd.Read(buf)
			buf = buf[cnt:]
			br.offset += int64(cnt)
			if err != nil {
				return len(bufBase) - len(buf), err
			}
		}
	}
	return len(bufBase) - len(buf), nil
}

// ReadBits reads nb bits in LSB order from the underlying reader.
// If an IO error occurs, then it panics.
func (br *bitReader) ReadBits(nb uint) uint {
	for br.numBits < nb {
		c, err := br.rd.ReadByte()
		if err != nil {
			if err == io.EOF {
				err = io.ErrUnexpectedEOF
			}
			panic(err)
		}
		br.offset++
		br.bufBits |= uint32(c) << br.numBits
		br.numBits += 8
	}
	val := uint(br.bufBits & uint32(1<<nb-1))
	br.bufBits >>= nb
	br.numBits -= nb
	return val
}

// ReadPads reads 0-7 bits from the bit buffer to achieve byte-alignment.
func (br *bitReader) ReadPads() uint {
	nb := br.numBits % 8
	val := uint(br.bufBits & uint32(1<<nb-1))
	br.bufBits >>= nb
	br.numBits -= nb
	return val
}

// ReadSymbol reads the next prefix symbol using the provided prefixDecoder.
// If an IO error occurs, then it panics.
func (br *bitReader) ReadSymbol(pd *prefixDecoder) uint {
	if len(pd.chunks) == 0 {
		panic(ErrCorrupt) // Decode with empty tree
	}

	nb := uint(pd.minBits)
	for {
		for br.numBits < nb {
			// This section is an inlined version of the inner loop of ReadBits
			// for performance reasons.
			c, err := br.rd.ReadByte()
			if err != nil {
				if err == io.EOF {
					err = io.ErrUnexpectedEOF
				}
				panic(err)
			}
			br.offset++
			br.bufBits |= uint32(c) << br.numBits
			br.numBits += 8
		}
		chunk := pd.chunks[uint16(br.bufBits)&pd.chunkMask]
		nb = uint(chunk & prefixCountMask)
		if nb > uint(pd.chunkBits) {
			linkIdx := chunk >> prefixCountBits
			chunk = pd.links[linkIdx][uint16(br.bufBits>>pd.chunkBits)&pd.linkMask]
			nb = uint(chunk & prefixCountMask)
		}
		if nb <= br.numBits {
			br.bufBits >>= nb
			br.numBits -= nb
			return uint(chunk >> prefixCountBits)
		}
	}
}

// ReadPrefixCode reads the prefix definition from the stream and initializes
// the provided prefixDecoder.
func (br *bitReader) ReadPrefixCode(pd *prefixDecoder, numSyms int) {
	hskip := int(br.ReadBits(2))
	if hskip == 1 {
		br.readSimplePrefixCode(pd, numSyms)
	} else {
		br.readComplexPrefixCode(pd, numSyms, hskip)
	}
}

// readSimplePrefixCode reads the prefix code according to RFC section 3.4.
func (br *bitReader) readSimplePrefixCode(pd *prefixDecoder, numSyms int) {
	// TODO(dsnet): Test the following edge cases:
	// * Re-used symbol
	// * Out-of-order symbols
	// * Excessively large symbol
	// * Test each of the simple trees
	var codes [4]prefixCode
	nsym := int(br.ReadBits(2)) + 1
	clen := neededBits(uint16(numSyms))
	for i := 0; i < nsym; i++ {
		codes[i].sym = uint16(br.ReadBits(clen))
	}

	var copyLens = func(lens []uint8) {
		for i := 0; i < nsym; i++ {
			codes[i].len = lens[i]
		}
	}
	var compareSwap = func(i, j int) {
		if codes[i].sym > codes[j].sym {
			codes[i], codes[j] = codes[j], codes[i]
		}
	}

	switch nsym {
	case 1:
		copyLens(simpleLens1[:])
	case 2:
		copyLens(simpleLens2[:])
		compareSwap(0, 1)
	case 3:
		copyLens(simpleLens3[:])
		compareSwap(0, 1)
		compareSwap(1, 2)
		compareSwap(0, 2)
	case 4:
		if tsel := br.ReadBits(1) == 1; !tsel {
			copyLens(simpleLens4a[:])
		} else {
			copyLens(simpleLens4b[:])
		}
		compareSwap(0, 1)
		compareSwap(2, 3)
		compareSwap(0, 2)
		compareSwap(1, 3)
		compareSwap(1, 2)
	}
	if int(codes[nsym-1].sym) >= numSyms {
		panic(ErrCorrupt) // Symbol goes beyond range of alphabet
	}
	pd.Init(codes[:nsym], true)
}

// readComplexPrefixCode reads the prefix code according to RFC section 3.5.
func (br *bitReader) readComplexPrefixCode(pd *prefixDecoder, numSyms, hskip int) {
	// TODO(dsnet)
}

// Read the context map according to RFC section 7.3.
func (br *bitReader) ReadContextMap(cm []uint8, numTrees int) {
	// TODO(dsnet)
}
<|endoftext|>


<commit_before>package jmbg

import (
	"github.com/dvrkps/valida/internal/digits"
	"github.com/dvrkps/valida/internal/validate"
)

func OK(in string) bool {
	digs, ok := digits.ParseDigits([]byte(in))
	if !ok {
		return false
	}

	const (
		noDigits = 12
		coef2    = 2
		coef7    = 7
		coef8    = 8
	)

	coef := coef7

	zzz := 0
	for _, d := range digs.First(noDigits) {
		zzz += d * coef

		if coef == coef2 {
			coef = coef8
		}
		coef--
	}

	return validate.IsValidJMBG(zzz, digs.Last())
}
<commit_msg>internal/jmbg: use check.OK<commit_after>package jmbg

import (
	"github.com/dvrkps/valida/internal/check"
	"github.com/dvrkps/valida/internal/digits"
)

func OK(in string) bool {
	digs, ok := digits.ParseDigits([]byte(in))
	if !ok {
		return false
	}

	const (
		noDigits = 12
		coef2    = 2
		coef7    = 7
		coef8    = 8
	)

	coef := coef7

	zzz := 0
	for _, d := range digs.First(noDigits) {
		zzz += d * coef

		if coef == coef2 {
			coef = coef8
		}
		coef--
	}

	const wantRem = 0

	return check.OK(zzz, digs.Last(), wantRem)
}
<|endoftext|>