<commit_before>// This file abstracts & exposes persistent volume provisioner features. All
// maya api server's persistent volume provisioners need to implement these
// contracts.
package provisioners

import (
	"github.com/openebs/maya/types/v1"
)

// VolumeInterface abstracts the persistent volume features of any persistent
// volume provisioner.
//
// NOTE:
//    maya api server can make use of any persistent volume provisioner & execute
// corresponding volume related operations.
type VolumeInterface interface {
	// Label assigned against this persistent volume provisioner
	Label() string

	// Name of the persistent volume provisioner
	Name() string

	// Profile will set the persistent volume provisioner's profile
	//
	// NOTE:
	//    Will return false if profile is not supported by the persistent
	// volume provisioner.
	//
	// NOTE:
	//    This is used to set the persistent volume provisioner profile lazily
	// i.e. much after the initialization of persistent volume provisioner instance.
	// It is assumed that persistent volume claim will be available at the time of
	// invoking this method.
	Profile(*v1.Volume) (bool, error)

	// Remover gets the instance capable of deleting volumes w.r.t this
	// persistent volume provisioner.
	//
	// Note:
	//    Will return false if deletion of volumes is not supported by the
	// persistent volume provisioner.
	Remover() (Remover, bool, error)

	// Reader gets the instance capable of providing persistent volume information
	// w.r.t this persistent volume provisioner.
	//
	// Note:
	//    Will return false if providing persistent volume information is not
	// supported by this persistent volume provisioner.
	Reader() (Reader, bool)

	// Adder gets the instance capable of creating a persistent volume
	// w.r.t this persistent volume provisioner.
	//
	// Note:
	//    Will return false if creating persistent volume is not
	// supported by this persistent volume provisioner.
	Adder() (Adder, bool)

	// Lister gets the instance capable of listing persistent volumes
	// w.r.t this persistent volume provisioner.
	//
	// Note:
	//    Will return false if listing persistent volumes is not
	// supported by this persistent volume provisioner.
	Lister() (Lister, bool, error)
}

// Lister interface abstracts listing of persistent volumes from a persistent
// volume provisioner.
type Lister interface {
	// List fetches a collection of persistent volumes created by this volume
	// provisioner
	List() (*v1.VolumeList, error)
}

// Reader interface abstracts fetching of persistent volume related information
// from a persistent volume provisioner.
type Reader interface {
	// Read fetches the volume details from the persistent volume
	// provisioner.
	Read(*v1.Volume) (*v1.Volume, error)
}

// Adder interface abstracts creation of persistent volume from a persistent
// volume provisioner.
type Adder interface {
	// Add creates a new persistent volume
	Add(*v1.Volume) (*v1.Volume, error)
}

// Remover interface abstracts deletion of volume of a persistent volume
// provisioner.
type Remover interface {
	// Delete tries to delete a volume of a persistent volume provisioner.
	Remove() (bool, error)
}
<commit_msg>Fixed linting in volume.go<commit_after>// This file abstracts & exposes persistent volume provisioner features. All
// maya api server's persistent volume provisioners need to implement these
// contracts.

package provisioners

import (
	"github.com/openebs/maya/types/v1"
)

// VolumeInterface abstracts the persistent volume features of any persistent
// volume provisioner.
//
// NOTE:
//    maya api server can make use of any persistent volume provisioner & execute
// corresponding volume related operations.
type VolumeInterface interface {
	// Label assigned against this persistent volume provisioner
	Label() string

	// Name of the persistent volume provisioner
	Name() string

	// Profile will set the persistent volume provisioner's profile
	//
	// NOTE:
	//    Will return false if profile is not supported by the persistent
	// volume provisioner.
	//
	// NOTE:
	//    This is used to set the persistent volume provisioner profile lazily
	// i.e. much after the initialization of persistent volume provisioner instance.
	// It is assumed that persistent volume claim will be available at the time of
	// invoking this method.
	Profile(*v1.Volume) (bool, error)

	// Remover gets the instance capable of deleting volumes w.r.t this
	// persistent volume provisioner.
	//
	// Note:
	//    Will return false if deletion of volumes is not supported by the
	// persistent volume provisioner.
	Remover() (Remover, bool, error)

	// Reader gets the instance capable of providing persistent volume information
	// w.r.t this persistent volume provisioner.
	//
	// Note:
	//    Will return false if providing persistent volume information is not
	// supported by this persistent volume provisioner.
	Reader() (Reader, bool)

	// Adder gets the instance capable of creating a persistent volume
	// w.r.t this persistent volume provisioner.
	//
	// Note:
	//    Will return false if creating persistent volume is not
	// supported by this persistent volume provisioner.
	Adder() (Adder, bool)

	// Lister gets the instance capable of listing persistent volumes
	// w.r.t this persistent volume provisioner.
	//
	// Note:
	//    Will return false if listing persistent volumes is not
	// supported by this persistent volume provisioner.
	Lister() (Lister, bool, error)
}

// Lister interface abstracts listing of persistent volumes from a persistent
// volume provisioner.
type Lister interface {
	// List fetches a collection of persistent volumes created by this volume
	// provisioner
	List() (*v1.VolumeList, error)
}

// Reader interface abstracts fetching of persistent volume related information
// from a persistent volume provisioner.
type Reader interface {
	// Read fetches the volume details from the persistent volume
	// provisioner.
	Read(*v1.Volume) (*v1.Volume, error)
}

// Adder interface abstracts creation of persistent volume from a persistent
// volume provisioner.
type Adder interface {
	// Add creates a new persistent volume
	Add(*v1.Volume) (*v1.Volume, error)
}

// Remover interface abstracts deletion of volume of a persistent volume
// provisioner.
type Remover interface {
	// Delete tries to delete a volume of a persistent volume provisioner.
	Remove() (bool, error)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package drive

import (
	"fmt"

	"github.com/odeke-em/log"
	drive "google.golang.org/api/drive/v2"
)

const Version = "0.3.2"

const (
	Barely = iota
	AlmostExceeded
	HalfwayExceeded
	Exceeded
	Unknown
)

const (
	AboutNone = 1 << iota
	AboutQuota
	AboutFileSizes
	AboutFeatures
)

func (g *Commands) About(mask int) (err error) {
	if mask == AboutNone {
		return nil
	}

	about, err := g.rem.About()
	if err != nil {
		return err
	}
	printSummary(g.log, about, mask)

	return nil
}

func quotaRequested(mask int) bool {
	return (mask & AboutQuota) != 0
}

func fileSizesRequested(mask int) bool {
	return (mask & AboutFileSizes) != 0
}

func featuresRequested(mask int) bool {
	return (mask & AboutFeatures) != 0
}

func printSummary(logy *log.Logger, about *drive.About, mask int) {
	if quotaRequested(mask) {
		quotaInformation(logy, about)
	}
	if fileSizesRequested(mask) {
		fileSizesInfo(logy, about)
	}

	if featuresRequested(mask) {
		featuresInformation(logy, about)
	}
}

func fileSizesInfo(logy *log.Logger, about *drive.About) {
	if len(about.MaxUploadSizes) >= 1 {
		logy.Logln("\n* Maximum upload sizes per file type *")
		logy.Logf("%-50s %-20s\n", "FileType", "Size")
		for _, uploadInfo := range about.MaxUploadSizes {
			logy.Logf("%-50s %-20s\n", uploadInfo.Type, prettyBytes(uploadInfo.Size))
		}
		logy.Logln()
	}
	return
}

func featuresInformation(logy *log.Logger, about *drive.About) {
	if len(about.Features) >= 1 {
		logy.Logf("%-30s %-30s\n", "Feature", "Request limit (queries/second)")
		for _, feature := range about.Features {
			if feature.FeatureName == "" {
				continue
			}
			logy.Logf("%-30s %-30f\n", feature.FeatureName, feature.FeatureRate)
		}
		logy.Logln()
	}
}

func quotaInformation(logy *log.Logger, about *drive.About) {
	freeBytes := about.QuotaBytesTotal - about.QuotaBytesUsed

	logy.Logf(
		"Name: %s\nAccount type:\t%s\nBytes Used:\t%-20d (%s)\n"+
			"Bytes Free:\t%-20d (%s)\nBytes InTrash:\t%-20d (%s)\n"+
			"Total Bytes:\t%-20d (%s)\n",
		about.Name, about.QuotaType,
		about.QuotaBytesUsed, prettyBytes(about.QuotaBytesUsed),
		freeBytes, prettyBytes(freeBytes),
		about.QuotaBytesUsedInTrash, prettyBytes(about.QuotaBytesUsedInTrash),
		about.QuotaBytesTotal, prettyBytes(about.QuotaBytesTotal))

	if len(about.QuotaBytesByService) >= 1 {
		logy.Logln("\n* Space used by Google Services *")
		logy.Logf("%-36s %-36s\n", "Service", "Bytes")
		for _, quotaService := range about.QuotaBytesByService {
			logy.Logf("%-36s %-36s\n", quotaService.ServiceName, prettyBytes(quotaService.BytesUsed))
		}
		logy.Logf("%-36s %-36s\n", "Space used by all Google Apps",
			prettyBytes(about.QuotaBytesUsedAggregate))
	}
	logy.Logln()
}

func (g *Commands) QuotaStatus(query int64) (status int, err error) {
	if query < 0 {
		return Unknown, err
	}

	about, err := g.rem.About()
	if err != nil {
		return Unknown, err
	}

	// Sanity check
	if about.QuotaBytesTotal < 1 {
		return Unknown, fmt.Errorf("QuotaBytesTotal < 1")
	}

	toBeUsed := query + about.QuotaBytesUsed
	if toBeUsed >= about.QuotaBytesTotal {
		return Exceeded, nil
	}

	percentage := float64(toBeUsed) / float64(about.QuotaBytesTotal)
	if percentage < 0.5 {
		return Barely, nil
	}
	if percentage < 0.8 {
		return HalfwayExceeded, nil
	}
	return AlmostExceeded, nil
}
<commit_msg>version bump to v0.3.3<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package drive

import (
	"fmt"

	"github.com/odeke-em/log"
	drive "google.golang.org/api/drive/v2"
)

const Version = "0.3.3"

const (
	Barely = iota
	AlmostExceeded
	HalfwayExceeded
	Exceeded
	Unknown
)

const (
	AboutNone = 1 << iota
	AboutQuota
	AboutFileSizes
	AboutFeatures
)

func (g *Commands) About(mask int) (err error) {
	if mask == AboutNone {
		return nil
	}

	about, err := g.rem.About()
	if err != nil {
		return err
	}
	printSummary(g.log, about, mask)

	return nil
}

func quotaRequested(mask int) bool {
	return (mask & AboutQuota) != 0
}

func fileSizesRequested(mask int) bool {
	return (mask & AboutFileSizes) != 0
}

func featuresRequested(mask int) bool {
	return (mask & AboutFeatures) != 0
}

func printSummary(logy *log.Logger, about *drive.About, mask int) {
	if quotaRequested(mask) {
		quotaInformation(logy, about)
	}
	if fileSizesRequested(mask) {
		fileSizesInfo(logy, about)
	}

	if featuresRequested(mask) {
		featuresInformation(logy, about)
	}
}

func fileSizesInfo(logy *log.Logger, about *drive.About) {
	if len(about.MaxUploadSizes) >= 1 {
		logy.Logln("\n* Maximum upload sizes per file type *")
		logy.Logf("%-50s %-20s\n", "FileType", "Size")
		for _, uploadInfo := range about.MaxUploadSizes {
			logy.Logf("%-50s %-20s\n", uploadInfo.Type, prettyBytes(uploadInfo.Size))
		}
		logy.Logln()
	}
	return
}

func featuresInformation(logy *log.Logger, about *drive.About) {
	if len(about.Features) >= 1 {
		logy.Logf("%-30s %-30s\n", "Feature", "Request limit (queries/second)")
		for _, feature := range about.Features {
			if feature.FeatureName == "" {
				continue
			}
			logy.Logf("%-30s %-30f\n", feature.FeatureName, feature.FeatureRate)
		}
		logy.Logln()
	}
}

func quotaInformation(logy *log.Logger, about *drive.About) {
	freeBytes := about.QuotaBytesTotal - about.QuotaBytesUsed

	logy.Logf(
		"Name: %s\nAccount type:\t%s\nBytes Used:\t%-20d (%s)\n"+
			"Bytes Free:\t%-20d (%s)\nBytes InTrash:\t%-20d (%s)\n"+
			"Total Bytes:\t%-20d (%s)\n",
		about.Name, about.QuotaType,
		about.QuotaBytesUsed, prettyBytes(about.QuotaBytesUsed),
		freeBytes, prettyBytes(freeBytes),
		about.QuotaBytesUsedInTrash, prettyBytes(about.QuotaBytesUsedInTrash),
		about.QuotaBytesTotal, prettyBytes(about.QuotaBytesTotal))

	if len(about.QuotaBytesByService) >= 1 {
		logy.Logln("\n* Space used by Google Services *")
		logy.Logf("%-36s %-36s\n", "Service", "Bytes")
		for _, quotaService := range about.QuotaBytesByService {
			logy.Logf("%-36s %-36s\n", quotaService.ServiceName, prettyBytes(quotaService.BytesUsed))
		}
		logy.Logf("%-36s %-36s\n", "Space used by all Google Apps",
			prettyBytes(about.QuotaBytesUsedAggregate))
	}
	logy.Logln()
}

func (g *Commands) QuotaStatus(query int64) (status int, err error) {
	if query < 0 {
		return Unknown, err
	}

	about, err := g.rem.About()
	if err != nil {
		return Unknown, err
	}

	// Sanity check
	if about.QuotaBytesTotal < 1 {
		return Unknown, fmt.Errorf("QuotaBytesTotal < 1")
	}

	toBeUsed := query + about.QuotaBytesUsed
	if toBeUsed >= about.QuotaBytesTotal {
		return Exceeded, nil
	}

	percentage := float64(toBeUsed) / float64(about.QuotaBytesTotal)
	if percentage < 0.5 {
		return Barely, nil
	}
	if percentage < 0.8 {
		return HalfwayExceeded, nil
	}
	return AlmostExceeded, nil
}
<|endoftext|>


<commit_before>package pki

import (
	"fmt"

	"github.com/hashicorp/vault/helper/certutil"
	"github.com/hashicorp/vault/logical"
	"github.com/hashicorp/vault/logical/framework"
)

func pathRevoke(b *backend) *framework.Path {
	return &framework.Path{
		Pattern: `revoke`,
		Fields: map[string]*framework.FieldSchema{
			"serial_number": &framework.FieldSchema{
				Type: framework.TypeString,
				Description: `Certificate serial number, in colon- or
hyphen-separated octal`,
			},
		},

		Callbacks: map[logical.Operation]framework.OperationFunc{
			logical.UpdateOperation: b.pathRevokeWrite,
		},

		HelpSynopsis:    pathRevokeHelpSyn,
		HelpDescription: pathRevokeHelpDesc,
	}
}

func pathRotateCRL(b *backend) *framework.Path {
	return &framework.Path{
		Pattern: `crl/rotate`,

		Callbacks: map[logical.Operation]framework.OperationFunc{
			logical.ReadOperation: b.pathRotateCRLRead,
		},

		HelpSynopsis:    pathRotateCRLHelpSyn,
		HelpDescription: pathRotateCRLHelpDesc,
	}
}

func (b *backend) pathRevokeWrite(req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	serial := data.Get("serial_number").(string)
	if len(serial) == 0 {
		return logical.ErrorResponse("The serial number must be provided"), nil
	}

	b.revokeStorageLock.Lock()
	defer b.revokeStorageLock.Unlock()

	return revokeCert(b, req, serial)
}

func (b *backend) pathRotateCRLRead(req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	b.revokeStorageLock.RLock()
	defer b.revokeStorageLock.RUnlock()

	crlErr := buildCRL(b, req)
	switch crlErr.(type) {
	case certutil.UserError:
		return logical.ErrorResponse(fmt.Sprintf("Error during CRL building: %s", crlErr)), nil
	case certutil.InternalError:
		return nil, fmt.Errorf("Error encountered during CRL building: %s", crlErr)
	default:
		return &logical.Response{
			Data: map[string]interface{}{
				"success": true,
			},
		}, nil
	}
}

const pathRevokeHelpSyn = `
Revoke a certificate by serial number.
`

const pathRevokeHelpDesc = `
This allows certificates to be revoked using its serial number. A root token is required.
`

const pathRotateCRLHelpSyn = `
Force a rebuild of the CRL.
`

const pathRotateCRLHelpDesc = `
Force a rebuild of the CRL. This can be used to remove expired certificates from it if no certificates have been revoked. A root token is required.
`
<commit_msg>Sanitize serial number in revocation path.<commit_after>package pki

import (
	"fmt"
	"strings"

	"github.com/hashicorp/vault/helper/certutil"
	"github.com/hashicorp/vault/logical"
	"github.com/hashicorp/vault/logical/framework"
)

func pathRevoke(b *backend) *framework.Path {
	return &framework.Path{
		Pattern: `revoke`,
		Fields: map[string]*framework.FieldSchema{
			"serial_number": &framework.FieldSchema{
				Type: framework.TypeString,
				Description: `Certificate serial number, in colon- or
hyphen-separated octal`,
			},
		},

		Callbacks: map[logical.Operation]framework.OperationFunc{
			logical.UpdateOperation: b.pathRevokeWrite,
		},

		HelpSynopsis:    pathRevokeHelpSyn,
		HelpDescription: pathRevokeHelpDesc,
	}
}

func pathRotateCRL(b *backend) *framework.Path {
	return &framework.Path{
		Pattern: `crl/rotate`,

		Callbacks: map[logical.Operation]framework.OperationFunc{
			logical.ReadOperation: b.pathRotateCRLRead,
		},

		HelpSynopsis:    pathRotateCRLHelpSyn,
		HelpDescription: pathRotateCRLHelpDesc,
	}
}

func (b *backend) pathRevokeWrite(req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	serial := data.Get("serial_number").(string)
	if len(serial) == 0 {
		return logical.ErrorResponse("The serial number must be provided"), nil
	}

	// We store and identify by lowercase colon-separated hex, but other
	// utilities use dashes and/or uppercase, so normalize
	serial = strings.Replace(strings.ToLower(serial), "-", ":", -1)

	b.revokeStorageLock.Lock()
	defer b.revokeStorageLock.Unlock()

	return revokeCert(b, req, serial)
}

func (b *backend) pathRotateCRLRead(req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	b.revokeStorageLock.RLock()
	defer b.revokeStorageLock.RUnlock()

	crlErr := buildCRL(b, req)
	switch crlErr.(type) {
	case certutil.UserError:
		return logical.ErrorResponse(fmt.Sprintf("Error during CRL building: %s", crlErr)), nil
	case certutil.InternalError:
		return nil, fmt.Errorf("Error encountered during CRL building: %s", crlErr)
	default:
		return &logical.Response{
			Data: map[string]interface{}{
				"success": true,
			},
		}, nil
	}
}

const pathRevokeHelpSyn = `
Revoke a certificate by serial number.
`

const pathRevokeHelpDesc = `
This allows certificates to be revoked using its serial number. A root token is required.
`

const pathRotateCRLHelpSyn = `
Force a rebuild of the CRL.
`

const pathRotateCRLHelpDesc = `
Force a rebuild of the CRL. This can be used to remove expired certificates from it if no certificates have been revoked. A root token is required.
`
<|endoftext|>


<commit_before>// +build e2e

/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"strconv"
	"testing"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"knative.dev/pkg/test/logstream"
	"knative.dev/serving/pkg/apis/autoscaling"
	"knative.dev/serving/pkg/apis/serving/v1alpha1"
	"knative.dev/serving/test"
	v1a1test "knative.dev/serving/test/v1alpha1"
)

func TestMinScale(t *testing.T) {
	t.Parallel()
	cancel := logstream.Start(t)
	defer cancel()

	const minScale = 4

	clients := Setup(t)

	names := test.ResourceNames{
		Config: test.ObjectNameForTest(t),
		Image:  "helloworld",
	}

	if _, err := v1a1test.CreateConfiguration(t, clients, names, func(cfg *v1alpha1.Configuration) {
		if cfg.Spec.Template.Annotations == nil {
			cfg.Spec.Template.Annotations = make(map[string]string)
		}

		cfg.Spec.Template.Annotations[autoscaling.MinScaleAnnotationKey] = strconv.Itoa(minScale)

	}); err != nil {
		t.Fatalf("Failed to create Configuration: %v", err)
	}

	test.CleanupOnInterrupt(func() { test.TearDown(clients, names) })
	defer test.TearDown(clients, names)

	// Wait for the Config have a LatestCreatedRevisionName
	if err := v1a1test.WaitForConfigurationState(clients.ServingAlphaClient, names.Config, v1a1test.ConfigurationHasCreatedRevision, "ConfigurationHasCreatedRevision"); err != nil {
		t.Fatalf("The Configuration %q does not have a LatestCreatedRevisionName: %v", names.Config, err)
	}

	config, err := clients.ServingAlphaClient.Configs.Get(names.Config, metav1.GetOptions{})
	if err != nil {
		t.Fatalf("Failed to get Configuration after it was seen to be live: %v", err)
	}

	revName := config.Status.LatestCreatedRevisionName

	if err = v1a1test.WaitForRevisionState(clients.ServingAlphaClient, revName, v1a1test.IsRevisionReady, "RevisionIsReady"); err != nil {
		t.Fatalf("The Revision %q did not become ready: %v", revName, err)
	}

	deployment, err := clients.KubeClient.Kube.ExtensionsV1beta1().Deployments(test.ServingNamespace).Get(revName+"-deployment", metav1.GetOptions{})
	if err != nil {
		t.Fatalf("Failed to get Deployment for Revision %s, err: %v", revName, err)
	}

	if deployment.Status.AvailableReplicas < int32(minScale) {
		t.Fatalf("Reported ready with %d replicas when minScale was %d", deployment.Status.AvailableReplicas, minScale)
	}
}
<commit_msg>Remove the usage of the extensions API for deployments. (#4760)<commit_after>// +build e2e

/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"strconv"
	"testing"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"knative.dev/pkg/test/logstream"
	"knative.dev/serving/pkg/apis/autoscaling"
	"knative.dev/serving/pkg/apis/serving/v1alpha1"
	"knative.dev/serving/test"
	v1a1test "knative.dev/serving/test/v1alpha1"
)

func TestMinScale(t *testing.T) {
	t.Parallel()
	cancel := logstream.Start(t)
	defer cancel()

	const minScale = 4

	clients := Setup(t)

	names := test.ResourceNames{
		Config: test.ObjectNameForTest(t),
		Image:  "helloworld",
	}

	if _, err := v1a1test.CreateConfiguration(t, clients, names, func(cfg *v1alpha1.Configuration) {
		if cfg.Spec.Template.Annotations == nil {
			cfg.Spec.Template.Annotations = make(map[string]string)
		}

		cfg.Spec.Template.Annotations[autoscaling.MinScaleAnnotationKey] = strconv.Itoa(minScale)

	}); err != nil {
		t.Fatalf("Failed to create Configuration: %v", err)
	}

	test.CleanupOnInterrupt(func() { test.TearDown(clients, names) })
	defer test.TearDown(clients, names)

	// Wait for the Config have a LatestCreatedRevisionName
	if err := v1a1test.WaitForConfigurationState(clients.ServingAlphaClient, names.Config, v1a1test.ConfigurationHasCreatedRevision, "ConfigurationHasCreatedRevision"); err != nil {
		t.Fatalf("The Configuration %q does not have a LatestCreatedRevisionName: %v", names.Config, err)
	}

	config, err := clients.ServingAlphaClient.Configs.Get(names.Config, metav1.GetOptions{})
	if err != nil {
		t.Fatalf("Failed to get Configuration after it was seen to be live: %v", err)
	}

	revName := config.Status.LatestCreatedRevisionName

	if err = v1a1test.WaitForRevisionState(clients.ServingAlphaClient, revName, v1a1test.IsRevisionReady, "RevisionIsReady"); err != nil {
		t.Fatalf("The Revision %q did not become ready: %v", revName, err)
	}

	deployment, err := clients.KubeClient.Kube.AppsV1().Deployments(test.ServingNamespace).Get(revName+"-deployment", metav1.GetOptions{})
	if err != nil {
		t.Fatalf("Failed to get Deployment for Revision %s, err: %v", revName, err)
	}

	if deployment.Status.AvailableReplicas < int32(minScale) {
		t.Fatalf("Reported ready with %d replicas when minScale was %d", deployment.Status.AvailableReplicas, minScale)
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 Bitcartel Software. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.

// +build darwin

// Package xattr wraps OS X functions to manipulate the extended attributes of a file, directory and symbolic link.
//
// The functions are a wrapper around xattr system calls.
// One difference is that position and size arguments are not implemented, since the caller can create slices from returned data.
package xattr

//#include <stdlib.h>
//#include <sys/xattr.h>
import "C"
import "unsafe"
import "bytes"

// Option flags to Xattr wrapping those in <sys/xattr.h>
const (
	XATTR_NOFOLLOW          int = C.XATTR_NOFOLLOW        /* Don't follow symbolic links */
	XATTR_CREATE            int = C.XATTR_CREATE          /* set the value, fail if attr already exists */
	XATTR_REPLACE           int = C.XATTR_REPLACE         /* set the value, fail if attr does not exist */
	XATTR_SHOWCOMPRESSION   int = C.XATTR_SHOWCOMPRESSION /* option for f/getxattr() and f/listxattr() to expose the HFS Compression extended attributes */
	XATTR_NOSECURITY        int = C.XATTR_NOSECURITY      /* Set this to bypass authorization checking (eg. if doing auth-related work) */
	XATTR_NODEFAULT         int = C.XATTR_NODEFAULT       /* Set this to bypass the default extended attribute file (dot-underscore file) */
	XATTR_MAXNAMELEN            = 127
	XATTR_FINDERINFO_NAME       = "com.apple.FinderInfo"
	XATTR_RESOURCEFORK_NAME     = "com.apple.ResourceFork"
	XATTR_MAXSIZE               = (64 * 1024 * 1024)
)

// Removexattr will remove the named attribute from file at path
func Removexattr(path string, name string, options int) error {
	p := C.CString(path)
	n := C.CString(name)
	defer C.free(unsafe.Pointer(p))
	defer C.free(unsafe.Pointer(n))
	_, err := C.removexattr(p, n, C.int(options))
	if err != nil {
		return err
	}
	return nil
}

// Setxattr will set the value for a named attribute on file at path
func Setxattr(path string, name string, data []byte, options int) error {
	p := C.CString(path)
	n := C.CString(name)
	defer C.free(unsafe.Pointer(p))
	defer C.free(unsafe.Pointer(n))

	_, err := C.setxattr(p, n, unsafe.Pointer(&data[0]), C.size_t(len(data)), 0, C.int(options))
	if err != nil {
		return err
	}
	return nil
}

// Getxattr will return the value for the named attribute from a file at path
func Getxattr(path string, name string, options int) ([]byte, error) {
	p := C.CString(path)
	n := C.CString(name)
	defer C.free(unsafe.Pointer(p))
	defer C.free(unsafe.Pointer(n))

	// get size of data for attribute, type _Ctype_ssize_t
	attrsize, err := C.getxattr(p, n, nil, 0, 0, 0)
	if err != nil {
		return nil, err
	}
	size := int(attrsize)

	// get data for attribute
	buf := make([]byte, size)
	x, err := C.getxattr(p, n, unsafe.Pointer(&buf[0]), C.size_t(size), 0, C.int(options))
	if err != nil {
		return nil, err
	}

	return buf[:x], nil
}

// Listxattr will return a list of attribute names found for a file at path
func Listxattr(path string, options int) ([]string, error) {
	p := C.CString(path)
	defer C.free(unsafe.Pointer(p))

	// get size of buffer needed for attribute names, type is _Ctype_ssize_t
	listsize, err := C.listxattr(p, nil, 0, 0)
	if err != nil {
		return nil, err
	}

	// get attribute names
	size := int(listsize)
	buf := make([]byte, size)
	_, err = C.listxattr(p, (*C.char)(unsafe.Pointer(&buf[0])), C.size_t(size), C.int(options))
	if err != nil {
		return nil, err
	}

	// split attribute names into string array
	var result []string
	b := bytes.Split(buf, []byte{0})
	for _, v := range b {
		// Split returns an empty slice after last separator, so check length.
		if len(v) > 0 {
			result = append(result, string(v))
		}
	}
	return result, nil
}
<commit_msg>Added check for case where file has no extended attributes at all, to avoid a runtime error of index out of range.<commit_after>// Copyright 2012 Bitcartel Software. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.

// +build darwin

// Package xattr wraps OS X functions to manipulate the extended attributes of a file, directory and symbolic link.
//
// The functions are a wrapper around xattr system calls.
// One difference is that position and size arguments are not implemented, since the caller can create slices from returned data.
package xattr

//#include <stdlib.h>
//#include <sys/xattr.h>
import "C"
import "unsafe"
import "bytes"

// Option flags to Xattr wrapping those in <sys/xattr.h>
const (
	XATTR_NOFOLLOW          int = C.XATTR_NOFOLLOW        /* Don't follow symbolic links */
	XATTR_CREATE            int = C.XATTR_CREATE          /* set the value, fail if attr already exists */
	XATTR_REPLACE           int = C.XATTR_REPLACE         /* set the value, fail if attr does not exist */
	XATTR_SHOWCOMPRESSION   int = C.XATTR_SHOWCOMPRESSION /* option for f/getxattr() and f/listxattr() to expose the HFS Compression extended attributes */
	XATTR_NOSECURITY        int = C.XATTR_NOSECURITY      /* Set this to bypass authorization checking (eg. if doing auth-related work) */
	XATTR_NODEFAULT         int = C.XATTR_NODEFAULT       /* Set this to bypass the default extended attribute file (dot-underscore file) */
	XATTR_MAXNAMELEN            = 127
	XATTR_FINDERINFO_NAME       = "com.apple.FinderInfo"
	XATTR_RESOURCEFORK_NAME     = "com.apple.ResourceFork"
	XATTR_MAXSIZE               = (64 * 1024 * 1024)
)

// Removexattr will remove the named attribute from file at path
func Removexattr(path string, name string, options int) error {
	p := C.CString(path)
	n := C.CString(name)
	defer C.free(unsafe.Pointer(p))
	defer C.free(unsafe.Pointer(n))
	_, err := C.removexattr(p, n, C.int(options))
	if err != nil {
		return err
	}
	return nil
}

// Setxattr will set the value for a named attribute on file at path
func Setxattr(path string, name string, data []byte, options int) error {
	p := C.CString(path)
	n := C.CString(name)
	defer C.free(unsafe.Pointer(p))
	defer C.free(unsafe.Pointer(n))

	_, err := C.setxattr(p, n, unsafe.Pointer(&data[0]), C.size_t(len(data)), 0, C.int(options))
	if err != nil {
		return err
	}
	return nil
}

// Getxattr will return the value for the named attribute from a file at path
func Getxattr(path string, name string, options int) ([]byte, error) {
	p := C.CString(path)
	n := C.CString(name)
	defer C.free(unsafe.Pointer(p))
	defer C.free(unsafe.Pointer(n))

	// get size of data for attribute, type _Ctype_ssize_t
	attrsize, err := C.getxattr(p, n, nil, 0, 0, 0)
	if err != nil {
		return nil, err
	}
	size := int(attrsize)

	// get data for attribute
	buf := make([]byte, size)
	x, err := C.getxattr(p, n, unsafe.Pointer(&buf[0]), C.size_t(size), 0, C.int(options))
	if err != nil {
		return nil, err
	}

	return buf[:x], nil
}

// Listxattr will return a list of attribute names found for a file at path
func Listxattr(path string, options int) ([]string, error) {
	p := C.CString(path)
	defer C.free(unsafe.Pointer(p))

	// get size of buffer needed for attribute names, type is _Ctype_ssize_t
	listsize, err := C.listxattr(p, nil, 0, 0)
	if err != nil || int(listsize)==0 {
		return nil, err
	}

	// get attribute names
	size := int(listsize)
	buf := make([]byte, size)
	_, err = C.listxattr(p, (*C.char)(unsafe.Pointer(&buf[0])), C.size_t(size), C.int(options))
	if err != nil {
		return nil, err
	}

	// split attribute names into string array
	var result []string
	b := bytes.Split(buf, []byte{0})
	for _, v := range b {
		// Split returns an empty slice after last separator, so check length.
		if len(v) > 0 {
			result = append(result, string(v))
		}
	}
	return result, nil
}
<|endoftext|>


<commit_before>package repo

import (
	"fmt"
	"github.com/omakoto/go-common/src/common"
	"github.com/omakoto/go-common/src/fileutils"
	"os"
	"path/filepath"
)

const EnvBuildTop = "ANDROID_BUILD_TOP"

func FindRepoTop(path string) (string, error) {
	atop := os.Getenv(EnvBuildTop)

	path, err := filepath.Abs(path)
	if err != nil {
		return "", err
	}
	for {
		s, err := os.Stat(filepath.Join(path, ".repo"))
		if err == nil && s.IsDir() {
			if atop != "" && !fileutils.SamePath(atop, path) {
				return "", fmt.Errorf("not in $%s", EnvBuildTop)
			}
			return path, nil
		}
		if path == "/" {
			return "", fmt.Errorf("repo top directory not found")
		}
		path = filepath.Dir(path)
	}
}

func MustFindRepoTop(path string) string {
	ret, err := FindRepoTop(path)
	common.Check(err, "Not in repo")
	return ret
}
<commit_msg>Make in-repo strict<commit_after>package repo

import (
	"fmt"
	"github.com/omakoto/go-common/src/common"
	"github.com/omakoto/go-common/src/fileutils"
	"os"
	"path/filepath"
)

const EnvBuildTop = "ANDROID_BUILD_TOP"

func FindRepoTop(path string) (string, error) {
	atop := os.Getenv(EnvBuildTop)

	if atop == "" {
		return "", fmt.Errorf("%s not set", EnvBuildTop)
	}

	path, err := filepath.Abs(path)
	if err != nil {
		return "", err
	}
	for {
		s, err := os.Stat(filepath.Join(path, ".repo"))
		if err == nil && s.IsDir() {
			if !fileutils.SamePath(atop, path) {
				return "", fmt.Errorf("not in $%s", EnvBuildTop)
			}
			return path, nil
		}
		if path == "/" {
			return "", fmt.Errorf("repo top directory not found")
		}
		path = filepath.Dir(path)
	}
}

func MustFindRepoTop(path string) string {
	ret, err := FindRepoTop(path)
	common.Check(err, "Not in repo")
	return ret
}
<|endoftext|>


<commit_before>package compiler

import (
	"github.com/GeertJohan/go.rice"
	"github.com/dop251/goja"
	"github.com/mitchellh/mapstructure"
)

var (
	lib      = rice.MustFindBox("lib")
	babelSrc = lib.MustString("lib/babel-standalone-bower/babel.js")

	DefaultOpts = map[string]interface{}{
		"presets":       []string{"latest"},
		"ast":           false,
		"sourceMaps":    true,
		"babelrc":       false,
		"compact":       false,
		"highlightCode": false,
	}
)

// A Compiler uses Babel to compile ES6 code into something ES5-compatible.
type Compiler struct {
	vm *goja.Runtime

	// JS pointers.
	this      goja.Value
	transform goja.Callable
}

// Constructs a new compiler.
func New() (*Compiler, error) {
	c := &Compiler{vm: goja.New()}
	if _, err := c.vm.RunString(babelSrc); err != nil {
		return nil, err
	}

	c.this = c.vm.Get("Babel")
	thisObj := c.this.ToObject(c.vm)
	if err := c.vm.ExportTo(thisObj.Get("transform"), &c.transform); err != nil {
		return nil, err
	}

	return c, nil
}

func (c *Compiler) Transform(src, filename string) (code string, srcmap SourceMap, err error) {
	opts := make(map[string]interface{})
	for k, v := range DefaultOpts {
		opts[k] = v
	}
	opts["filename"] = filename

	v, err := c.transform(c.this, c.vm.ToValue(src), c.vm.ToValue(opts))
	if err != nil {
		return code, srcmap, err
	}
	vO := v.ToObject(c.vm)

	if err := c.vm.ExportTo(vO.Get("code"), &code); err != nil {
		return code, srcmap, err
	}

	var rawmap map[string]interface{}
	if err := c.vm.ExportTo(vO.Get("map"), &rawmap); err != nil {
		return code, srcmap, err
	}
	if err := mapstructure.Decode(rawmap, &srcmap); err != nil {
		return code, srcmap, err
	}

	return code, srcmap, nil
}
<commit_msg>(I'm an idiot)<commit_after>package compiler

import (
	"github.com/GeertJohan/go.rice"
	"github.com/dop251/goja"
	"github.com/mitchellh/mapstructure"
)

var (
	lib      = rice.MustFindBox("lib")
	babelSrc = lib.MustString("babel-standalone-bower/babel.js")

	DefaultOpts = map[string]interface{}{
		"presets":       []string{"latest"},
		"ast":           false,
		"sourceMaps":    true,
		"babelrc":       false,
		"compact":       false,
		"highlightCode": false,
	}
)

// A Compiler uses Babel to compile ES6 code into something ES5-compatible.
type Compiler struct {
	vm *goja.Runtime

	// JS pointers.
	this      goja.Value
	transform goja.Callable
}

// Constructs a new compiler.
func New() (*Compiler, error) {
	c := &Compiler{vm: goja.New()}
	if _, err := c.vm.RunString(babelSrc); err != nil {
		return nil, err
	}

	c.this = c.vm.Get("Babel")
	thisObj := c.this.ToObject(c.vm)
	if err := c.vm.ExportTo(thisObj.Get("transform"), &c.transform); err != nil {
		return nil, err
	}

	return c, nil
}

func (c *Compiler) Transform(src, filename string) (code string, srcmap SourceMap, err error) {
	opts := make(map[string]interface{})
	for k, v := range DefaultOpts {
		opts[k] = v
	}
	opts["filename"] = filename

	v, err := c.transform(c.this, c.vm.ToValue(src), c.vm.ToValue(opts))
	if err != nil {
		return code, srcmap, err
	}
	vO := v.ToObject(c.vm)

	if err := c.vm.ExportTo(vO.Get("code"), &code); err != nil {
		return code, srcmap, err
	}

	var rawmap map[string]interface{}
	if err := c.vm.ExportTo(vO.Get("map"), &rawmap); err != nil {
		return code, srcmap, err
	}
	if err := mapstructure.Decode(rawmap, &srcmap); err != nil {
		return code, srcmap, err
	}

	return code, srcmap, nil
}
<|endoftext|>


<commit_before>package web

import (
    "crypto/md5"
    "fmt"
    "io"
    "mime"
    "os"
    "path"
    "strconv"
    "strings"
    "time"
    "utf8"
)

func isText(b []byte) bool {
    for len(b) > 0 && utf8.FullRune(b) {
        rune, size := utf8.DecodeRune(b)
        if size == 1 && rune == utf8.RuneError {
            // decoding error
            return false
        }
        if 0x80 <= rune && rune <= 0x9F {
            return false
        }
        if rune < ' ' {
            switch rune {
            case '\n', '\r', '\t':
                // okay
            default:
                // binary garbage
                return false
            }
        }
        b = b[size:]
    }
    return true
}

func getmd5(data string) string {
    hash := md5.New()
    hash.Write([]byte(data))
    return fmt.Sprintf("%x", hash.Sum())
}

func serveFile(ctx *Context, name string) {
    f, err := os.Open(name, os.O_RDONLY, 0)

    if err != nil {
        ctx.Abort(404, "Invalid file")
        return
    }

    defer f.Close()

    info, _ := os.Stat(name)
    //set content-length
    ctx.SetHeader("Content-Length", strconv.Itoa64(info.Size), true)

    //set the last-modified header
    lm := time.SecondsToLocalTime(info.Mtime_ns / 1e9)
    ctx.SetHeader("Last-Modified", webTime(lm), true)

    //generate a simple etag with heuristic MD5(filename, size, lastmod)
    etagparts := []string{name, strconv.Itoa64(info.Size), strconv.Itoa64(info.Mtime_ns)}
    etag := fmt.Sprintf(`"%s"`, getmd5(strings.Join(etagparts, "|")))
    ctx.SetHeader("ETag", etag, true)

    ext := path.Ext(name)
    if ctype := mime.TypeByExtension(ext); ctype != "" {
        ctx.SetHeader("Content-Type", ctype, true)
    } else {
        // read first chunk to decide between utf-8 text and binary
        buf := make([]byte, 1024)
        n, _ := io.ReadFull(f, buf)
        b := buf[0:n]
        if isText(b) {
            ctx.SetHeader("Content-Type", "text-plain; charset=utf-8", true)
        } else {
            ctx.SetHeader("Content-Type", "application/octet-stream", true) // generic binary
        }
        if ctx.Request.Method != "HEAD" {
            ctx.Write(b)
        }
    }
    if ctx.Request.Method != "HEAD" {
        io.Copy(ctx, f)
    }
}
<commit_msg>Fixes to serveFile:     · include DEL in the test for unprintable characters     · call Stat() on the already-opened file (to save a       potential walk in the kernel)     · reduce the number of conversions from int to string<commit_after>package web

import (
    "crypto/md5"
    "fmt"
    "io"
    "mime"
    "os"
    "path"
    "strconv"
    "strings"
    "time"
    "utf8"
)

func isText(b []byte) bool {
    for len(b) > 0 && utf8.FullRune(b) {
        rune, size := utf8.DecodeRune(b)
        if size == 1 && rune == utf8.RuneError {
            // decoding error
            return false
        }
        if 0x7F <= rune && rune <= 0x9F {
            return false
        }
        if rune < ' ' {
            switch rune {
            case '\n', '\r', '\t':
                // okay
            default:
                // binary garbage
                return false
            }
        }
        b = b[size:]
    }
    return true
}

func getmd5(data string) string {
    hash := md5.New()
    hash.Write([]byte(data))
    return fmt.Sprintf("%x", hash.Sum())
}

func serveFile(ctx *Context, name string) {
    f, err := os.Open(name, os.O_RDONLY, 0)

    if err != nil {
        ctx.Abort(404, "Invalid file")
        return
    }

    defer f.Close()

    info, _ := f.Stat()
    size := strconv.Itoa64(info.Size)
    mtime := strconv.Itoa64(info.Mtime_ns)

    //set content-length
    ctx.SetHeader("Content-Length", size, true)

    //set the last-modified header
    lm := time.SecondsToLocalTime(info.Mtime_ns / 1e9)
    ctx.SetHeader("Last-Modified", webTime(lm), true)

    //generate a simple etag with heuristic MD5(filename, size, lastmod)
    etagparts := []string{name, size, mtime}
    etag := fmt.Sprintf(`"%s"`, getmd5(strings.Join(etagparts, "|")))
    ctx.SetHeader("ETag", etag, true)

    ext := path.Ext(name)
    if ctype := mime.TypeByExtension(ext); ctype != "" {
        ctx.SetHeader("Content-Type", ctype, true)
    } else {
        // read first chunk to decide between utf-8 text and binary
        buf := make([]byte, 1024)
        n, _ := io.ReadFull(f, buf)
        b := buf[0:n]
        if isText(b) {
            ctx.SetHeader("Content-Type", "text-plain; charset=utf-8", true)
        } else {
            ctx.SetHeader("Content-Type", "application/octet-stream", true) // generic binary
        }
        if ctx.Request.Method != "HEAD" {
            ctx.Write(b)
        }
    }
    if ctx.Request.Method != "HEAD" {
        io.Copy(ctx, f)
    }
}
<|endoftext|>


<commit_before>// Package xopen makes it easy to get buffered readers and writers.
// Ropen opens a (possibly gzipped) file/process/http site for buffered reading.
// Wopen opens a (possibly gzipped) file for buffered writing.
// Both will use gzip when appropriate and will user buffered IO.
package xopen

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"os/user"
	"strings"

	//gzip "github.com/klauspost/pgzip"
	//"github.com/klauspost/compress/gzip"

	"compress/gzip"
)

// IsGzip returns true buffered Reader has the gzip magic.
func IsGzip(b *bufio.Reader) (bool, error) {
	return CheckBytes(b, []byte{0x1f, 0x8b})
}

// IsStdin checks if we are getting data from stdin.
func IsStdin() bool {
	// http://stackoverflow.com/a/26567513
	stat, err := os.Stdin.Stat()
	if err != nil {
		return false
	}
	return (stat.Mode() & os.ModeCharDevice) == 0
}

// ExpandUser expands ~/path and ~otheruser/path appropriately.
func ExpandUser(path string) (string, error) {
	if path[0] != '~' {
		return path, nil
	}
	var u *user.User
	var err error
	if len(path) == 1 || path[1] == '/' {
		u, err = user.Current()
	} else {
		name := strings.Split(path[1:], "/")[0]
		u, err = user.Lookup(name)
	}
	if err != nil {
		return "", err
	}
	home := u.HomeDir
	path = home + "/" + path[1:]
	return path, nil
}

// Exists checks if a local file exits
func Exists(path string) bool {
	path, perr := ExpandUser(path)
	if perr != nil {
		return false
	}
	_, err := os.Stat(path)
	return err == nil
}

// CheckBytes peeks at a buffered stream and checks if the first read bytes match.
func CheckBytes(b *bufio.Reader, buf []byte) (bool, error) {

	m, err := b.Peek(len(buf))
	if err != nil {
		return false, err
	}
	for i := range buf {
		if m[i] != buf[i] {
			return false, nil
		}
	}
	return true, nil
}

// Reader is returned by Ropen
type Reader struct {
	*bufio.Reader
	rdr io.Reader
	gz  io.ReadCloser
}

// Close the associated files.
func (r *Reader) Close() error {
	if r.gz != nil {
		r.gz.Close()
	}
	if c, ok := r.rdr.(io.ReadCloser); ok {
		c.Close()
	}
	return nil
}

// Writer is returned by Wopen
type Writer struct {
	*bufio.Writer
	wtr *os.File
	gz  *gzip.Writer
}

// Name returns the path to the underlying file.
func (w *Writer) Name() string {
	return w.wtr.Name()
}

// Close the associated files.
func (w *Writer) Close() error {
	w.Flush()
	if w.gz != nil {
		w.gz.Close()
	}
	w.wtr.Close()
	return nil
}

// Flush the writer.
func (w *Writer) Flush() {
	w.Writer.Flush()
	if w.gz != nil {
		w.gz.Flush()
	}
}

var pageSize = os.Getpagesize() * 2

// Buf returns a buffered reader from an io.Reader.
// If f == "-", then it will attempt to read from os.Stdin.
// If the file is gzipped, it will be read as such.
func Buf(r io.Reader) *Reader {
	b := bufio.NewReaderSize(r, pageSize)
	var rdr io.ReadCloser
	if is, err := IsGzip(b); err != nil && err != io.EOF {
		log.Fatal(err)
	} else if is {
		rdr, err = gzip.NewReader(b)
		if err != nil {
			log.Fatal(err)
		}
		b = bufio.NewReaderSize(rdr, pageSize)
	}
	return &Reader{b, r, rdr}
}

// XReader returns a reader from a url string or a file.
func XReader(f string) (io.Reader, error) {
	if strings.HasPrefix(f, "http://") || strings.HasPrefix(f, "https://") {
		var rsp *http.Response
		rsp, err := http.Get(f)
		if err != nil {
			return nil, err
		}
		if rsp.StatusCode != 200 {
			return nil, fmt.Errorf("http error downloading %s. status: %s", f, rsp.Status)
		}
		rdr := rsp.Body
		return rdr, nil
	}
	f, err := ExpandUser(f)
	if err != nil {
		return nil, err
	}
	return os.Open(f)
}

// Ropen opens a buffered reader.
func Ropen(f string) (*Reader, error) {
	var err error
	var rdr io.Reader
	if f == "" {
		return nil, errors.New("sent empty file-name to xopen.Ropen")
	}
	if f == "-" {
		if !IsStdin() {
			return nil, errors.New("warning: stdin not detected")
		}
		b := Buf(os.Stdin)
		return b, nil
	} else if f[0] == '|' {
		// TODO: use csv to handle quoted file names.
		cmdStrs := strings.Split(f[1:], " ")
		var cmd *exec.Cmd
		if len(cmdStrs) == 2 {
			cmd = exec.Command(cmdStrs[0], cmdStrs[1:]...)
		} else {
			cmd = exec.Command(cmdStrs[0])
		}
		rdr, err = cmd.StdoutPipe()
		if err != nil {
			return nil, err
		}
		err = cmd.Start()
		if err != nil {
			return nil, err
		}
	} else {
		rdr, err = XReader(f)
	}
	if err != nil {
		return nil, err
	}
	b := Buf(rdr)
	return b, nil
}

// Wopen opens a buffered reader.
// If f == "-", then stdout will be used.
// If f endswith ".gz", then the output will be gzipped.
// If f startswith "tmp:" then a tmpfile will be created with the prefix being the value after tmp:. e.g. tmp:fx.gz will create a gzip writer for /tmp/fx${random}.gz
func Wopen(f string) (*Writer, error) {
	var wtr *os.File
	var err error
	if f == "-" {
		wtr = os.Stdout
	} else if strings.HasPrefix(f, "tmp:") {
		prefix := ""
		if len(f) > 4 {
			prefix = strings.TrimSuffix(strings.Split(f, ":")[1], ".gz")
		}
		wtr, err = ioutil.TempFile("", prefix)
	} else {
		wtr, err = os.Create(f)
		if err != nil {
			return nil, err
		}
	}
	if !strings.HasSuffix(f, ".gz") {
		return &Writer{bufio.NewWriterSize(wtr, pageSize), wtr, nil}, nil
	}
	gz := gzip.NewWriter(wtr)
	w, err := &Writer{bufio.NewWriterSize(gz, pageSize), wtr, gz}, nil
	return w, err
}
<commit_msg>fix empty path<commit_after>// Package xopen makes it easy to get buffered readers and writers.
// Ropen opens a (possibly gzipped) file/process/http site for buffered reading.
// Wopen opens a (possibly gzipped) file for buffered writing.
// Both will use gzip when appropriate and will user buffered IO.
package xopen

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"os/user"
	"strings"

	//gzip "github.com/klauspost/pgzip"
	//"github.com/klauspost/compress/gzip"

	"compress/gzip"
)

// IsGzip returns true buffered Reader has the gzip magic.
func IsGzip(b *bufio.Reader) (bool, error) {
	return CheckBytes(b, []byte{0x1f, 0x8b})
}

// IsStdin checks if we are getting data from stdin.
func IsStdin() bool {
	// http://stackoverflow.com/a/26567513
	stat, err := os.Stdin.Stat()
	if err != nil {
		return false
	}
	return (stat.Mode() & os.ModeCharDevice) == 0
}

// ExpandUser expands ~/path and ~otheruser/path appropriately.
func ExpandUser(path string) (string, error) {
	if len(path) == 0 || path[0] != '~' {
		return path, nil
	}
	var u *user.User
	var err error
	if len(path) == 1 || path[1] == '/' {
		u, err = user.Current()
	} else {
		name := strings.Split(path[1:], "/")[0]
		u, err = user.Lookup(name)
	}
	if err != nil {
		return "", err
	}
	home := u.HomeDir
	path = home + "/" + path[1:]
	return path, nil
}

// Exists checks if a local file exits
func Exists(path string) bool {
	path, perr := ExpandUser(path)
	if perr != nil {
		return false
	}
	_, err := os.Stat(path)
	return err == nil
}

// CheckBytes peeks at a buffered stream and checks if the first read bytes match.
func CheckBytes(b *bufio.Reader, buf []byte) (bool, error) {

	m, err := b.Peek(len(buf))
	if err != nil {
		return false, err
	}
	for i := range buf {
		if m[i] != buf[i] {
			return false, nil
		}
	}
	return true, nil
}

// Reader is returned by Ropen
type Reader struct {
	*bufio.Reader
	rdr io.Reader
	gz  io.ReadCloser
}

// Close the associated files.
func (r *Reader) Close() error {
	if r.gz != nil {
		r.gz.Close()
	}
	if c, ok := r.rdr.(io.ReadCloser); ok {
		c.Close()
	}
	return nil
}

// Writer is returned by Wopen
type Writer struct {
	*bufio.Writer
	wtr *os.File
	gz  *gzip.Writer
}

// Name returns the path to the underlying file.
func (w *Writer) Name() string {
	return w.wtr.Name()
}

// Close the associated files.
func (w *Writer) Close() error {
	w.Flush()
	if w.gz != nil {
		w.gz.Close()
	}
	w.wtr.Close()
	return nil
}

// Flush the writer.
func (w *Writer) Flush() {
	w.Writer.Flush()
	if w.gz != nil {
		w.gz.Flush()
	}
}

var pageSize = os.Getpagesize() * 2

// Buf returns a buffered reader from an io.Reader.
// If f == "-", then it will attempt to read from os.Stdin.
// If the file is gzipped, it will be read as such.
func Buf(r io.Reader) *Reader {
	b := bufio.NewReaderSize(r, pageSize)
	var rdr io.ReadCloser
	if is, err := IsGzip(b); err != nil && err != io.EOF {
		log.Fatal(err)
	} else if is {
		rdr, err = gzip.NewReader(b)
		if err != nil {
			log.Fatal(err)
		}
		b = bufio.NewReaderSize(rdr, pageSize)
	}
	return &Reader{b, r, rdr}
}

// XReader returns a reader from a url string or a file.
func XReader(f string) (io.Reader, error) {
	if strings.HasPrefix(f, "http://") || strings.HasPrefix(f, "https://") {
		var rsp *http.Response
		rsp, err := http.Get(f)
		if err != nil {
			return nil, err
		}
		if rsp.StatusCode != 200 {
			return nil, fmt.Errorf("http error downloading %s. status: %s", f, rsp.Status)
		}
		rdr := rsp.Body
		return rdr, nil
	}
	f, err := ExpandUser(f)
	if err != nil {
		return nil, err
	}
	return os.Open(f)
}

// Ropen opens a buffered reader.
func Ropen(f string) (*Reader, error) {
	var err error
	var rdr io.Reader
	if f == "" {
		return nil, errors.New("sent empty file-name to xopen.Ropen")
	}
	if f == "-" {
		if !IsStdin() {
			return nil, errors.New("warning: stdin not detected")
		}
		b := Buf(os.Stdin)
		return b, nil
	} else if f[0] == '|' {
		// TODO: use csv to handle quoted file names.
		cmdStrs := strings.Split(f[1:], " ")
		var cmd *exec.Cmd
		if len(cmdStrs) == 2 {
			cmd = exec.Command(cmdStrs[0], cmdStrs[1:]...)
		} else {
			cmd = exec.Command(cmdStrs[0])
		}
		rdr, err = cmd.StdoutPipe()
		if err != nil {
			return nil, err
		}
		err = cmd.Start()
		if err != nil {
			return nil, err
		}
	} else {
		rdr, err = XReader(f)
	}
	if err != nil {
		return nil, err
	}
	b := Buf(rdr)
	return b, nil
}

// Wopen opens a buffered reader.
// If f == "-", then stdout will be used.
// If f endswith ".gz", then the output will be gzipped.
// If f startswith "tmp:" then a tmpfile will be created with the prefix being the value after tmp:. e.g. tmp:fx.gz will create a gzip writer for /tmp/fx${random}.gz
func Wopen(f string) (*Writer, error) {
	var wtr *os.File
	var err error
	if f == "-" {
		wtr = os.Stdout
	} else if strings.HasPrefix(f, "tmp:") {
		prefix := ""
		if len(f) > 4 {
			prefix = strings.TrimSuffix(strings.Split(f, ":")[1], ".gz")
		}
		wtr, err = ioutil.TempFile("", prefix)
	} else {
		wtr, err = os.Create(f)
		if err != nil {
			return nil, err
		}
	}
	if !strings.HasSuffix(f, ".gz") {
		return &Writer{bufio.NewWriterSize(wtr, pageSize), wtr, nil}, nil
	}
	gz := gzip.NewWriter(wtr)
	w, err := &Writer{bufio.NewWriterSize(gz, pageSize), wtr, gz}, nil
	return w, err
}
<|endoftext|>


<commit_before>/*
   Copyright 2014 Outbrain Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

// zk provides with higher level commands over the lower level zookeeper connector
package zk

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/outbrain/golib/log"
	"github.com/samuel/go-zookeeper/zk"
	"math"
	gopath "path"
	"sort"
	"strconv"
	"strings"
	"time"
)

var servers []string
var authScheme string
var authExpression []byte

// We assume complete access to all
var flags int32 = int32(0)
var acl []zk.ACL = zk.WorldACL(zk.PermAll)

// SetServers sets the list of servers for the zookeeper client to connect to.
// Each element in the array should be in either of following forms:
// - "servername"
// - "servername:port"
func SetServers(serversArray []string) {
	servers = serversArray
}

func SetAuth(scheme string, auth []byte) {
	log.Debug("Setting Auth ")
	authScheme = scheme
	authExpression = auth
}

// Returns acls
func BuildACL(authScheme string, user string, pwd string, acls string) (perms []zk.ACL, err error) {
	aclsList := strings.Split(acls, ",")
	for _, elem := range aclsList {
		acl, err := strconv.ParseInt(elem, 10, 32)
		if err != nil {
			break
		}
		perm := zk.DigestACL(int32(acl), user, pwd)
		perms = append(perms, perm[0])
	}
	return perms, err
}

// connect
func connect() (*zk.Conn, error) {
	conn, _, err := zk.Connect(servers, time.Second)
	if err == nil && authScheme != "" {
		log.Debugf("Add Auth %s %s", authScheme, authExpression)
		err = conn.AddAuth(authScheme, authExpression)
	}

	return conn, err
}

// Exists returns true when the given path exists
func Exists(path string) (bool, error) {
	connection, err := connect()
	if err != nil {
		return false, err
	}
	defer connection.Close()

	exists, _, err := connection.Exists(path)
	return exists, err
}

// Get returns value associated with given path, or error if path does not exist
func Get(path string) ([]byte, error) {
	connection, err := connect()
	if err != nil {
		return []byte{}, err
	}
	defer connection.Close()

	data, _, err := connection.Get(path)
	return data, err
}

func GetACL(path string) (data []string, err error) {
	connection, err := connect()
	if err != nil {
		return nil, err
	}
	defer connection.Close()

	perms, _, err := connection.GetACL(path)
	return aclsToString(perms), err
}

func aclsToString(acls []zk.ACL) (result []string) {
	for _, acl := range acls {
		var buffer bytes.Buffer

		buffer.WriteString(fmt.Sprintf("%v:%v:", acl.Scheme, acl.ID))

		if acl.Perms&zk.PermCreate != 0 {
			buffer.WriteString("c")
		}
		if acl.Perms&zk.PermDelete != 0 {
			buffer.WriteString("d")
		}
		if acl.Perms&zk.PermRead != 0 {
			buffer.WriteString("r")
		}
		if acl.Perms&zk.PermWrite != 0 {
			buffer.WriteString("w")
		}
		if acl.Perms&zk.PermAdmin != 0 {
			buffer.WriteString("a")
		}
		result = append(result, buffer.String())
	}
	return result
}

// Children returns sub-paths of given path, optionally empty array, or error if path does not exist
func Children(path string) ([]string, error) {
	connection, err := connect()
	if err != nil {
		return []string{}, err
	}
	defer connection.Close()

	children, _, err := connection.Children(path)
	return children, err
}

// childrenRecursiveInternal: internal implementation of recursive-children query.
func childrenRecursiveInternal(connection *zk.Conn, path string, incrementalPath string) ([]string, error) {
	children, _, err := connection.Children(path)
	if err != nil {
		return children, err
	}
	sort.Sort(sort.StringSlice(children))
	recursiveChildren := []string{}
	for _, child := range children {
		incrementalChild := gopath.Join(incrementalPath, child)
		recursiveChildren = append(recursiveChildren, incrementalChild)
		log.Debugf("incremental child: %+v", incrementalChild)
		incrementalChildren, err := childrenRecursiveInternal(connection, gopath.Join(path, child), incrementalChild)
		if err != nil {
			return children, err
		}
		recursiveChildren = append(recursiveChildren, incrementalChildren...)
	}
	return recursiveChildren, err
}

// ChildrenRecursive returns list of all descendants of given path (optionally empty), or error if the path
// does not exist.
// Every element in result list is a relative subpath for the given path.
func ChildrenRecursive(path string) ([]string, error) {
	connection, err := connect()
	if err != nil {
		return []string{}, err
	}
	defer connection.Close()

	result, err := childrenRecursiveInternal(connection, path, "")
	return result, err
}

// createInternal: create a new path
func createInternal(connection *zk.Conn, path string, data []byte, acl []zk.ACL, force bool) (string, error) {
	if path == "/" {
		return "/", nil
	}

	log.Debugf("creating: %s", path)
	attempts := 0
	for {
		attempts += 1
		returnValue, err := connection.Create(path, data, flags, acl)
		log.Debugf("create status for %s: %s, %+v", path, returnValue, err)
		if err != nil && force && attempts < 2 {
			returnValue, err = createInternal(connection, gopath.Dir(path), []byte("zookeepercli auto-generated"), acl, force)
		} else {
			return returnValue, err
		}
	}
	return "", nil
}

// createInternalWithACL: create a new path with acl
func createInternalWithACL(connection *zk.Conn, path string, data []byte, force bool, perms []zk.ACL) (string, error) {
	if path == "/" {
		return "/", nil
	}
	log.Debugf("creating: %s with acl ", path)
	attempts := 0
	for {
		attempts += 1
		returnValue, err := connection.Create(path, data, flags, perms)
		log.Debugf("create status for %s: %s, %+v", path, returnValue, err)
		if err != nil && force && attempts < 2 {
			returnValue, err = createInternalWithACL(connection, gopath.Dir(path), []byte("zookeepercli auto-generated"), force, perms)
		} else {
			return returnValue, err
		}
	}
	return "", nil
}

// Create will create a new path, or exit with error should the path exist.
// The "force" param controls the behavior when path's parent directory does not exist.
// When "force" is false, the function returns with error/ When "force" is true, it recursively
// attempts to create required parent directories.
func Create(path string, data []byte, aclstr string, force bool) (string, error) {
	connection, err := connect()
	if err != nil {
		return "", err
	}
	defer connection.Close()

	if len(aclstr) > 0 {
		acl, err = parseACLString(aclstr)
		if err != nil {
			return "", err
		}
	}

	return createInternal(connection, path, data, acl, force)
}

func CreateWithACL(path string, data []byte, force bool, perms []zk.ACL) (string, error) {
	connection, err := connect()
	if err != nil {
		return "", err
	}
	defer connection.Close()

	return createInternalWithACL(connection, path, data, force, perms)
}

// Set updates a value for a given path, or returns with error if the path does not exist
func Set(path string, data []byte) (*zk.Stat, error) {
	connection, err := connect()
	if err != nil {
		return nil, err
	}
	defer connection.Close()

	return connection.Set(path, data, -1)
}

// updates the ACL on a given path
func SetACL(path string, aclstr string, force bool) (string, error) {
	connection, err := connect()
	if err != nil {
		return "", err
	}
	defer connection.Close()

	acl, err := parseACLString(aclstr)
	if err != nil {
		return "", err
	}

	if force {
		exists, _, err := connection.Exists(path)
		if err != nil {
			return "", err
		}

		if !exists {
			return createInternal(connection, path, []byte(""), acl, force)
		}
	}

	_, err = connection.SetACL(path, acl, -1)
	return path, err
}

func parseACLString(aclstr string) (acl []zk.ACL, err error) {
	aclsList := strings.Split(aclstr, ",")
	for _, entry := range aclsList {
		parts := strings.Split(entry, ":")
		var scheme, id string
		var perms int32
		if len(parts) > 3 && parts[0] == "digest" {
			scheme = parts[0]
			id = fmt.Sprintf("%s:%s", parts[1], parts[2])
			perms, err = parsePermsString(parts[3])
		} else {
			scheme, id = parts[0], parts[1]
			perms, err = parsePermsString(parts[2])
		}

		if err == nil {
			perm := zk.ACL{Scheme: scheme, ID: id, Perms: perms}
			acl = append(acl, perm)
		}
	}
	return acl, err
}

func parsePermsString(permstr string) (perms int32, err error) {
	if x, e := strconv.ParseFloat(permstr, 64); e == nil {
		perms = int32(math.Min(x, 31))
	} else {
		for _, rune := range strings.Split(permstr, "") {
			switch rune {
			case "r":
				perms |= zk.PermRead
				break
			case "w":
				perms |= zk.PermWrite
				break
			case "c":
				perms |= zk.PermCreate
				break
			case "d":
				perms |= zk.PermDelete
				break
			case "a":
				perms |= zk.PermAdmin
				break
			default:
				err = errors.New("invalid ACL string specified")
			}

			if err != nil {
				break
			}
		}
	}
	return perms, err
}

// Delete removes a path entry. It exits with error if the path does not exist, or has subdirectories.
func Delete(path string) error {
	connection, err := connect()
	if err != nil {
		return err
	}
	defer connection.Close()

	return connection.Delete(path, -1)
}

// Delete recursive if has subdirectories.
func DeleteRecursive(path string) error {
	result, err := ChildrenRecursive(path)
	if err != nil {
		log.Fatale(err)
	}

	for i := len(result) - 1; i >= 0; i-- {
		znode := path + "/" + result[i]
		if err = Delete(znode); err != nil {
			log.Fatale(err)
		}
	}

	return Delete(path)
}
<commit_msg>show go-zookeeper logs only if -verbose provided<commit_after>/*
   Copyright 2014 Outbrain Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

// zk provides with higher level commands over the lower level zookeeper connector
package zk

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/outbrain/golib/log"
	"github.com/samuel/go-zookeeper/zk"
	"math"
	gopath "path"
	"sort"
	"strconv"
	"strings"
	"time"
)

var servers []string
var authScheme string
var authExpression []byte

// We assume complete access to all
var flags int32 = int32(0)
var acl []zk.ACL = zk.WorldACL(zk.PermAll)

// SetServers sets the list of servers for the zookeeper client to connect to.
// Each element in the array should be in either of following forms:
// - "servername"
// - "servername:port"
func SetServers(serversArray []string) {
	servers = serversArray
}

func SetAuth(scheme string, auth []byte) {
	log.Debug("Setting Auth ")
	authScheme = scheme
	authExpression = auth
}

// Returns acls
func BuildACL(authScheme string, user string, pwd string, acls string) (perms []zk.ACL, err error) {
	aclsList := strings.Split(acls, ",")
	for _, elem := range aclsList {
		acl, err := strconv.ParseInt(elem, 10, 32)
		if err != nil {
			break
		}
		perm := zk.DigestACL(int32(acl), user, pwd)
		perms = append(perms, perm[0])
	}
	return perms, err
}

type infoLogger struct{}

func (_ infoLogger) Printf(format string, a ...interface{}) {
	log.Infof(format, a...)
}

// connect
func connect() (*zk.Conn, error) {
	zk.DefaultLogger = &infoLogger{}
	conn, _, err := zk.Connect(servers, time.Second)
	if err == nil && authScheme != "" {
		log.Debugf("Add Auth %s %s", authScheme, authExpression)
		err = conn.AddAuth(authScheme, authExpression)
	}

	return conn, err
}

// Exists returns true when the given path exists
func Exists(path string) (bool, error) {
	connection, err := connect()
	if err != nil {
		return false, err
	}
	defer connection.Close()

	exists, _, err := connection.Exists(path)
	return exists, err
}

// Get returns value associated with given path, or error if path does not exist
func Get(path string) ([]byte, error) {
	connection, err := connect()
	if err != nil {
		return []byte{}, err
	}
	defer connection.Close()

	data, _, err := connection.Get(path)
	return data, err
}

func GetACL(path string) (data []string, err error) {
	connection, err := connect()
	if err != nil {
		return nil, err
	}
	defer connection.Close()

	perms, _, err := connection.GetACL(path)
	return aclsToString(perms), err
}

func aclsToString(acls []zk.ACL) (result []string) {
	for _, acl := range acls {
		var buffer bytes.Buffer

		buffer.WriteString(fmt.Sprintf("%v:%v:", acl.Scheme, acl.ID))

		if acl.Perms&zk.PermCreate != 0 {
			buffer.WriteString("c")
		}
		if acl.Perms&zk.PermDelete != 0 {
			buffer.WriteString("d")
		}
		if acl.Perms&zk.PermRead != 0 {
			buffer.WriteString("r")
		}
		if acl.Perms&zk.PermWrite != 0 {
			buffer.WriteString("w")
		}
		if acl.Perms&zk.PermAdmin != 0 {
			buffer.WriteString("a")
		}
		result = append(result, buffer.String())
	}
	return result
}

// Children returns sub-paths of given path, optionally empty array, or error if path does not exist
func Children(path string) ([]string, error) {
	connection, err := connect()
	if err != nil {
		return []string{}, err
	}
	defer connection.Close()

	children, _, err := connection.Children(path)
	return children, err
}

// childrenRecursiveInternal: internal implementation of recursive-children query.
func childrenRecursiveInternal(connection *zk.Conn, path string, incrementalPath string) ([]string, error) {
	children, _, err := connection.Children(path)
	if err != nil {
		return children, err
	}
	sort.Sort(sort.StringSlice(children))
	recursiveChildren := []string{}
	for _, child := range children {
		incrementalChild := gopath.Join(incrementalPath, child)
		recursiveChildren = append(recursiveChildren, incrementalChild)
		log.Debugf("incremental child: %+v", incrementalChild)
		incrementalChildren, err := childrenRecursiveInternal(connection, gopath.Join(path, child), incrementalChild)
		if err != nil {
			return children, err
		}
		recursiveChildren = append(recursiveChildren, incrementalChildren...)
	}
	return recursiveChildren, err
}

// ChildrenRecursive returns list of all descendants of given path (optionally empty), or error if the path
// does not exist.
// Every element in result list is a relative subpath for the given path.
func ChildrenRecursive(path string) ([]string, error) {
	connection, err := connect()
	if err != nil {
		return []string{}, err
	}
	defer connection.Close()

	result, err := childrenRecursiveInternal(connection, path, "")
	return result, err
}

// createInternal: create a new path
func createInternal(connection *zk.Conn, path string, data []byte, acl []zk.ACL, force bool) (string, error) {
	if path == "/" {
		return "/", nil
	}

	log.Debugf("creating: %s", path)
	attempts := 0
	for {
		attempts += 1
		returnValue, err := connection.Create(path, data, flags, acl)
		log.Debugf("create status for %s: %s, %+v", path, returnValue, err)
		if err != nil && force && attempts < 2 {
			returnValue, err = createInternal(connection, gopath.Dir(path), []byte("zookeepercli auto-generated"), acl, force)
		} else {
			return returnValue, err
		}
	}
	return "", nil
}

// createInternalWithACL: create a new path with acl
func createInternalWithACL(connection *zk.Conn, path string, data []byte, force bool, perms []zk.ACL) (string, error) {
	if path == "/" {
		return "/", nil
	}
	log.Debugf("creating: %s with acl ", path)
	attempts := 0
	for {
		attempts += 1
		returnValue, err := connection.Create(path, data, flags, perms)
		log.Debugf("create status for %s: %s, %+v", path, returnValue, err)
		if err != nil && force && attempts < 2 {
			returnValue, err = createInternalWithACL(connection, gopath.Dir(path), []byte("zookeepercli auto-generated"), force, perms)
		} else {
			return returnValue, err
		}
	}
	return "", nil
}

// Create will create a new path, or exit with error should the path exist.
// The "force" param controls the behavior when path's parent directory does not exist.
// When "force" is false, the function returns with error/ When "force" is true, it recursively
// attempts to create required parent directories.
func Create(path string, data []byte, aclstr string, force bool) (string, error) {
	connection, err := connect()
	if err != nil {
		return "", err
	}
	defer connection.Close()

	if len(aclstr) > 0 {
		acl, err = parseACLString(aclstr)
		if err != nil {
			return "", err
		}
	}

	return createInternal(connection, path, data, acl, force)
}

func CreateWithACL(path string, data []byte, force bool, perms []zk.ACL) (string, error) {
	connection, err := connect()
	if err != nil {
		return "", err
	}
	defer connection.Close()

	return createInternalWithACL(connection, path, data, force, perms)
}

// Set updates a value for a given path, or returns with error if the path does not exist
func Set(path string, data []byte) (*zk.Stat, error) {
	connection, err := connect()
	if err != nil {
		return nil, err
	}
	defer connection.Close()

	return connection.Set(path, data, -1)
}

// updates the ACL on a given path
func SetACL(path string, aclstr string, force bool) (string, error) {
	connection, err := connect()
	if err != nil {
		return "", err
	}
	defer connection.Close()

	acl, err := parseACLString(aclstr)
	if err != nil {
		return "", err
	}

	if force {
		exists, _, err := connection.Exists(path)
		if err != nil {
			return "", err
		}

		if !exists {
			return createInternal(connection, path, []byte(""), acl, force)
		}
	}

	_, err = connection.SetACL(path, acl, -1)
	return path, err
}

func parseACLString(aclstr string) (acl []zk.ACL, err error) {
	aclsList := strings.Split(aclstr, ",")
	for _, entry := range aclsList {
		parts := strings.Split(entry, ":")
		var scheme, id string
		var perms int32
		if len(parts) > 3 && parts[0] == "digest" {
			scheme = parts[0]
			id = fmt.Sprintf("%s:%s", parts[1], parts[2])
			perms, err = parsePermsString(parts[3])
		} else {
			scheme, id = parts[0], parts[1]
			perms, err = parsePermsString(parts[2])
		}

		if err == nil {
			perm := zk.ACL{Scheme: scheme, ID: id, Perms: perms}
			acl = append(acl, perm)
		}
	}
	return acl, err
}

func parsePermsString(permstr string) (perms int32, err error) {
	if x, e := strconv.ParseFloat(permstr, 64); e == nil {
		perms = int32(math.Min(x, 31))
	} else {
		for _, rune := range strings.Split(permstr, "") {
			switch rune {
			case "r":
				perms |= zk.PermRead
				break
			case "w":
				perms |= zk.PermWrite
				break
			case "c":
				perms |= zk.PermCreate
				break
			case "d":
				perms |= zk.PermDelete
				break
			case "a":
				perms |= zk.PermAdmin
				break
			default:
				err = errors.New("invalid ACL string specified")
			}

			if err != nil {
				break
			}
		}
	}
	return perms, err
}

// Delete removes a path entry. It exits with error if the path does not exist, or has subdirectories.
func Delete(path string) error {
	connection, err := connect()
	if err != nil {
		return err
	}
	defer connection.Close()

	return connection.Delete(path, -1)
}

// Delete recursive if has subdirectories.
func DeleteRecursive(path string) error {
	result, err := ChildrenRecursive(path)
	if err != nil {
		log.Fatale(err)
	}

	for i := len(result) - 1; i >= 0; i-- {
		znode := path + "/" + result[i]
		if err = Delete(znode); err != nil {
			log.Fatale(err)
		}
	}

	return Delete(path)
}
<|endoftext|>


<commit_before>//Copyright (c) 2017, Will Dixon. All rights reserved.
//Use of this source code is governed by a BSD-style
//license that can be found in the LICENSE file.

//Package skywalker walks through a filesystem concurrently.
package skywalker

import (
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/gobwas/glob"
)

//Worker is anything that knows what to do with a path.
type Worker interface {
	Work(path string)
}

//ListType is used to specify how to handle the contents of a list
type ListType int

const (
	//LTBlacklist is used to specify that a list is to exclude the contents.
	LTBlacklist ListType = iota
	//LTWhitelist is used to specify that a list is to include the contents.
	LTWhitelist
)

//Skywalker can concurrently go through files in Root and call Worker on everything.
//It is recommended to use DirList and ExtList as much as possible as it is more perfomant than List.
type Skywalker struct {
	//Root is where the file walker starts. It is converted to an absolute path before start.
	Root string

	//List and ListType should only be used for fine filtering of paths.
	//It uses https://github.com/gobwas/glob for glob checking on each patch check.
	ListType ListType
	List     []string
	list     []glob.Glob

	//ExtList and ExtListType are used to narrow down the files by their extensions.
	//Make sure to include the preceding ".".
	ExtListType ListType
	ExtList     []string
	extMap      map[string]struct{}

	//DirList and DirListType are used to narrow down by directories.
	//Will skip the appropriate directories and their files/subfolders.
	DirListType ListType
	DirList     []string
	dirMap      map[string]bool

	//NumWorkers are how many workers are listening to the queue to do the work.
	NumWorkers int

	//QueueSize is how many paths to queue up at a time.
	//Useful for fine control over memory usage if needed.
	QueueSize int

	//Worker is the function that is called on each file/directory.
	Worker Worker

	//FilesOnly should be set to true if you only want to queue up files.
	FilesOnly bool
}

//New creates a new Skywalker that can walk through the specified root and calls the Worker on each file and/or directory.
//Defaults Skywalker to have 20 workers, a QueueSize of 100 and only queue files.
func New(root string, worker Worker) *Skywalker {
	return &Skywalker{
		Root:       root,
		NumWorkers: 20,
		QueueSize:  100,
		Worker:     worker,
		FilesOnly:  true,
	}
}

//Walk goes through the files and folders in Root and calls the worker on each.
//Checks the lists specified to check whether it should ignore files or directories.
//It also handles the creation of workers and queues needed for walking.
func (sw *Skywalker) Walk() error {
	if err := sw.init(); err != nil {
		return err
	}
	workerChan := make(chan string, sw.QueueSize)
	workerWG := new(sync.WaitGroup)
	workerWG.Add(sw.NumWorkers)
	for i := 0; i < sw.NumWorkers; i++ {
		go sw.worker(workerWG, workerChan)
	}
	err := filepath.Walk(sw.Root, sw.walker(workerChan))
	close(workerChan)
	workerWG.Wait()
	return err
}

func (sw *Skywalker) init() error {
	root, err := filepath.Abs(sw.Root)
	if err != nil {
		return err
	}
	sw.Root = root
	dirMap := make(map[string]bool, len(sw.DirList))
	for _, dir := range sw.DirList {
		if sw.DirListType == LTWhitelist {
			dirs := strings.Split(cleanDir(dir), string(filepath.Separator))
			for i := len(dirs); i > 0; i-- {
				dirMap[filepath.Join(root, filepath.Join(dirs[:i]...))] = i == len(dirs)
			}
		} else {
			dirMap[filepath.Join(root, cleanDir(dir))] = true
		}
	}
	sw.dirMap = dirMap
	extMap := make(map[string]struct{}, len(sw.ExtList))
	for _, ext := range sw.ExtList {
		extMap[ext] = struct{}{}
	}
	sw.extMap = extMap
	list := make([]glob.Glob, len(sw.List))
	for i, g := range sw.List {
		gl, er := glob.Compile(g)
		if er != nil {
			return er
		}
		list[i] = gl
	}
	sw.list = list
	return nil
}

func (sw *Skywalker) worker(workerWG *sync.WaitGroup, workerChan chan string) {
	defer workerWG.Done()
	for w := range workerChan {
		sw.Worker.Work(w)
	}
}

func (sw *Skywalker) walker(workerChan chan string) func(path string, info os.FileInfo, err error) error {
	return func(path string, info os.FileInfo, _ error) error {
		if info.IsDir() {
			if doSomething, err := sw.skipDir(path); doSomething {
				return err
			}
			if sw.FilesOnly {
				return nil
			}
		} else {
			if sw.skipFile(path) {
				return nil
			}
		}
		if sw.matchPath(path) == (sw.ListType == LTBlacklist) {
			return nil
		}
		workerChan <- path
		return nil
	}
}

func (sw *Skywalker) skipDir(path string) (bool, error) {
	switch sw.DirListType {
	case LTBlacklist:
		_, inList := sw.dirMap[path]
		if inList {
			return true, filepath.SkipDir
		}
	case LTWhitelist:
		if path == sw.Root {
			return false, nil
		}
		return sw.whiteListDir(path)
	}
	return false, nil
}

func (sw *Skywalker) whiteListDir(path string) (bool, error) {
	dirs := strings.Split(cleanDir(strings.Replace(path, sw.Root, "", 1)), string(filepath.Separator))
	for i := 1; i < len(dirs)+1; i++ {
		try := filepath.Join(sw.Root, filepath.Join(dirs[:i]...))
		root, found := sw.dirMap[try]
		if found && root {
			return false, nil // if it is the root no need to continue. Just use it
		}
		if !found {
			return true, filepath.SkipDir // if it was not found at all ignore. the order of the search is important
		}
	}
	return true, nil // if it was found but not the root and was the last iteration
}

func (sw *Skywalker) skipFile(path string) bool {
	dir, name := filepath.Split(path)
	if sw.DirListType == LTWhitelist {
		if skipDir, _ := sw.whiteListDir(dir); skipDir {
			return true
		}
	}
	_, inList := sw.extMap[filepath.Ext(name)]
	switch sw.ExtListType {
	case LTBlacklist:
		if inList {
			return true
		}
	case LTWhitelist:
		if !inList {
			return true
		}
	}
	return false
}

func (sw *Skywalker) matchPath(path string) bool {
	for _, gl := range sw.list {
		if match := gl.Match(path); match {
			return true
		}
	}
	return false
}

func cleanDir(dir string) string {
	return strings.Trim(dir, "/")
}
<commit_msg>Fixed an issue with cleaning and storing the path on Windows machines<commit_after>//Copyright (c) 2017, Will Dixon. All rights reserved.
//Use of this source code is governed by a BSD-style
//license that can be found in the LICENSE file.

//Package skywalker walks through a filesystem concurrently.
package skywalker

import (
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/gobwas/glob"
)

//Worker is anything that knows what to do with a path.
type Worker interface {
	Work(path string)
}

//ListType is used to specify how to handle the contents of a list
type ListType int

const (
	//LTBlacklist is used to specify that a list is to exclude the contents.
	LTBlacklist ListType = iota
	//LTWhitelist is used to specify that a list is to include the contents.
	LTWhitelist
)

//Skywalker can concurrently go through files in Root and call Worker on everything.
//It is recommended to use DirList and ExtList as much as possible as it is more perfomant than List.
type Skywalker struct {
	//Root is where the file walker starts. It is converted to an absolute path before start.
	Root string

	//List and ListType should only be used for fine filtering of paths.
	//It uses https://github.com/gobwas/glob for glob checking on each patch check.
	ListType ListType
	List     []string
	list     []glob.Glob

	//ExtList and ExtListType are used to narrow down the files by their extensions.
	//Make sure to include the preceding ".".
	ExtListType ListType
	ExtList     []string
	extMap      map[string]struct{}

	//DirList and DirListType are used to narrow down by directories.
	//Will skip the appropriate directories and their files/subfolders.
	DirListType ListType
	DirList     []string
	dirMap      map[string]bool

	//NumWorkers are how many workers are listening to the queue to do the work.
	NumWorkers int

	//QueueSize is how many paths to queue up at a time.
	//Useful for fine control over memory usage if needed.
	QueueSize int

	//Worker is the function that is called on each file/directory.
	Worker Worker

	//FilesOnly should be set to true if you only want to queue up files.
	FilesOnly bool
}

//New creates a new Skywalker that can walk through the specified root and calls the Worker on each file and/or directory.
//Defaults Skywalker to have 20 workers, a QueueSize of 100 and only queue files.
func New(root string, worker Worker) *Skywalker {
	return &Skywalker{
		Root:       root,
		NumWorkers: 20,
		QueueSize:  100,
		Worker:     worker,
		FilesOnly:  true,
	}
}

//Walk goes through the files and folders in Root and calls the worker on each.
//Checks the lists specified to check whether it should ignore files or directories.
//It also handles the creation of workers and queues needed for walking.
func (sw *Skywalker) Walk() error {
	if err := sw.init(); err != nil {
		return err
	}
	workerChan := make(chan string, sw.QueueSize)
	workerWG := new(sync.WaitGroup)
	workerWG.Add(sw.NumWorkers)
	for i := 0; i < sw.NumWorkers; i++ {
		go sw.worker(workerWG, workerChan)
	}
	err := filepath.Walk(sw.Root, sw.walker(workerChan))
	close(workerChan)
	workerWG.Wait()
	return err
}

func (sw *Skywalker) init() error {
	root, err := filepath.Abs(sw.Root)
	if err != nil {
		return err
	}
	sw.Root = root
	dirMap := make(map[string]bool, len(sw.DirList))
	for _, dir := range sw.DirList {
		if sw.DirListType == LTWhitelist {
			dirs := splitPath(dir)
			for i := len(dirs); i > 0; i-- {
				dirMap[filepath.Join(root, filepath.Join(dirs[:i]...))] = i == len(dirs)
			}
		} else {
			dirMap[filepath.Join(root, cleanDir(dir))] = true
		}
	}
	sw.dirMap = dirMap
	extMap := make(map[string]struct{}, len(sw.ExtList))
	for _, ext := range sw.ExtList {
		extMap[ext] = struct{}{}
	}
	sw.extMap = extMap
	list := make([]glob.Glob, len(sw.List))
	for i, g := range sw.List {
		gl, er := glob.Compile(g)
		if er != nil {
			return er
		}
		list[i] = gl
	}
	sw.list = list
	return nil
}

func (sw *Skywalker) worker(workerWG *sync.WaitGroup, workerChan chan string) {
	defer workerWG.Done()
	for w := range workerChan {
		sw.Worker.Work(w)
	}
}

func (sw *Skywalker) walker(workerChan chan string) func(path string, info os.FileInfo, err error) error {
	return func(path string, info os.FileInfo, _ error) error {
		if info.IsDir() {
			if doSomething, err := sw.skipDir(path); doSomething {
				return err
			}
			if sw.FilesOnly {
				return nil
			}
		} else {
			if sw.skipFile(path) {
				return nil
			}
		}
		if sw.matchPath(path) == (sw.ListType == LTBlacklist) {
			return nil
		}
		workerChan <- path
		return nil
	}
}

func (sw *Skywalker) skipDir(path string) (bool, error) {
	switch sw.DirListType {
	case LTBlacklist:
		_, inList := sw.dirMap[path]
		if inList {
			return true, filepath.SkipDir
		}
	case LTWhitelist:
		if path == sw.Root {
			return false, nil
		}
		return sw.whiteListDir(path)
	}
	return false, nil
}

func (sw *Skywalker) whiteListDir(path string) (bool, error) {
	dirs := splitPath(strings.Replace(path, sw.Root, "", 1))
	for i := 1; i < len(dirs)+1; i++ {
		try := filepath.Join(sw.Root, filepath.Join(dirs[:i]...))
		root, found := sw.dirMap[try]
		if found && root {
			return false, nil // if it is the root no need to continue. Just use it
		}
		if !found {
			return true, filepath.SkipDir // if it was not found at all ignore. the order of the search is important
		}
	}
	return true, nil // if it was found but not the root and was the last iteration
}

func (sw *Skywalker) skipFile(path string) bool {
	dir, name := filepath.Split(path)
	if sw.DirListType == LTWhitelist {
		if skipDir, _ := sw.whiteListDir(dir); skipDir {
			return true
		}
	}
	_, inList := sw.extMap[filepath.Ext(name)]
	switch sw.ExtListType {
	case LTBlacklist:
		if inList {
			return true
		}
	case LTWhitelist:
		if !inList {
			return true
		}
	}
	return false
}

func (sw *Skywalker) matchPath(path string) bool {
	for _, gl := range sw.list {
		if match := gl.Match(path); match {
			return true
		}
	}
	return false
}

func cleanDir(dir string) string {
	return filepath.Clean(dir)
}

func splitPath(path string) []string {
	return strings.Split(strings.Trim(cleanDir(path), string(filepath.Separator)), string(filepath.Separator))
}
<|endoftext|>


<commit_before>//Copyright (c) 2017, Will Dixon. All rights reserved.
//Use of this source code is governed by a BSD-style
//license that can be found in the LICENSE file.

//Package skywalker walks through a filesystem concurrently.
package skywalker

import (
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/gobwas/glob"
)

//Worker is anything that knows what to do with a path.
type Worker interface {
	Work(path string)
}

//ListType is used to specify how to handle the contents of a list
type ListType int

const (
	//LTBlacklist is used to specify that a list is to exclude the contents.
	LTBlacklist ListType = iota
	//LTWhitelist is used to specify that a list is to include the contents.
	LTWhitelist
)

//Skywalker can concurrently go through files in Root and call Worker on everything.
//It is recommended to use DirList and ExtList as much as possible as it is more perfomant than List.
type Skywalker struct {
	//Root is where the file walker starts. It is converted to an absolute path before start.
	Root string

	//List and ListType should only be used for fine filtering of paths.
	//It uses https://github.com/gobwas/glob for glob checking on each patch check.
	ListType ListType
	List     []string
	list     []glob.Glob

	//ExtList and ExtListType are used to narrow down the files by their extensions.
	//Make sure to include the preceding ".".
	ExtListType ListType
	ExtList     []string
	extMap      map[string]struct{}

	//DirList and DirListType are used to narrow down by directories.
	//Will skip the appropriate directories and their files/subfolders.
	DirListType ListType
	DirList     []string
	dirMap      map[string]bool

	//NumWorkers are how many workers are listening to the queue to do the work.
	NumWorkers int

	//QueueSize is how many paths to queue up at a time.
	//Useful for fine control over memory usage if needed.
	QueueSize int

	//Worker is the function that is called on each file/directory.
	Worker Worker

	//FilesOnly should be set to true if you only want to queue up files.
	FilesOnly bool
}

//New creates a new Skywalker that can walk through the specified root and calls the Worker on each file and/or directory.
//Defaults Skywalker to have 20 workers, a QueueSize of 100 and only queue files.
func New(root string, worker Worker) *Skywalker {
	return &Skywalker{
		Root:       root,
		NumWorkers: 20,
		QueueSize:  100,
		Worker:     worker,
		FilesOnly:  true,
	}
}

//Walk goes through the files and folders in Root and calls the worker on each.
//Checks the lists specified to check whether it should ignore files or directories.
//It also handles the creation of workers and queues needed for walking.
func (sw *Skywalker) Walk() error {
	if err := sw.init(); err != nil {
		return err
	}
	workerChan := make(chan string, sw.QueueSize)
	workerWG := new(sync.WaitGroup)
	workerWG.Add(sw.NumWorkers)
	for i := 0; i < sw.NumWorkers; i++ {
		go sw.worker(workerWG, workerChan)
	}
	err := filepath.Walk(sw.Root, sw.walker(workerChan))
	close(workerChan)
	workerWG.Wait()
	return err
}

func (sw *Skywalker) init() error {
	root, err := filepath.Abs(sw.Root)
	if err != nil {
		return err
	}
	sw.Root = root
	dirMap := make(map[string]bool, len(sw.DirList))
	for _, dir := range sw.DirList {
		if sw.DirListType == LTWhitelist {
			dirs := strings.Split(cleanDir(dir), string(filepath.Separator))
			for i := len(dirs); i > 0; i-- {
				dirMap[filepath.Join(root, filepath.Join(dirs[:i]...))] = i == len(dirs)
			}
		} else {
			dirMap[filepath.Join(root, cleanDir(dir))] = true
		}
	}
	sw.dirMap = dirMap
	extMap := make(map[string]struct{}, len(sw.ExtList))
	for _, ext := range sw.ExtList {
		extMap[ext] = struct{}{}
	}
	sw.extMap = extMap
	list := make([]glob.Glob, len(sw.List))
	for i, g := range sw.List {
		gl, er := glob.Compile(g)
		if er != nil {
			return er
		}
		list[i] = gl
	}
	sw.list = list
	return nil
}

func (sw *Skywalker) worker(workerWG *sync.WaitGroup, workerChan chan string) {
	defer workerWG.Done()
	for w := range workerChan {
		sw.Worker.Work(w)
	}
}

func (sw *Skywalker) walker(workerChan chan string) func(path string, info os.FileInfo, err error) error {
	return func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			if doSomething, err := sw.skipDir(path); doSomething {
				return err
			}
			if sw.FilesOnly {
				return nil
			}
		} else {
			if sw.skipFile(info.Name()) {
				return nil
			}
		}
		if sw.matchPath(path) == (sw.ListType == LTBlacklist) {
			return nil
		}
		workerChan <- path
		return nil
	}
}

func (sw *Skywalker) skipDir(path string) (bool, error) {
	switch sw.DirListType {
	case LTBlacklist:
		_, inList := sw.dirMap[path]
		if inList {
			return true, filepath.SkipDir
		}
	case LTWhitelist:
		if path == sw.Root {
			return false, nil
		}
		dirs := strings.Split(cleanDir(strings.Replace(path, sw.Root, "", 1)), string(filepath.Separator))
		for i := 1; i < len(dirs)+1; i++ {
			try := filepath.Join(sw.Root, filepath.Join(dirs[:i]...))
			root, found := sw.dirMap[try]
			if found && root {
				return false, nil // if it is the root no need to continue. Just use it
			}
			if !found {
				return true, filepath.SkipDir // if it was not found at all ignore. the order of the search is important
			}
		}
		return true, nil // if it was found but not the root and was the last iteration
	}
	return false, nil
}

func (sw *Skywalker) skipFile(name string) bool {
	_, inList := sw.extMap[filepath.Ext(name)]
	switch sw.ExtListType {
	case LTBlacklist:
		if inList {
			return true
		}
	case LTWhitelist:
		if !inList {
			return true
		}
	}
	return false
}

func (sw *Skywalker) matchPath(path string) bool {
	for _, gl := range sw.list {
		if match := gl.Match(path); match {
			return true
		}
	}
	return false
}

func cleanDir(dir string) string {
	return strings.Trim(dir, "/")
}
<commit_msg>Ignore files of parent folders of whitelisted folders<commit_after>//Copyright (c) 2017, Will Dixon. All rights reserved.
//Use of this source code is governed by a BSD-style
//license that can be found in the LICENSE file.

//Package skywalker walks through a filesystem concurrently.
package skywalker

import (
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/gobwas/glob"
)

//Worker is anything that knows what to do with a path.
type Worker interface {
	Work(path string)
}

//ListType is used to specify how to handle the contents of a list
type ListType int

const (
	//LTBlacklist is used to specify that a list is to exclude the contents.
	LTBlacklist ListType = iota
	//LTWhitelist is used to specify that a list is to include the contents.
	LTWhitelist
)

//Skywalker can concurrently go through files in Root and call Worker on everything.
//It is recommended to use DirList and ExtList as much as possible as it is more perfomant than List.
type Skywalker struct {
	//Root is where the file walker starts. It is converted to an absolute path before start.
	Root string

	//List and ListType should only be used for fine filtering of paths.
	//It uses https://github.com/gobwas/glob for glob checking on each patch check.
	ListType ListType
	List     []string
	list     []glob.Glob

	//ExtList and ExtListType are used to narrow down the files by their extensions.
	//Make sure to include the preceding ".".
	ExtListType ListType
	ExtList     []string
	extMap      map[string]struct{}

	//DirList and DirListType are used to narrow down by directories.
	//Will skip the appropriate directories and their files/subfolders.
	DirListType ListType
	DirList     []string
	dirMap      map[string]bool

	//NumWorkers are how many workers are listening to the queue to do the work.
	NumWorkers int

	//QueueSize is how many paths to queue up at a time.
	//Useful for fine control over memory usage if needed.
	QueueSize int

	//Worker is the function that is called on each file/directory.
	Worker Worker

	//FilesOnly should be set to true if you only want to queue up files.
	FilesOnly bool
}

//New creates a new Skywalker that can walk through the specified root and calls the Worker on each file and/or directory.
//Defaults Skywalker to have 20 workers, a QueueSize of 100 and only queue files.
func New(root string, worker Worker) *Skywalker {
	return &Skywalker{
		Root:       root,
		NumWorkers: 20,
		QueueSize:  100,
		Worker:     worker,
		FilesOnly:  true,
	}
}

//Walk goes through the files and folders in Root and calls the worker on each.
//Checks the lists specified to check whether it should ignore files or directories.
//It also handles the creation of workers and queues needed for walking.
func (sw *Skywalker) Walk() error {
	if err := sw.init(); err != nil {
		return err
	}
	workerChan := make(chan string, sw.QueueSize)
	workerWG := new(sync.WaitGroup)
	workerWG.Add(sw.NumWorkers)
	for i := 0; i < sw.NumWorkers; i++ {
		go sw.worker(workerWG, workerChan)
	}
	err := filepath.Walk(sw.Root, sw.walker(workerChan))
	close(workerChan)
	workerWG.Wait()
	return err
}

func (sw *Skywalker) init() error {
	root, err := filepath.Abs(sw.Root)
	if err != nil {
		return err
	}
	sw.Root = root
	dirMap := make(map[string]bool, len(sw.DirList))
	for _, dir := range sw.DirList {
		if sw.DirListType == LTWhitelist {
			dirs := strings.Split(cleanDir(dir), string(filepath.Separator))
			for i := len(dirs); i > 0; i-- {
				dirMap[filepath.Join(root, filepath.Join(dirs[:i]...))] = i == len(dirs)
			}
		} else {
			dirMap[filepath.Join(root, cleanDir(dir))] = true
		}
	}
	sw.dirMap = dirMap
	extMap := make(map[string]struct{}, len(sw.ExtList))
	for _, ext := range sw.ExtList {
		extMap[ext] = struct{}{}
	}
	sw.extMap = extMap
	list := make([]glob.Glob, len(sw.List))
	for i, g := range sw.List {
		gl, er := glob.Compile(g)
		if er != nil {
			return er
		}
		list[i] = gl
	}
	sw.list = list
	return nil
}

func (sw *Skywalker) worker(workerWG *sync.WaitGroup, workerChan chan string) {
	defer workerWG.Done()
	for w := range workerChan {
		sw.Worker.Work(w)
	}
}

func (sw *Skywalker) walker(workerChan chan string) func(path string, info os.FileInfo, err error) error {
	return func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			if doSomething, err := sw.skipDir(path); doSomething {
				return err
			}
			if sw.FilesOnly {
				return nil
			}
		} else {
			if sw.skipFile(path) {
				return nil
			}
		}
		if sw.matchPath(path) == (sw.ListType == LTBlacklist) {
			return nil
		}
		workerChan <- path
		return nil
	}
}

func (sw *Skywalker) skipDir(path string) (bool, error) {
	switch sw.DirListType {
	case LTBlacklist:
		_, inList := sw.dirMap[path]
		if inList {
			return true, filepath.SkipDir
		}
	case LTWhitelist:
		if path == sw.Root {
			return false, nil
		}
		return sw.whiteListDir(path)
	}
	return false, nil
}

func (sw *Skywalker) whiteListDir(path string) (bool, error) {
	dirs := strings.Split(cleanDir(strings.Replace(path, sw.Root, "", 1)), string(filepath.Separator))
	for i := 1; i < len(dirs)+1; i++ {
		try := filepath.Join(sw.Root, filepath.Join(dirs[:i]...))
		root, found := sw.dirMap[try]
		if found && root {
			return false, nil // if it is the root no need to continue. Just use it
		}
		if !found {
			return true, filepath.SkipDir // if it was not found at all ignore. the order of the search is important
		}
	}
	return true, nil // if it was found but not the root and was the last iteration
}

func (sw *Skywalker) skipFile(path string) bool {
	dir, name := filepath.Split(path)
	if sw.DirListType == LTWhitelist {
		if skipDir, _ := sw.whiteListDir(dir); skipDir {
			return true
		}
	}
	_, inList := sw.extMap[filepath.Ext(name)]
	switch sw.ExtListType {
	case LTBlacklist:
		if inList {
			return true
		}
	case LTWhitelist:
		if !inList {
			return true
		}
	}
	return false
}

func (sw *Skywalker) matchPath(path string) bool {
	for _, gl := range sw.list {
		if match := gl.Match(path); match {
			return true
		}
	}
	return false
}

func cleanDir(dir string) string {
	return strings.Trim(dir, "/")
}
<|endoftext|>