<commit_before>/*
 * Minio Client (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"fmt"
	"path/filepath"
	"strings"

	"github.com/minio/cli"
	"github.com/minio/mc/pkg/client"
	"github.com/minio/minio-xl/pkg/probe"
)

type copyURLs struct {
	SourceContent *client.Content
	TargetContent *client.Content
	Error         *probe.Error `json:"-"`
}

type copyURLsType uint8

const (
	copyURLsTypeInvalid copyURLsType = iota
	copyURLsTypeA                    // file to file
	copyURLsTypeB                    // file to dir
	copyURLsTypeC                    // recursive to dir
	copyURLsTypeD                    // complex to dir
)

//   NOTE: All the parse rules should reduced to A: Copy(Source, Target).
//
//   * VALID RULES
//   =======================
//   A: copy(f, f) -> copy(f, f)
//   B: copy(f, d) -> copy(f, d/f) -> A
//   C: copy(d1..., d2) -> []copy(d1/f, d2/d1/f) -> []A
//   D: copy([]{d1... | f}, d2) -> []{copy(d1/f, d2/d1/f) | copy(f, d2/f )} -> []A
//
//   * INVALID RULES
//   =========================
//   A: copy(d, *)
//   B: copy(d..., f)
//   C: copy(*, d...)
//
func checkCopySyntax(ctx *cli.Context) {
	if len(ctx.Args()) < 2 {
		cli.ShowCommandHelpAndExit(ctx, "cp", 1) // last argument is exit code.
	}
	// extract URLs.
	URLs, err := args2URLs(ctx.Args())
	fatalIf(err.Trace(ctx.Args()...), fmt.Sprintf("One or more unknown URL types passed."))

	srcURLs := URLs[:len(URLs)-1]
	tgtURL := URLs[len(URLs)-1]

	/****** Generic rules *******/
	// Recursive URLs are not allowed in target.
	if isURLRecursive(tgtURL) {
		fatalIf(errDummy().Trace(), fmt.Sprintf("Recursive option is not supported for target ‘%s’ argument.", tgtURL))
	}

	url := client.NewURL(tgtURL)
	if url.Host != "" {
		// This check is for type URL.
		if url.Path == string(url.Separator) {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("Target ‘%s’ does not contain bucket name.", tgtURL))
		}
	}

	switch guessCopyURLType(srcURLs, tgtURL) {
	case copyURLsTypeA: // File -> File.
		checkCopySyntaxTypeA(srcURLs, tgtURL)
	case copyURLsTypeB: // File -> Folder.
		checkCopySyntaxTypeB(srcURLs, tgtURL)
	case copyURLsTypeC: // Folder... -> Folder.
		checkCopySyntaxTypeC(srcURLs, tgtURL)
	case copyURLsTypeD: // File | Folder... -> Folder.
		checkCopySyntaxTypeD(srcURLs, tgtURL)
	default:
		fatalIf(errInvalidArgument().Trace(), "Invalid arguments to copy command.")
	}
}

// checkCopySyntaxTypeA verifies if the source and target are valid file arguments.
func checkCopySyntaxTypeA(srcURLs []string, tgtURL string) {
	if len(srcURLs) != 1 {
		fatalIf(errInvalidArgument().Trace(), "Invalid number of source arguments to copy command.")
	}
	srcURL := srcURLs[0]
	_, srcContent, err := url2Stat(srcURL)
	fatalIf(err.Trace(srcURL), "Unable to stat source ‘"+srcURL+"’.")

	if srcContent.Type.IsDir() {
		fatalIf(errSourceIsDir(srcURL).Trace(srcURL), fmt.Sprintf("Folder cannot copied. Please use ‘%s...’ to copy this folder and its contents recursively.", srcURL))
	}
	if !srcContent.Type.IsRegular() {
		fatalIf(errInvalidArgument().Trace(), "Source ‘"+srcURL+"’ is not a file.")
	}
}

// checkCopySyntaxTypeB verifies if the source is a valid file and target is a valid dir.
func checkCopySyntaxTypeB(srcURLs []string, tgtURL string) {
	if len(srcURLs) != 1 {
		fatalIf(errInvalidArgument().Trace(), "Invalid number of source arguments to copy command.")
	}
	srcURL := srcURLs[0]
	_, srcContent, err := url2Stat(srcURL)
	fatalIf(err.Trace(srcURL), "Unable to stat source ‘"+srcURL+"’.")

	if srcContent.Type.IsDir() {
		fatalIf(errSourceIsDir(srcURL).Trace(srcURL), fmt.Sprintf("Folder cannot be copied. Please use ‘%s...’ argument to copy this folder and its contents recursively.", srcURL))
	}
	if !srcContent.Type.IsRegular() {
		fatalIf(errInvalidArgument().Trace(srcURL), "Source ‘"+srcURL+"’ is not a file.")
	}

	_, tgtContent, err := url2Stat(tgtURL)
	// Target exist?.
	if err == nil {
		if !tgtContent.Type.IsDir() {
			fatalIf(errInvalidArgument().Trace(tgtURL), "Target ‘"+tgtURL+"’ is not a folder.")
		}
	}
}

// checkCopySyntaxTypeC verifies if the source is a valid recursive dir and target is a valid dir.
func checkCopySyntaxTypeC(srcURLs []string, tgtURL string) {
	if len(srcURLs) != 1 {
		fatalIf(errInvalidArgument().Trace(), "Invalid number of source arguments to copy command.")
	}

	srcURL := srcURLs[0]
	srcURL = stripRecursiveURL(srcURL)
	_, srcContent, err := url2Stat(srcURL)
	if err != nil && !prefixExists(srcURL) {
		fatalIf(err.Trace(srcURL), "Unable to stat source ‘"+srcURL+"’.")
	}
	if err == nil && srcContent.Type.IsRegular() { // Ellipses is supported only for folders.
		fatalIf(errInvalidArgument().Trace(), "Source ‘"+srcURL+"’ is not a folder.")
	}
	_, tgtContent, err := url2Stat(tgtURL)
	// Target exist?.
	if err == nil {
		if !tgtContent.Type.IsDir() {
			fatalIf(errInvalidArgument().Trace(tgtURL), "Target ‘"+tgtURL+"’ is not a folder.")
		}
	}
}

// checkCopySyntaxTypeD verifies if the source is a valid list of file or valid recursive dir and target is a valid dir.
func checkCopySyntaxTypeD(srcURLs []string, tgtURL string) {
	for _, srcURL := range srcURLs {
		if isURLRecursive(srcURL) {
			srcURL = stripRecursiveURL(srcURL)
			_, srcContent, err := url2Stat(srcURL)
			if err != nil && !prefixExists(srcURL) {
				fatalIf(err.Trace(srcURL), "Unable to stat source ‘"+srcURL+"’.")
			}
			if err == nil && !srcContent.Type.IsDir() { // Ellipses is supported only for folders.
				fatalIf(errInvalidArgument().Trace(srcURL), "Source ‘"+srcURL+"’ is not a folder.")
			}
		}
	}
	_, tgtContent, err := url2Stat(tgtURL)
	// Target exist?.
	if err == nil {
		if !tgtContent.Type.IsDir() {
			fatalIf(errInvalidArgument().Trace(tgtURL), "Target ‘"+tgtURL+"’ is not a folder.")
		}
	}
}

// guessCopyURLType guesses the type of URL. This approach all allows prepareURL
// functions to accurately report failure causes.
func guessCopyURLType(sourceURLs []string, targetURL string) copyURLsType {
	if strings.TrimSpace(targetURL) == "" || targetURL == "" { // Target is empty
		return copyURLsTypeInvalid
	}
	if len(sourceURLs) == 0 || sourceURLs == nil { // Source list is empty
		return copyURLsTypeInvalid
	}
	for _, sourceURL := range sourceURLs {
		if sourceURL == "" { // One of the source is empty
			return copyURLsTypeInvalid
		}
	}
	if len(sourceURLs) == 1 { // 1 Source, 1 Target
		switch {
		// Type C
		case isURLRecursive(sourceURLs[0]):
			return copyURLsTypeC
		// Type B
		case isTargetURLDir(targetURL):
			return copyURLsTypeB
		// Type A
		default:
			return copyURLsTypeA
		}
	} // else Type D
	return copyURLsTypeD
}

// SINGLE SOURCE - Type A: copy(f, f) -> copy(f, f)
// prepareCopyURLsTypeA - prepares target and source URLs for copying.
func prepareCopyURLsTypeA(sourceURL string, targetURL string) copyURLs {
	_, sourceContent, err := url2Stat(sourceURL)
	if err != nil {
		// Source does not exist or insufficient privileges.
		return copyURLs{Error: err.Trace(sourceURL)}
	}
	if !sourceContent.Type.IsRegular() {
		// Source is not a regular file
		return copyURLs{Error: errInvalidSource(sourceURL).Trace()}
	}

	// All OK.. We can proceed. Type A
	return copyURLs{SourceContent: sourceContent, TargetContent: &client.Content{URL: *client.NewURL(targetURL)}}
}

// SINGLE SOURCE - Type B: copy(f, d) -> copy(f, d/f) -> A
// prepareCopyURLsTypeB - prepares target and source URLs for copying.
func prepareCopyURLsTypeB(sourceURL string, targetURL string) copyURLs {
	_, sourceContent, err := url2Stat(sourceURL)
	if err != nil {
		// Source does not exist or insufficient privileges.
		return copyURLs{Error: err.Trace(sourceURL)}
	}

	if !sourceContent.Type.IsRegular() {
		if sourceContent.Type.IsDir() {
			return copyURLs{Error: errSourceIsDir(sourceURL).Trace()}
		}
		// Source is not a regular file.
		return copyURLs{Error: errInvalidSource(sourceURL).Trace()}
	}

	// All OK.. We can proceed. Type B: source is a file, target is a folder and exists.
	sourceURLParse := client.NewURL(sourceURL)
	targetURLParse := client.NewURL(targetURL)
	targetURLParse.Path = filepath.Join(targetURLParse.Path, filepath.Base(sourceURLParse.Path))
	return prepareCopyURLsTypeA(sourceURL, targetURLParse.String())
}

// SINGLE SOURCE - Type C: copy(d1..., d2) -> []copy(d1/f, d1/d2/f) -> []A
// prepareCopyRecursiveURLTypeC - prepares target and source URLs for copying.
func prepareCopyURLsTypeC(sourceURL, targetURL string) <-chan copyURLs {
	copyURLsCh := make(chan copyURLs)
	go func(sourceURL, targetURL string, copyURLsCh chan copyURLs) {
		defer close(copyURLsCh)
		if !isURLRecursive(sourceURL) {
			// Source is not of recursive type.
			copyURLsCh <- copyURLs{Error: errSourceNotRecursive(sourceURL).Trace()}
			return
		}

		// add `/` after trimming off `...` to emulate folders
		sourceURL = stripRecursiveURL(sourceURL)
		sourceClient, err := url2Client(sourceURL)
		if err != nil {
			// Source initialization failed.
			copyURLsCh <- copyURLs{Error: err.Trace(sourceURL)}
			return
		}

		for sourceContent := range sourceClient.List(true, false) {
			if sourceContent.Err != nil {
				// Listing failed.
				copyURLsCh <- copyURLs{Error: sourceContent.Err.Trace()}
				continue
			}

			if !sourceContent.Content.Type.IsRegular() {
				// Source is not a regular file. Skip it for copy.
				continue
			}

			// All OK.. We can proceed. Type B: source is a file, target is a folder and exists.
			newSourceURL := sourceContent.Content.URL.String()
			newSourceSuffix := strings.TrimPrefix(newSourceURL, sourceURL)
			newTargetURL := urlJoinPath(targetURL, newSourceSuffix)
			copyURLsCh <- prepareCopyURLsTypeA(sourceContent.Content.URL.String(), newTargetURL)
		}
	}(sourceURL, targetURL, copyURLsCh)
	return copyURLsCh
}

// MULTI-SOURCE - Type D: copy([]f, d) -> []B
// prepareCopyURLsTypeD - prepares target and source URLs for copying.
func prepareCopyURLsTypeD(sourceURLs []string, targetURL string) <-chan copyURLs {
	copyURLsCh := make(chan copyURLs)
	go func(sourceURLs []string, targetURL string, copyURLsCh chan copyURLs) {
		defer close(copyURLsCh)

		if sourceURLs == nil {
			// Source list is empty.
			copyURLsCh <- copyURLs{Error: errSourceListEmpty().Trace()}
			return
		}

		for _, sourceURL := range sourceURLs {
			// Target is folder. Possibilities are only Type B and C
			// Is it a recursive URL "..."?
			if isURLRecursive(sourceURL) {
				for cURLs := range prepareCopyURLsTypeC(sourceURL, targetURL) {
					copyURLsCh <- cURLs
				}
			} else {
				copyURLsCh <- prepareCopyURLsTypeB(sourceURL, targetURL)
			}
		}
	}(sourceURLs, targetURL, copyURLsCh)
	return copyURLsCh
}

// prepareCopyURLs - prepares target and source URLs for copying.
func prepareCopyURLs(sourceURLs []string, targetURL string) <-chan copyURLs {
	copyURLsCh := make(chan copyURLs)
	go func(sourceURLs []string, targetURL string, copyURLsCh chan copyURLs) {
		defer close(copyURLsCh)
		switch guessCopyURLType(sourceURLs, targetURL) {
		case copyURLsTypeA:
			copyURLsCh <- prepareCopyURLsTypeA(sourceURLs[0], targetURL)
		case copyURLsTypeB:
			copyURLsCh <- prepareCopyURLsTypeB(sourceURLs[0], targetURL)
		case copyURLsTypeC:
			for cURLs := range prepareCopyURLsTypeC(sourceURLs[0], targetURL) {
				copyURLsCh <- cURLs
			}
		case copyURLsTypeD:
			for cURLs := range prepareCopyURLsTypeD(sourceURLs, targetURL) {
				copyURLsCh <- cURLs
			}
		default:
			copyURLsCh <- copyURLs{Error: errInvalidArgument().Trace()}
		}
	}(sourceURLs, targetURL, copyURLsCh)

	return copyURLsCh
}
<commit_msg>cp: more trimming necessary.<commit_after>/*
 * Minio Client (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"fmt"
	"path/filepath"
	"strings"

	"github.com/minio/cli"
	"github.com/minio/mc/pkg/client"
	"github.com/minio/minio-xl/pkg/probe"
)

type copyURLs struct {
	SourceContent *client.Content
	TargetContent *client.Content
	Error         *probe.Error `json:"-"`
}

type copyURLsType uint8

const (
	copyURLsTypeInvalid copyURLsType = iota
	copyURLsTypeA                    // file to file
	copyURLsTypeB                    // file to dir
	copyURLsTypeC                    // recursive to dir
	copyURLsTypeD                    // complex to dir
)

//   NOTE: All the parse rules should reduced to A: Copy(Source, Target).
//
//   * VALID RULES
//   =======================
//   A: copy(f, f) -> copy(f, f)
//   B: copy(f, d) -> copy(f, d/f) -> A
//   C: copy(d1..., d2) -> []copy(d1/f, d2/d1/f) -> []A
//   D: copy([]{d1... | f}, d2) -> []{copy(d1/f, d2/d1/f) | copy(f, d2/f )} -> []A
//
//   * INVALID RULES
//   =========================
//   A: copy(d, *)
//   B: copy(d..., f)
//   C: copy(*, d...)
//
func checkCopySyntax(ctx *cli.Context) {
	if len(ctx.Args()) < 2 {
		cli.ShowCommandHelpAndExit(ctx, "cp", 1) // last argument is exit code.
	}
	// extract URLs.
	URLs, err := args2URLs(ctx.Args())
	fatalIf(err.Trace(ctx.Args()...), fmt.Sprintf("One or more unknown URL types passed."))

	srcURLs := URLs[:len(URLs)-1]
	tgtURL := URLs[len(URLs)-1]

	/****** Generic rules *******/
	// Recursive URLs are not allowed in target.
	if isURLRecursive(tgtURL) {
		fatalIf(errDummy().Trace(), fmt.Sprintf("Recursive option is not supported for target ‘%s’ argument.", tgtURL))
	}

	url := client.NewURL(tgtURL)
	if url.Host != "" {
		// This check is for type URL.
		if url.Path == string(url.Separator) {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("Target ‘%s’ does not contain bucket name.", tgtURL))
		}
	}

	switch guessCopyURLType(srcURLs, tgtURL) {
	case copyURLsTypeA: // File -> File.
		checkCopySyntaxTypeA(srcURLs, tgtURL)
	case copyURLsTypeB: // File -> Folder.
		checkCopySyntaxTypeB(srcURLs, tgtURL)
	case copyURLsTypeC: // Folder... -> Folder.
		checkCopySyntaxTypeC(srcURLs, tgtURL)
	case copyURLsTypeD: // File | Folder... -> Folder.
		checkCopySyntaxTypeD(srcURLs, tgtURL)
	default:
		fatalIf(errInvalidArgument().Trace(), "Invalid arguments to copy command.")
	}
}

// checkCopySyntaxTypeA verifies if the source and target are valid file arguments.
func checkCopySyntaxTypeA(srcURLs []string, tgtURL string) {
	if len(srcURLs) != 1 {
		fatalIf(errInvalidArgument().Trace(), "Invalid number of source arguments to copy command.")
	}
	srcURL := srcURLs[0]
	_, srcContent, err := url2Stat(srcURL)
	fatalIf(err.Trace(srcURL), "Unable to stat source ‘"+srcURL+"’.")

	if srcContent.Type.IsDir() {
		fatalIf(errSourceIsDir(srcURL).Trace(srcURL), fmt.Sprintf("Folder cannot copied. Please use ‘%s...’ to copy this folder and its contents recursively.", srcURL))
	}
	if !srcContent.Type.IsRegular() {
		fatalIf(errInvalidArgument().Trace(), "Source ‘"+srcURL+"’ is not a file.")
	}
}

// checkCopySyntaxTypeB verifies if the source is a valid file and target is a valid dir.
func checkCopySyntaxTypeB(srcURLs []string, tgtURL string) {
	if len(srcURLs) != 1 {
		fatalIf(errInvalidArgument().Trace(), "Invalid number of source arguments to copy command.")
	}
	srcURL := srcURLs[0]
	_, srcContent, err := url2Stat(srcURL)
	fatalIf(err.Trace(srcURL), "Unable to stat source ‘"+srcURL+"’.")

	if srcContent.Type.IsDir() {
		fatalIf(errSourceIsDir(srcURL).Trace(srcURL), fmt.Sprintf("Folder cannot be copied. Please use ‘%s...’ argument to copy this folder and its contents recursively.", srcURL))
	}
	if !srcContent.Type.IsRegular() {
		fatalIf(errInvalidArgument().Trace(srcURL), "Source ‘"+srcURL+"’ is not a file.")
	}

	_, tgtContent, err := url2Stat(tgtURL)
	// Target exist?.
	if err == nil {
		if !tgtContent.Type.IsDir() {
			fatalIf(errInvalidArgument().Trace(tgtURL), "Target ‘"+tgtURL+"’ is not a folder.")
		}
	}
}

// checkCopySyntaxTypeC verifies if the source is a valid recursive dir and target is a valid dir.
func checkCopySyntaxTypeC(srcURLs []string, tgtURL string) {
	if len(srcURLs) != 1 {
		fatalIf(errInvalidArgument().Trace(), "Invalid number of source arguments to copy command.")
	}

	srcURL := srcURLs[0]
	srcURL = stripRecursiveURL(srcURL)
	_, srcContent, err := url2Stat(srcURL)
	if err != nil && !prefixExists(srcURL) {
		fatalIf(err.Trace(srcURL), "Unable to stat source ‘"+srcURL+"’.")
	}
	if err == nil && srcContent.Type.IsRegular() { // Ellipses is supported only for folders.
		fatalIf(errInvalidArgument().Trace(), "Source ‘"+srcURL+"’ is not a folder.")
	}
	_, tgtContent, err := url2Stat(tgtURL)
	// Target exist?.
	if err == nil {
		if !tgtContent.Type.IsDir() {
			fatalIf(errInvalidArgument().Trace(tgtURL), "Target ‘"+tgtURL+"’ is not a folder.")
		}
	}
}

// checkCopySyntaxTypeD verifies if the source is a valid list of file or valid recursive dir and target is a valid dir.
func checkCopySyntaxTypeD(srcURLs []string, tgtURL string) {
	for _, srcURL := range srcURLs {
		if isURLRecursive(srcURL) {
			srcURL = stripRecursiveURL(srcURL)
			_, srcContent, err := url2Stat(srcURL)
			if err != nil && !prefixExists(srcURL) {
				fatalIf(err.Trace(srcURL), "Unable to stat source ‘"+srcURL+"’.")
			}
			if err == nil && !srcContent.Type.IsDir() { // Ellipses is supported only for folders.
				fatalIf(errInvalidArgument().Trace(srcURL), "Source ‘"+srcURL+"’ is not a folder.")
			}
		}
	}
	_, tgtContent, err := url2Stat(tgtURL)
	// Target exist?.
	if err == nil {
		if !tgtContent.Type.IsDir() {
			fatalIf(errInvalidArgument().Trace(tgtURL), "Target ‘"+tgtURL+"’ is not a folder.")
		}
	}
}

// guessCopyURLType guesses the type of URL. This approach all allows prepareURL
// functions to accurately report failure causes.
func guessCopyURLType(sourceURLs []string, targetURL string) copyURLsType {
	if strings.TrimSpace(targetURL) == "" || targetURL == "" { // Target is empty
		return copyURLsTypeInvalid
	}
	if len(sourceURLs) == 0 || sourceURLs == nil { // Source list is empty
		return copyURLsTypeInvalid
	}
	for _, sourceURL := range sourceURLs {
		if sourceURL == "" { // One of the source is empty
			return copyURLsTypeInvalid
		}
	}
	if len(sourceURLs) == 1 { // 1 Source, 1 Target
		switch {
		// Type C
		case isURLRecursive(sourceURLs[0]):
			return copyURLsTypeC
		// Type B
		case isTargetURLDir(targetURL):
			return copyURLsTypeB
		// Type A
		default:
			return copyURLsTypeA
		}
	} // else Type D
	return copyURLsTypeD
}

// SINGLE SOURCE - Type A: copy(f, f) -> copy(f, f)
// prepareCopyURLsTypeA - prepares target and source URLs for copying.
func prepareCopyURLsTypeA(sourceURL string, targetURL string) copyURLs {
	_, sourceContent, err := url2Stat(sourceURL)
	if err != nil {
		// Source does not exist or insufficient privileges.
		return copyURLs{Error: err.Trace(sourceURL)}
	}
	if !sourceContent.Type.IsRegular() {
		// Source is not a regular file
		return copyURLs{Error: errInvalidSource(sourceURL).Trace()}
	}

	// All OK.. We can proceed. Type A
	return copyURLs{SourceContent: sourceContent, TargetContent: &client.Content{URL: *client.NewURL(targetURL)}}
}

// SINGLE SOURCE - Type B: copy(f, d) -> copy(f, d/f) -> A
// prepareCopyURLsTypeB - prepares target and source URLs for copying.
func prepareCopyURLsTypeB(sourceURL string, targetURL string) copyURLs {
	_, sourceContent, err := url2Stat(sourceURL)
	if err != nil {
		// Source does not exist or insufficient privileges.
		return copyURLs{Error: err.Trace(sourceURL)}
	}

	if !sourceContent.Type.IsRegular() {
		if sourceContent.Type.IsDir() {
			return copyURLs{Error: errSourceIsDir(sourceURL).Trace()}
		}
		// Source is not a regular file.
		return copyURLs{Error: errInvalidSource(sourceURL).Trace()}
	}

	// All OK.. We can proceed. Type B: source is a file, target is a folder and exists.
	sourceURLParse := client.NewURL(sourceURL)
	targetURLParse := client.NewURL(targetURL)
	targetURLParse.Path = filepath.Join(targetURLParse.Path, filepath.Base(sourceURLParse.Path))
	return prepareCopyURLsTypeA(sourceURL, targetURLParse.String())
}

// SINGLE SOURCE - Type C: copy(d1..., d2) -> []copy(d1/f, d1/d2/f) -> []A
// prepareCopyRecursiveURLTypeC - prepares target and source URLs for copying.
func prepareCopyURLsTypeC(sourceURL, targetURL string) <-chan copyURLs {
	copyURLsCh := make(chan copyURLs)
	go func(sourceURL, targetURL string, copyURLsCh chan copyURLs) {
		defer close(copyURLsCh)
		if !isURLRecursive(sourceURL) {
			// Source is not of recursive type.
			copyURLsCh <- copyURLs{Error: errSourceNotRecursive(sourceURL).Trace()}
			return
		}

		// add `/` after trimming off `...` to emulate folders
		sourceURL = stripRecursiveURL(sourceURL)
		sourceClient, err := url2Client(sourceURL)
		if err != nil {
			// Source initialization failed.
			copyURLsCh <- copyURLs{Error: err.Trace(sourceURL)}
			return
		}

		for sourceContent := range sourceClient.List(true, false) {
			if sourceContent.Err != nil {
				// Listing failed.
				copyURLsCh <- copyURLs{Error: sourceContent.Err.Trace()}
				continue
			}

			if !sourceContent.Content.Type.IsRegular() {
				// Source is not a regular file. Skip it for copy.
				continue
			}

			// All OK.. We can proceed. Type B: source is a file, target is a folder and exists.
			srcURL := sourceClient.GetURL()
			newSourceURL := sourceContent.Content.URL
			newSourceSuffix := strings.TrimPrefix(newSourceURL.Path,
				srcURL.Path[:strings.LastIndex(srcURL.Path, string(srcURL.Separator))])
			newTargetURL := urlJoinPath(targetURL, newSourceSuffix)
			copyURLsCh <- prepareCopyURLsTypeA(sourceContent.Content.URL.String(), newTargetURL)
		}
	}(sourceURL, targetURL, copyURLsCh)
	return copyURLsCh
}

// MULTI-SOURCE - Type D: copy([]f, d) -> []B
// prepareCopyURLsTypeD - prepares target and source URLs for copying.
func prepareCopyURLsTypeD(sourceURLs []string, targetURL string) <-chan copyURLs {
	copyURLsCh := make(chan copyURLs)
	go func(sourceURLs []string, targetURL string, copyURLsCh chan copyURLs) {
		defer close(copyURLsCh)

		if sourceURLs == nil {
			// Source list is empty.
			copyURLsCh <- copyURLs{Error: errSourceListEmpty().Trace()}
			return
		}

		for _, sourceURL := range sourceURLs {
			// Target is folder. Possibilities are only Type B and C
			// Is it a recursive URL "..."?
			if isURLRecursive(sourceURL) {
				for cURLs := range prepareCopyURLsTypeC(sourceURL, targetURL) {
					copyURLsCh <- cURLs
				}
			} else {
				copyURLsCh <- prepareCopyURLsTypeB(sourceURL, targetURL)
			}
		}
	}(sourceURLs, targetURL, copyURLsCh)
	return copyURLsCh
}

// prepareCopyURLs - prepares target and source URLs for copying.
func prepareCopyURLs(sourceURLs []string, targetURL string) <-chan copyURLs {
	copyURLsCh := make(chan copyURLs)
	go func(sourceURLs []string, targetURL string, copyURLsCh chan copyURLs) {
		defer close(copyURLsCh)
		switch guessCopyURLType(sourceURLs, targetURL) {
		case copyURLsTypeA:
			copyURLsCh <- prepareCopyURLsTypeA(sourceURLs[0], targetURL)
		case copyURLsTypeB:
			copyURLsCh <- prepareCopyURLsTypeB(sourceURLs[0], targetURL)
		case copyURLsTypeC:
			for cURLs := range prepareCopyURLsTypeC(sourceURLs[0], targetURL) {
				copyURLsCh <- cURLs
			}
		case copyURLsTypeD:
			for cURLs := range prepareCopyURLsTypeD(sourceURLs, targetURL) {
				copyURLsCh <- cURLs
			}
		default:
			copyURLsCh <- copyURLs{Error: errInvalidArgument().Trace()}
		}
	}(sourceURLs, targetURL, copyURLsCh)

	return copyURLsCh
}
<|endoftext|>


<commit_before>package main

import (
	t "github.com/nsf/termbox-go"
	"log"
)

var (
	width  int
	height int

	message string
)

var chActions = map[rune]func(){}

var keyActions = map[uint16]func(){}

func unknownAction() {
	panic("unrecognized keystroke")
}

func act(a func()) {
	defer func() {
		if er := recover(); er != nil {
			e, ok := er.(error)
			if ok {
				message = e.Error()
			} else {
				message = "unknown error"
			}
		}
	}()
	a()
}

func updateSize() {
	width = t.Width()
	height = t.Height()
}

func display() {

}

func UIMain() {
	t.Init()
	updateSize()
	e := t.Event{}
	for {
		display()
		e.Poll()
		switch e.Type {
		case t.EVENT_KEY:
			if e.Ch == 'q' {
				goto Exit
			}
			a, ok := chActions[e.Ch]
			if !ok {
				a, ok = keyActions[e.Key]
			}
			if ok {
				act(a)
			} else {
				act(unknownAction)
			}
		case t.EVENT_RESIZE:
			updateSize()
		default:
			log.Print("warning: unknown event type")
		}
	}

Exit:
	t.Shutdown()
}
<commit_msg>Displaying error message<commit_after>package main

import (
	t "github.com/nsf/termbox-go"
	"log"
)

var (
	width  int
	height int

	message string
)

var chActions = map[rune]func(){}

var keyActions = map[uint16]func(){}

func unknownAction() {
	panic("unrecognized keystroke")
}

func act(a func()) {
	defer func() {
		if er := recover(); er != nil {
			e, ok := er.(string)
			if ok {
				message = e
			} else {
				message = "unknown error"
			}
		}
	}()
	a()
}

func updateSize() {
	width = t.Width()
	height = t.Height()
}

func display() {
	t.Clear()
	i := 0
	for _, r := range message {
		t.ChangeCell(i, height-1, r, t.WHITE, t.BLACK)
		i++
	}
	t.Present()
}

func UIMain() {
	t.Init()
	updateSize()
	e := t.Event{}
	for {
		display()
		e.Poll()
		switch e.Type {
		case t.EVENT_KEY:
			if e.Ch == 'q' {
				goto Exit
			}
			a, ok := chActions[e.Ch]
			if !ok {
				a, ok = keyActions[e.Key]
			}
			if ok {
				act(a)
			} else {
				act(unknownAction)
			}
		case t.EVENT_RESIZE:
			updateSize()
		default:
			log.Print("warning: unknown event type")
		}
	}

Exit:
	t.Shutdown()
}
<|endoftext|>


<commit_before>package main

import (
	"math"

	"github.com/bcicen/ctop/connector"
	"github.com/bcicen/ctop/container"
	ui "github.com/gizak/termui"
)

type GridCursor struct {
	selectedID  string // id of currently selected container
	filtered    container.Containers
	cSource     connector.Connector
	isScrolling bool // toggled when actively scrolling
}

func (gc *GridCursor) Len() int { return len(gc.filtered) }

func (gc *GridCursor) Selected() *container.Container {
	idx := gc.Idx()
	if idx < gc.Len() {
		return gc.filtered[idx]
	}
	return nil
}

// Refresh containers from source
func (gc *GridCursor) RefreshContainers() (lenChanged bool) {
	oldLen := gc.Len()

	// Containers filtered by display bool
	gc.filtered = container.Containers{}
	var cursorVisible bool
	for _, c := range gc.cSource.All() {
		if c.Display {
			if c.Id == gc.selectedID {
				cursorVisible = true
			}
			gc.filtered = append(gc.filtered, c)
		}
	}

	if oldLen != gc.Len() {
		lenChanged = true
	}

	if !cursorVisible {
		gc.Reset()
	}
	if gc.selectedID == "" {
		gc.Reset()
	}
	return lenChanged
}

// Set an initial cursor position, if possible
func (gc *GridCursor) Reset() {
	for _, c := range gc.cSource.All() {
		c.Widgets.Name.UnHighlight()
	}
	if gc.Len() > 0 {
		gc.selectedID = gc.filtered[0].Id
		gc.filtered[0].Widgets.Name.Highlight()
	}
}

// Return current cursor index
func (gc *GridCursor) Idx() int {
	for n, c := range gc.filtered {
		if c.Id == gc.selectedID {
			return n
		}
	}
	gc.Reset()
	return 0
}

func (gc *GridCursor) ScrollPage() {
	// skip scroll if no need to page
	if gc.Len() < cGrid.MaxRows() {
		cGrid.Offset = 0
		return
	}

	idx := gc.Idx()

	// page down
	if idx >= cGrid.Offset+cGrid.MaxRows() {
		cGrid.Offset++
		cGrid.Align()
	}
	// page up
	if idx < cGrid.Offset {
		cGrid.Offset--
		cGrid.Align()
	}

}

func (gc *GridCursor) Up() {
	gc.isScrolling = true
	defer func() { gc.isScrolling = false }()

	idx := gc.Idx()
	if idx <= 0 { // already at top
		return
	}
	active := gc.filtered[idx]
	next := gc.filtered[idx-1]

	active.Widgets.Name.UnHighlight()
	gc.selectedID = next.Id
	next.Widgets.Name.Highlight()

	gc.ScrollPage()
	ui.Render(cGrid)
}

func (gc *GridCursor) Down() {
	gc.isScrolling = true
	defer func() { gc.isScrolling = false }()

	idx := gc.Idx()
	if idx >= gc.Len()-1 { // already at bottom
		return
	}
	active := gc.filtered[idx]
	next := gc.filtered[idx+1]

	active.Widgets.Name.UnHighlight()
	gc.selectedID = next.Id
	next.Widgets.Name.Highlight()

	gc.ScrollPage()
	ui.Render(cGrid)
}

func (gc *GridCursor) PgUp() {
	idx := gc.Idx()
	if idx <= 0 { // already at top
		return
	}

	var nextidx int
	nextidx = int(math.Max(0.0, float64(idx-cGrid.MaxRows())))
	cGrid.Offset = int(math.Max(float64(cGrid.Offset-cGrid.MaxRows()),
		float64(0)))

	active := gc.filtered[idx]
	next := gc.filtered[nextidx]

	active.Widgets.Name.UnHighlight()
	gc.selectedID = next.Id
	next.Widgets.Name.Highlight()

	cGrid.Align()
	ui.Render(cGrid)
}

func (gc *GridCursor) PgDown() {
	idx := gc.Idx()
	if idx >= gc.Len()-1 { // already at bottom
		return
	}

	var nextidx int
	nextidx = int(math.Min(float64(gc.Len()-1),
		float64(idx+cGrid.MaxRows())))
	cGrid.Offset = int(math.Min(float64(cGrid.Offset+cGrid.MaxRows()),
		float64(gc.Len()-cGrid.MaxRows())))

	active := gc.filtered[idx]
	next := gc.filtered[nextidx]

	active.Widgets.Name.UnHighlight()
	gc.selectedID = next.Id
	next.Widgets.Name.Highlight()

	cGrid.Align()
	ui.Render(cGrid)
}
<commit_msg>add pgCount() method to GridCursor<commit_after>package main

import (
	"math"

	"github.com/bcicen/ctop/connector"
	"github.com/bcicen/ctop/container"
	ui "github.com/gizak/termui"
)

type GridCursor struct {
	selectedID  string // id of currently selected container
	filtered    container.Containers
	cSource     connector.Connector
	isScrolling bool // toggled when actively scrolling
}

func (gc *GridCursor) Len() int { return len(gc.filtered) }

func (gc *GridCursor) Selected() *container.Container {
	idx := gc.Idx()
	if idx < gc.Len() {
		return gc.filtered[idx]
	}
	return nil
}

// Refresh containers from source
func (gc *GridCursor) RefreshContainers() (lenChanged bool) {
	oldLen := gc.Len()

	// Containers filtered by display bool
	gc.filtered = container.Containers{}
	var cursorVisible bool
	for _, c := range gc.cSource.All() {
		if c.Display {
			if c.Id == gc.selectedID {
				cursorVisible = true
			}
			gc.filtered = append(gc.filtered, c)
		}
	}

	if oldLen != gc.Len() {
		lenChanged = true
	}

	if !cursorVisible {
		gc.Reset()
	}
	if gc.selectedID == "" {
		gc.Reset()
	}
	return lenChanged
}

// Set an initial cursor position, if possible
func (gc *GridCursor) Reset() {
	for _, c := range gc.cSource.All() {
		c.Widgets.Name.UnHighlight()
	}
	if gc.Len() > 0 {
		gc.selectedID = gc.filtered[0].Id
		gc.filtered[0].Widgets.Name.Highlight()
	}
}

// Return current cursor index
func (gc *GridCursor) Idx() int {
	for n, c := range gc.filtered {
		if c.Id == gc.selectedID {
			return n
		}
	}
	gc.Reset()
	return 0
}

func (gc *GridCursor) ScrollPage() {
	// skip scroll if no need to page
	if gc.Len() < cGrid.MaxRows() {
		cGrid.Offset = 0
		return
	}

	idx := gc.Idx()

	// page down
	if idx >= cGrid.Offset+cGrid.MaxRows() {
		cGrid.Offset++
		cGrid.Align()
	}
	// page up
	if idx < cGrid.Offset {
		cGrid.Offset--
		cGrid.Align()
	}

}

func (gc *GridCursor) Up() {
	gc.isScrolling = true
	defer func() { gc.isScrolling = false }()

	idx := gc.Idx()
	if idx <= 0 { // already at top
		return
	}
	active := gc.filtered[idx]
	next := gc.filtered[idx-1]

	active.Widgets.Name.UnHighlight()
	gc.selectedID = next.Id
	next.Widgets.Name.Highlight()

	gc.ScrollPage()
	ui.Render(cGrid)
}

func (gc *GridCursor) Down() {
	gc.isScrolling = true
	defer func() { gc.isScrolling = false }()

	idx := gc.Idx()
	if idx >= gc.Len()-1 { // already at bottom
		return
	}
	active := gc.filtered[idx]
	next := gc.filtered[idx+1]

	active.Widgets.Name.UnHighlight()
	gc.selectedID = next.Id
	next.Widgets.Name.Highlight()

	gc.ScrollPage()
	ui.Render(cGrid)
}

func (gc *GridCursor) PgUp() {
	idx := gc.Idx()
	if idx <= 0 { // already at top
		return
	}

	nextidx := int(math.Max(0.0, float64(idx-cGrid.MaxRows())))
	cGrid.Offset = int(math.Max(float64(cGrid.Offset-cGrid.MaxRows()),
		float64(0)))

	active := gc.filtered[idx]
	next := gc.filtered[nextidx]

	active.Widgets.Name.UnHighlight()
	gc.selectedID = next.Id
	next.Widgets.Name.Highlight()

	cGrid.Align()
	ui.Render(cGrid)
}

func (gc *GridCursor) PgDown() {
	idx := gc.Idx()
	if idx >= gc.Len()-1 { // already at bottom
		return
	}

	nextidx := int(math.Min(float64(gc.Len()-1),
		float64(idx+cGrid.MaxRows())))
	cGrid.Offset = int(math.Min(float64(cGrid.Offset+cGrid.MaxRows()),
		float64(gc.Len()-cGrid.MaxRows())))

	active := gc.filtered[idx]
	next := gc.filtered[nextidx]

	active.Widgets.Name.UnHighlight()
	gc.selectedID = next.Id
	next.Widgets.Name.Highlight()

	cGrid.Align()
	ui.Render(cGrid)
}

// number of pages at current row count and term height
func (gc *GridCursor) pgCount() int {
	pages := gc.Len() / cGrid.MaxRows()
	if gc.Len()%cGrid.MaxRows() > 0 {
		pages++
	}
	return pages
}
<|endoftext|>


<commit_before>package gorethink

import (
	"encoding/json"
	"errors"
	"reflect"
	"sync"

	"github.com/dancannon/gorethink/encoding"
	p "github.com/dancannon/gorethink/ql2"
)

var (
	errCursorClosed = errors.New("connection closed, cannot read cursor")
)

func newCursor(conn *Connection, token int64, term *Term, opts map[string]interface{}) *Cursor {
	cursor := &Cursor{
		conn:  conn,
		token: token,
		term:  term,
		opts:  opts,
	}

	return cursor
}

// Cursor is the result of a query. Its cursor starts before the first row
// of the result set. Use Next to advance through the rows:
//
//     cursor, err := query.Run(session)
//     ...
//     defer cursor.Close()
//
//     var response interface{}
//     for cursor.Next(&response) {
//         ...
//     }
//     err = cursor.Err() // get any error encountered during iteration
//     ...
type Cursor struct {
	pc          *poolConn
	releaseConn func(error)

	conn  *Connection
	token int64
	query Query
	term  *Term
	opts  map[string]interface{}

	sync.Mutex
	lastErr   error
	fetching  bool
	closed    bool
	finished  bool
	buffer    queue
	responses queue
	profile   interface{}
}

// Profile returns the information returned from the query profiler.
func (c *Cursor) Profile() interface{} {
	c.Lock()
	defer c.Unlock()

	return c.profile
}

// Err returns nil if no errors happened during iteration, or the actual
// error otherwise.
func (c *Cursor) Err() error {
	c.Lock()
	defer c.Unlock()

	return c.lastErr
}

// Close closes the cursor, preventing further enumeration. If the end is
// encountered, the cursor is closed automatically. Close is idempotent.
func (c *Cursor) Close() error {
	c.Lock()
	defer c.Unlock()

	var err error

	if c.closed {
		return nil
	}

	conn := c.conn
	if conn == nil {
		return nil
	}
	if conn.conn == nil {
		return nil
	}

	// Stop any unfinished queries
	if !c.closed && !c.finished {
		q := Query{
			Type:  p.Query_STOP,
			Token: c.token,
		}

		_, _, err = conn.Query(q, map[string]interface{}{})
	}

	c.releaseConn(err)

	c.closed = true
	c.conn = nil

	return err
}

// Next retrieves the next document from the result set, blocking if necessary.
// This method will also automatically retrieve another batch of documents from
// the server when the current one is exhausted, or before that in background
// if possible.
//
// Next returns true if a document was successfully unmarshalled onto result,
// and false at the end of the result set or if an error happened.
// When Next returns false, the Err method should be called to verify if
// there was an error during iteration.
func (c *Cursor) Next(dest interface{}) bool {
	if c.closed {
		return false
	}

	hasMore, err := c.loadNext(dest)
	if c.handleError(err) != nil {
		c.Close()
		return false
	}

	return hasMore
}

func (c *Cursor) loadNext(dest interface{}) (bool, error) {
	c.Lock()

	for c.lastErr == nil {
		// Check if response is closed/finished
		if c.buffer.Len() == 0 && c.responses.Len() == 0 && c.closed {
			c.Unlock()
			return false, errCursorClosed
		}

		if c.buffer.Len() == 0 && c.responses.Len() == 0 && !c.finished {
			c.Unlock()
			err := c.fetchMore()
			if err != nil {
				return false, err
			}
			c.Lock()
		}

		if c.buffer.Len() == 0 && c.responses.Len() == 0 && c.finished {
			c.Unlock()
			return false, nil
		}

		if c.buffer.Len() == 0 && c.responses.Len() > 0 {
			if response, ok := c.responses.Pop().(json.RawMessage); ok {
				c.Unlock()
				var value interface{}
				err := json.Unmarshal(response, &value)
				if err != nil {
					return false, err
				}

				value, err = recursivelyConvertPseudotype(value, c.opts)
				if err != nil {
					return false, err
				}
				c.Lock()

				if data, ok := value.([]interface{}); ok {
					for _, v := range data {
						c.buffer.Push(v)
					}
				} else if value == nil {
					c.buffer.Push(nil)
				} else {
					c.buffer.Push(value)
				}
			}
		}

		if c.buffer.Len() > 0 {
			data := c.buffer.Pop()
			c.Unlock()

			err := encoding.Decode(dest, data)
			if err != nil {
				return false, err
			}

			return true, nil
		}
	}

	c.Unlock()

	return false, c.lastErr
}

// All retrieves all documents from the result set into the provided slice
// and closes the cursor.
//
// The result argument must necessarily be the address for a slice. The slice
// may be nil or previously allocated.
func (c *Cursor) All(result interface{}) error {
	resultv := reflect.ValueOf(result)
	if resultv.Kind() != reflect.Ptr || resultv.Elem().Kind() != reflect.Slice {
		panic("result argument must be a slice address")
	}
	slicev := resultv.Elem()
	slicev = slicev.Slice(0, slicev.Cap())
	elemt := slicev.Type().Elem()
	i := 0
	for {
		if slicev.Len() == i {
			elemp := reflect.New(elemt)
			if !c.Next(elemp.Interface()) {
				break
			}
			slicev = reflect.Append(slicev, elemp.Elem())
			slicev = slicev.Slice(0, slicev.Cap())
		} else {
			if !c.Next(slicev.Index(i).Addr().Interface()) {
				break
			}
		}
		i++
	}
	resultv.Elem().Set(slicev.Slice(0, i))

	if err := c.Err(); err != nil {
		c.Close()
		return err
	}

	if err := c.Close(); err != nil {
		return err
	}

	return nil
}

// One retrieves a single document from the result set into the provided
// slice and closes the cursor.
func (c *Cursor) One(result interface{}) error {
	if c.IsNil() {
		return ErrEmptyResult
	}

	hasResult := c.Next(result)

	if err := c.Err(); err != nil {
		c.Close()
		return err
	}

	if err := c.Close(); err != nil {
		return err
	}

	if !hasResult {
		return ErrEmptyResult
	}

	return nil
}

// IsNil tests if the current row is nil.
func (c *Cursor) IsNil() bool {
	c.Lock()
	defer c.Unlock()
	if c.buffer.Len() > 0 {
		bufferedItem := c.buffer.Peek()
		if bufferedItem == nil {
			return true
		}

		if bufferedItem == nil {
			return true
		}

		return false
	}

	if c.responses.Len() > 0 {
		response := c.responses.Peek()
		if response == nil {
			return true
		}

		if response, ok := response.(json.RawMessage); ok {
			if string(response) == "null" {
				return true
			}
		}

		return false
	}

	return true
}

// fetchMore fetches more rows from the database.
//
// If wait is true then it will wait for the database to reply otherwise it
// will return after sending the continue query.
func (c *Cursor) fetchMore() error {
	c.Lock()
	defer c.Unlock()

	var err error
	if !c.fetching {
		c.fetching = true

		if c.closed {
			return errCursorClosed
		}

		q := Query{
			Type:  p.Query_CONTINUE,
			Token: c.token,
		}
		c.Unlock()
		_, _, err = c.conn.Query(q, map[string]interface{}{
			"noreply": async,
		})
		c.handleError(err)
		c.Lock()
	}

	return err
}

// handleError sets the value of lastErr to err if lastErr is not yet set.
func (c *Cursor) handleError(err error) error {
	c.Lock()
	defer c.Unlock()

	return c.handleErrorLocked(err)
}

// handleError sets the value of lastErr to err if lastErr is not yet set.
func (c *Cursor) handleErrorLocked(err error) error {
	if c.lastErr == nil {
		c.lastErr = err
	}

	return c.lastErr
}

// extend adds the result of a continue query to the cursor.
func (c *Cursor) extend(response *Response) {
	c.Lock()
	defer c.Unlock()

	for _, response := range response.Responses {
		c.responses.Push(response)
	}

	c.finished = response.Type != p.Response_SUCCESS_PARTIAL && response.Type != p.Response_SUCCESS_FEED
	c.fetching = false
}

// Queue structure used for storing responses

type queue struct {
	elems               []interface{}
	nelems, popi, pushi int
}

func (q *queue) Len() int {
	return q.nelems
}
func (q *queue) Push(elem interface{}) {
	if q.nelems == len(q.elems) {
		q.expand()
	}
	q.elems[q.pushi] = elem
	q.nelems++
	q.pushi = (q.pushi + 1) % len(q.elems)
}
func (q *queue) Pop() (elem interface{}) {
	if q.nelems == 0 {
		return nil
	}
	elem = q.elems[q.popi]
	q.elems[q.popi] = nil // Help GC.
	q.nelems--
	q.popi = (q.popi + 1) % len(q.elems)
	return elem
}
func (q *queue) Peek() (elem interface{}) {
	if q.nelems == 0 {
		return nil
	}
	return q.elems[q.popi]
}
func (q *queue) expand() {
	curcap := len(q.elems)
	var newcap int
	if curcap == 0 {
		newcap = 8
	} else if curcap < 1024 {
		newcap = curcap * 2
	} else {
		newcap = curcap + (curcap / 4)
	}
	elems := make([]interface{}, newcap)
	if q.popi == 0 {
		copy(elems, q.elems)
		q.pushi = curcap
	} else {
		newpopi := newcap - (curcap - q.popi)
		copy(elems, q.elems[:q.popi])
		copy(elems[newpopi:], q.elems[q.popi:])
		q.popi = newpopi
	}
	for i := range q.elems {
		q.elems[i] = nil // Help GC.
	}
	q.elems = elems
}
<commit_msg>Fixed build<commit_after>package gorethink

import (
	"encoding/json"
	"errors"
	"reflect"
	"sync"

	"github.com/dancannon/gorethink/encoding"
	p "github.com/dancannon/gorethink/ql2"
)

var (
	errCursorClosed = errors.New("connection closed, cannot read cursor")
)

func newCursor(conn *Connection, token int64, term *Term, opts map[string]interface{}) *Cursor {
	cursor := &Cursor{
		conn:  conn,
		token: token,
		term:  term,
		opts:  opts,
	}

	return cursor
}

// Cursor is the result of a query. Its cursor starts before the first row
// of the result set. Use Next to advance through the rows:
//
//     cursor, err := query.Run(session)
//     ...
//     defer cursor.Close()
//
//     var response interface{}
//     for cursor.Next(&response) {
//         ...
//     }
//     err = cursor.Err() // get any error encountered during iteration
//     ...
type Cursor struct {
	pc          *poolConn
	releaseConn func(error)

	conn  *Connection
	token int64
	query Query
	term  *Term
	opts  map[string]interface{}

	sync.Mutex
	lastErr   error
	fetching  bool
	closed    bool
	finished  bool
	buffer    queue
	responses queue
	profile   interface{}
}

// Profile returns the information returned from the query profiler.
func (c *Cursor) Profile() interface{} {
	c.Lock()
	defer c.Unlock()

	return c.profile
}

// Err returns nil if no errors happened during iteration, or the actual
// error otherwise.
func (c *Cursor) Err() error {
	c.Lock()
	defer c.Unlock()

	return c.lastErr
}

// Close closes the cursor, preventing further enumeration. If the end is
// encountered, the cursor is closed automatically. Close is idempotent.
func (c *Cursor) Close() error {
	c.Lock()
	defer c.Unlock()

	var err error

	if c.closed {
		return nil
	}

	conn := c.conn
	if conn == nil {
		return nil
	}
	if conn.conn == nil {
		return nil
	}

	// Stop any unfinished queries
	if !c.closed && !c.finished {
		q := Query{
			Type:  p.Query_STOP,
			Token: c.token,
		}

		_, _, err = conn.Query(q, map[string]interface{}{})
	}

	c.releaseConn(err)

	c.closed = true
	c.conn = nil

	return err
}

// Next retrieves the next document from the result set, blocking if necessary.
// This method will also automatically retrieve another batch of documents from
// the server when the current one is exhausted, or before that in background
// if possible.
//
// Next returns true if a document was successfully unmarshalled onto result,
// and false at the end of the result set or if an error happened.
// When Next returns false, the Err method should be called to verify if
// there was an error during iteration.
func (c *Cursor) Next(dest interface{}) bool {
	if c.closed {
		return false
	}

	hasMore, err := c.loadNext(dest)
	if c.handleError(err) != nil {
		c.Close()
		return false
	}

	return hasMore
}

func (c *Cursor) loadNext(dest interface{}) (bool, error) {
	c.Lock()

	for c.lastErr == nil {
		// Check if response is closed/finished
		if c.buffer.Len() == 0 && c.responses.Len() == 0 && c.closed {
			c.Unlock()
			return false, errCursorClosed
		}

		if c.buffer.Len() == 0 && c.responses.Len() == 0 && !c.finished {
			c.Unlock()
			err := c.fetchMore()
			if err != nil {
				return false, err
			}
			c.Lock()
		}

		if c.buffer.Len() == 0 && c.responses.Len() == 0 && c.finished {
			c.Unlock()
			return false, nil
		}

		if c.buffer.Len() == 0 && c.responses.Len() > 0 {
			if response, ok := c.responses.Pop().(json.RawMessage); ok {
				c.Unlock()
				var value interface{}
				err := json.Unmarshal(response, &value)
				if err != nil {
					return false, err
				}

				value, err = recursivelyConvertPseudotype(value, c.opts)
				if err != nil {
					return false, err
				}
				c.Lock()

				if data, ok := value.([]interface{}); ok {
					for _, v := range data {
						c.buffer.Push(v)
					}
				} else if value == nil {
					c.buffer.Push(nil)
				} else {
					c.buffer.Push(value)
				}
			}
		}

		if c.buffer.Len() > 0 {
			data := c.buffer.Pop()
			c.Unlock()

			err := encoding.Decode(dest, data)
			if err != nil {
				return false, err
			}

			return true, nil
		}
	}

	c.Unlock()

	return false, c.lastErr
}

// All retrieves all documents from the result set into the provided slice
// and closes the cursor.
//
// The result argument must necessarily be the address for a slice. The slice
// may be nil or previously allocated.
func (c *Cursor) All(result interface{}) error {
	resultv := reflect.ValueOf(result)
	if resultv.Kind() != reflect.Ptr || resultv.Elem().Kind() != reflect.Slice {
		panic("result argument must be a slice address")
	}
	slicev := resultv.Elem()
	slicev = slicev.Slice(0, slicev.Cap())
	elemt := slicev.Type().Elem()
	i := 0
	for {
		if slicev.Len() == i {
			elemp := reflect.New(elemt)
			if !c.Next(elemp.Interface()) {
				break
			}
			slicev = reflect.Append(slicev, elemp.Elem())
			slicev = slicev.Slice(0, slicev.Cap())
		} else {
			if !c.Next(slicev.Index(i).Addr().Interface()) {
				break
			}
		}
		i++
	}
	resultv.Elem().Set(slicev.Slice(0, i))

	if err := c.Err(); err != nil {
		c.Close()
		return err
	}

	if err := c.Close(); err != nil {
		return err
	}

	return nil
}

// One retrieves a single document from the result set into the provided
// slice and closes the cursor.
func (c *Cursor) One(result interface{}) error {
	if c.IsNil() {
		return ErrEmptyResult
	}

	hasResult := c.Next(result)

	if err := c.Err(); err != nil {
		c.Close()
		return err
	}

	if err := c.Close(); err != nil {
		return err
	}

	if !hasResult {
		return ErrEmptyResult
	}

	return nil
}

// IsNil tests if the current row is nil.
func (c *Cursor) IsNil() bool {
	c.Lock()
	defer c.Unlock()
	if c.buffer.Len() > 0 {
		bufferedItem := c.buffer.Peek()
		if bufferedItem == nil {
			return true
		}

		if bufferedItem == nil {
			return true
		}

		return false
	}

	if c.responses.Len() > 0 {
		response := c.responses.Peek()
		if response == nil {
			return true
		}

		if response, ok := response.(json.RawMessage); ok {
			if string(response) == "null" {
				return true
			}
		}

		return false
	}

	return true
}

// fetchMore fetches more rows from the database.
//
// If wait is true then it will wait for the database to reply otherwise it
// will return after sending the continue query.
func (c *Cursor) fetchMore() error {
	c.Lock()
	defer c.Unlock()

	var err error
	if !c.fetching {
		c.fetching = true

		if c.closed {
			return errCursorClosed
		}

		q := Query{
			Type:  p.Query_CONTINUE,
			Token: c.token,
		}
		c.Unlock()
		_, _, err = c.conn.Query(q, map[string]interface{}{})
		c.handleError(err)
		c.Lock()
	}

	return err
}

// handleError sets the value of lastErr to err if lastErr is not yet set.
func (c *Cursor) handleError(err error) error {
	c.Lock()
	defer c.Unlock()

	return c.handleErrorLocked(err)
}

// handleError sets the value of lastErr to err if lastErr is not yet set.
func (c *Cursor) handleErrorLocked(err error) error {
	if c.lastErr == nil {
		c.lastErr = err
	}

	return c.lastErr
}

// extend adds the result of a continue query to the cursor.
func (c *Cursor) extend(response *Response) {
	c.Lock()
	defer c.Unlock()

	for _, response := range response.Responses {
		c.responses.Push(response)
	}

	c.finished = response.Type != p.Response_SUCCESS_PARTIAL && response.Type != p.Response_SUCCESS_FEED
	c.fetching = false
}

// Queue structure used for storing responses

type queue struct {
	elems               []interface{}
	nelems, popi, pushi int
}

func (q *queue) Len() int {
	return q.nelems
}
func (q *queue) Push(elem interface{}) {
	if q.nelems == len(q.elems) {
		q.expand()
	}
	q.elems[q.pushi] = elem
	q.nelems++
	q.pushi = (q.pushi + 1) % len(q.elems)
}
func (q *queue) Pop() (elem interface{}) {
	if q.nelems == 0 {
		return nil
	}
	elem = q.elems[q.popi]
	q.elems[q.popi] = nil // Help GC.
	q.nelems--
	q.popi = (q.popi + 1) % len(q.elems)
	return elem
}
func (q *queue) Peek() (elem interface{}) {
	if q.nelems == 0 {
		return nil
	}
	return q.elems[q.popi]
}
func (q *queue) expand() {
	curcap := len(q.elems)
	var newcap int
	if curcap == 0 {
		newcap = 8
	} else if curcap < 1024 {
		newcap = curcap * 2
	} else {
		newcap = curcap + (curcap / 4)
	}
	elems := make([]interface{}, newcap)
	if q.popi == 0 {
		copy(elems, q.elems)
		q.pushi = curcap
	} else {
		newpopi := newcap - (curcap - q.popi)
		copy(elems, q.elems[:q.popi])
		copy(elems[newpopi:], q.elems[q.popi:])
		q.popi = newpopi
	}
	for i := range q.elems {
		q.elems[i] = nil // Help GC.
	}
	q.elems = elems
}
<|endoftext|>


<commit_before>package mailout

import (
	"crypto/tls"
	"net/http"
	"time"

	"gopkg.in/gomail.v2"
)

func startMailDaemon(mc *config) chan<- *http.Request {
	rChan := make(chan *http.Request)
	// this can be a bottleneck under high load because the channel is unbuffered.
	// maybe we can add a pool of sendmail workers.
	go goMailDaemonRecoverable(mc, rChan)
	return rChan
}

// goMailDaemonRecoverable self restarting goroutine.
// TODO(cs) limit restarting to e.g. 10 tries and then crash it.
func goMailDaemonRecoverable(mc *config, rChan <-chan *http.Request) {
	defer func() {
		if r := recover(); r != nil {
			mc.maillog.Errorf("[mailout] Catching panic %#v and restarting daemon ...", r)
			go goMailDaemonRecoverable(mc, rChan)
		}
	}()
	goMailDaemon(mc, rChan)
}

func goMailDaemon(mc *config, rChan <-chan *http.Request) {
	d := gomail.NewPlainDialer(mc.host, mc.port, mc.username, mc.password)
	d.TLSConfig = &tls.Config{ServerName: mc.host, InsecureSkipVerify: mc.skipTlsVerify}

	var s gomail.SendCloser
	var err error
	open := false
	for {
		select {
		case r, ok := <-rChan:
			if !ok {
				return
			}

			mails := newMessage(mc, r).build()
			// multiple mails will increase the rate limit at some MTAs.
			// so the REST API rate limit must be: rate / pgpEmailAddresses

			if !open {
				if s, err = d.Dial(); err != nil {
					mc.maillog.Errorf("Dial Error: %s", err)

					wc := mc.maillog.NewWriter()
					if _, errW := mails.WriteTo(wc); errW != nil {
						mc.maillog.Errorf("Dial: Message WriteTo Log Error: %s", errW)
					}
					if errC := wc.Close(); errC != nil {
						mc.maillog.Errorf("Dial wc.Close Error: %s", errC)
					}

					continue
				}
				open = true
			}

			wc := mc.maillog.NewWriter()
			if _, err2 := mails.WriteTo(wc); err2 != nil {
				mc.maillog.Errorf("Send: Message WriteTo Log Error: %s", err)
			}
			if err = wc.Close(); err != nil {
				mc.maillog.Errorf("Send wc.Close Error: %s", err)
			}

			if err := gomail.Send(s, mails...); err != nil {
				mc.maillog.Errorf("Send Error: %s", err)
			}

		// Close the connection to the SMTP server if no email was sent in
		// the last 30 seconds.
		case <-time.After(30 * time.Second):
			if open {
				if err := s.Close(); err != nil {
					mc.maillog.Errorf("Dial Close Error: %s", err)
				}
				open = false
			}
			//default: // http://www.jtolds.com/writing/2016/03/go-channels-are-bad-and-you-should-feel-bad/
		}
	}
}
<commit_msg>Fix: check hostname upon setting.<commit_after>package mailout

import (
	"crypto/tls"
	"net/http"
	"time"

	"gopkg.in/gomail.v2"
)

func startMailDaemon(mc *config) chan<- *http.Request {
	rChan := make(chan *http.Request)
	// this can be a bottleneck under high load because the channel is unbuffered.
	// maybe we can add a pool of sendmail workers.
	go goMailDaemonRecoverable(mc, rChan)
	return rChan
}

// goMailDaemonRecoverable self restarting goroutine.
// TODO(cs) limit restarting to e.g. 10 tries and then crash it.
func goMailDaemonRecoverable(mc *config, rChan <-chan *http.Request) {
	defer func() {
		if r := recover(); r != nil {
			mc.maillog.Errorf("[mailout] Catching panic %#v and restarting daemon ...", r)
			go goMailDaemonRecoverable(mc, rChan)
		}
	}()
	goMailDaemon(mc, rChan)
}

func goMailDaemon(mc *config, rChan <-chan *http.Request) {
	d := gomail.NewPlainDialer(mc.host, mc.port, mc.username, mc.password)
	if mc.port == 587 {
		d.TLSConfig = &tls.Config{
			ServerName: mc.host, // host names must match between this one and the one requested in the cert.
		}
	}
	if mc.skipTlsVerify {
		if d.TLSConfig == nil {
			d.TLSConfig = &tls.Config{}
		}
		d.TLSConfig.InsecureSkipVerify = true
	}

	var s gomail.SendCloser
	var err error
	open := false
	for {
		select {
		case r, ok := <-rChan:
			if !ok {
				return
			}

			mails := newMessage(mc, r).build()
			// multiple mails will increase the rate limit at some MTAs.
			// so the REST API rate limit must be: rate / pgpEmailAddresses

			if !open {
				if s, err = d.Dial(); err != nil {
					mc.maillog.Errorf("Dial Error: %s", err)

					wc := mc.maillog.NewWriter()
					if _, errW := mails.WriteTo(wc); errW != nil {
						mc.maillog.Errorf("Dial: Message WriteTo Log Error: %s", errW)
					}
					if errC := wc.Close(); errC != nil {
						mc.maillog.Errorf("Dial wc.Close Error: %s", errC)
					}

					continue
				}
				open = true
			}

			wc := mc.maillog.NewWriter()
			if _, err2 := mails.WriteTo(wc); err2 != nil {
				mc.maillog.Errorf("Send: Message WriteTo Log Error: %s", err)
			}
			if err = wc.Close(); err != nil {
				mc.maillog.Errorf("Send wc.Close Error: %s", err)
			}

			if err := gomail.Send(s, mails...); err != nil {
				mc.maillog.Errorf("Send Error: %s", err)
			}

		// Close the connection to the SMTP server if no email was sent in
		// the last 30 seconds.
		case <-time.After(30 * time.Second):
			if open {
				if err := s.Close(); err != nil {
					mc.maillog.Errorf("Dial Close Error: %s", err)
				}
				open = false
			}
			//default: // http://www.jtolds.com/writing/2016/03/go-channels-are-bad-and-you-should-feel-bad/
		}
	}
}
<|endoftext|>


<commit_before>/*
CompileDaemon is a very simple compile daemon for Go.

CompileDaemon watches your .go files in a directory and invokes `go build`
if a file changes.

Examples

In its simplest form, the defaults will do. With the current working directory set
to the source directory you can simply…

    $ CompileDaemon

… and it will recompile your code whenever you save a source file.

If you want it to also run your program each time it builds you might add…

    $ CompileDaemon -command="./MyProgram -my-options"

… and it will also keep a copy of your program running. Killing the old one and
starting a new one each time you build.

You may find that you need to exclude some directories and files from
monitoring, such as a .git repository or emacs temporary files…

    $ CompileDaemon -exclude-dir=.git -exclude=".#*"

If you want to monitor files other than .go and .c files you might…

    $ CompileDaemon -include=Makefile -include="*.less" -include="*.tmpl"

Options

There are command line options.

	FILE SELECTION
	-directory=XXX    – which directory to monitor for changes
	-recursive=XXX    – look into subdirectories
	-exclude-dir=XXX  – exclude directories matching glob pattern XXX
	-exlude=XXX       – exclude files whose basename matches glob pattern XXX
	-include=XXX      – include files whose basename matches glob pattern XXX
	-pattern=XXX      – include files whose path matches regexp XXX

	MISC
	-color            - enable colorized output
	-log-prefix       - Enable/disable stdout/stderr labelling for the child process
	-graceful-kill    - On supported platforms, send the child process a SIGTERM to
	                    allow it to exit gracefully if possible.
	ACTIONS
	-build=CCC        – Execute CCC to rebuild when a file changes
	-command=CCC      – Run command CCC after a successful build, stops previous command first

*/
package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/fatih/color"
	"github.com/howeyc/fsnotify"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"syscall"
	"time"
)

// Milliseconds to wait for the next job to begin after a file change
const WorkDelay = 900

// Default pattern to match files which trigger a build
const FilePattern = `(.+\.go|.+\.c)$`

type globList []string

func (g *globList) String() string {
	return fmt.Sprint(*g)
}
func (g *globList) Set(value string) error {
	*g = append(*g, value)
	return nil
}
func (g *globList) Matches(value string) bool {
	for _, v := range *g {
		if match, err := filepath.Match(v, value); err != nil {
			log.Fatalf("Bad pattern \"%s\": %s", v, err.Error())
		} else if match {
			return true
		}
	}
	return false
}

var (
	flag_directory    = flag.String("directory", ".", "Directory to watch for changes")
	flag_pattern      = flag.String("pattern", FilePattern, "Pattern of watched files")
	flag_command      = flag.String("command", "", "Command to run and restart after build")
	flag_command_stop  = flag.Bool("command-stop", false, "Stop command before building")
	flag_recursive    = flag.Bool("recursive", true, "Watch all dirs. recursively")
	flag_build        = flag.String("build", "go build", "Command to rebuild after changes")
	flag_color        = flag.Bool("color", false, "Colorize output for CompileDaemon status messages")
	flag_logprefix    = flag.Bool("log-prefix", true, "Print log timestamps and subprocess stderr/stdout output")
	flag_gracefulkill = flag.Bool("graceful-kill", false, "Gracefully attempt to kill the child process by sending a SIGTERM first")

	// initialized in main() due to custom type.
	flag_excludedDirs globList
	flag_excludedFiles globList
	flag_includedFiles globList
)

func okColor(format string, args ...interface{}) string {
	if *flag_color {
		return color.GreenString(format, args...)
	} else {
		return fmt.Sprintf(format, args...)
	}
}

func failColor(format string, args ...interface{}) string {
	if *flag_color {
		return color.RedString(format, args...)
	} else {
		return fmt.Sprintf(format, args...)
	}
}

// Run `go build` and print the output if something's gone wrong.
func build() bool {
	log.Println(okColor("Running build command!"))

	args := strings.Split(*flag_build, " ")
	if len(args) == 0 {
		// If the user has specified and empty then we are done.
		return true
	}

	cmd := exec.Command(args[0], args[1:]...)

	cmd.Dir = *flag_directory

	output, err := cmd.CombinedOutput()

	if err == nil {
		log.Println(okColor("Build ok."))
	} else {
		log.Println(failColor("Error while building:\n"), failColor(string(output)))
	}

	return err == nil
}

func matchesPattern(pattern *regexp.Regexp, file string) bool {
	return pattern.MatchString(file)
}

// Accept build jobs and start building when there are no jobs rushing in.
// The inrush protection is WorkDelay milliseconds long, in this period
// every incoming job will reset the timer.
func builder(jobs <-chan string, buildDone chan<- struct{}) {
	createThreshold := func() <-chan time.Time {
		return time.After(time.Duration(WorkDelay * time.Millisecond))
	}

	threshold := createThreshold()

	for {
		select {
		case <-jobs:
			threshold = createThreshold()
		case <-threshold:
			if *flag_command_stop {
				buildDone <- struct{}{}
			}
			if build() {
				buildDone <- struct{}{}
			}
		}
	}
}

func logger(pipeChan <-chan io.ReadCloser) {
	dumper := func(pipe io.ReadCloser, prefix string) {
		reader := bufio.NewReader(pipe)

	readloop:
		for {
			line, err := reader.ReadString('\n')

			if err != nil {
				break readloop
			}

			if *flag_logprefix {
				log.Print(prefix, " ", line)
			} else {
				log.Print(line)
			}
		}
	}

	for {
		pipe := <-pipeChan
		go dumper(pipe, "stdout:")

		pipe = <-pipeChan
		go dumper(pipe, "stderr:")
	}
}

// Start the supplied command and return stdout and stderr pipes for logging.
func startCommand(command string) (cmd *exec.Cmd, stdout io.ReadCloser, stderr io.ReadCloser, err error) {
	args := strings.Split(command, " ")
	cmd = exec.Command(args[0], args[1:]...)

	if stdout, err = cmd.StdoutPipe(); err != nil {
		err = fmt.Errorf("can't get stdout pipe for command: %s", err)
		return
	}

	if stderr, err = cmd.StderrPipe(); err != nil {
		err = fmt.Errorf("can't get stderr pipe for command: %s", err)
		return
	}

	if err = cmd.Start(); err != nil {
		err = fmt.Errorf("can't start command: %s", err)
		return
	}

	return
}

// Run the command in the given string and restart it after
// a message was received on the buildDone channel.
func runner(command string, buildDone <-chan struct{}) {
	var currentProcess *os.Process
	pipeChan := make(chan io.ReadCloser)

	go logger(pipeChan)

	for {
		<-buildDone

		if currentProcess != nil {
			killProcess(currentProcess)
		}
		if *flag_command_stop {
			log.Println(okColor("Command stopped. Waiting for build to complete."))
			<-buildDone
		}

		log.Println(okColor("Restarting the given command."))
		cmd, stdoutPipe, stderrPipe, err := startCommand(command)

		if err != nil {
			log.Fatal(failColor("Could not start command:", err))
		}

		pipeChan <- stdoutPipe
		pipeChan <- stderrPipe

		currentProcess = cmd.Process
	}
}

func killProcess(process *os.Process) {
	if *flag_gracefulkill {
		killProcessGracefully(process)
	} else {
		killProcessHard(process)
	}
}

func killProcessHard(process *os.Process) {
	log.Println(okColor("Hard stopping the current process.."))

	if err := process.Kill(); err != nil {
		log.Fatal(failColor("Could not kill child process. Aborting due to danger of infinite forks."))
	}

	if _, err := process.Wait(); err != nil {
		log.Fatal(failColor("Could not wait for child process. Aborting due to danger of infinite forks."))
	}
}

func killProcessGracefully(process *os.Process) {
	done := make(chan error, 1)
	go func() {
		log.Println(okColor("Gracefully stopping the current process.."))
		if err := terminateGracefully(process); err != nil {
			done <- err
			return
		}
		_, err := process.Wait()
		done <- err
	}()

	select {
	case <-time.After(3 * time.Second):
		log.Println(failColor("Could not gracefully stop the current process, proceeding to hard stop."))
		killProcessHard(process)
		<-done
	case err := <-done:
		if err != nil {
			log.Fatal(failColor("Could not kill child process. Aborting due to danger of infinite forks."))
		}
	}
}

func flusher(buildDone <-chan struct{}) {
	for {
		if *flag_command_stop {
			<-buildDone
		}
		<-buildDone
	}
}

func main() {
	flag.Var(&flag_excludedDirs, "exclude-dir", " Don't watch directories matching this name")
	flag.Var(&flag_excludedFiles, "exclude", " Don't watch files matching this name")
	flag.Var(&flag_includedFiles, "include", " Watch files matching this name")

	flag.Parse()

	if !*flag_logprefix {
		log.SetFlags(0)
	}

	if *flag_directory == "" {
		fmt.Fprintf(os.Stderr, "-directory=... is required.\n")
		os.Exit(1)
	}

	if *flag_gracefulkill && !gracefulTerminationPossible() {
		log.Fatal("Graceful termination is not supported on your platform.")
	}

	watcher, err := fsnotify.NewWatcher()

	if err != nil {
		log.Fatal(err)
	}

	defer watcher.Close()

	if *flag_recursive == true {
		err = filepath.Walk(*flag_directory, func(path string, info os.FileInfo, err error) error {
			if err == nil && info.IsDir() {
				if flag_excludedDirs.Matches(info.Name()) {
					return filepath.SkipDir
				} else {
					return watcher.Watch(path)
				}
			}
			return err
		})

		if err != nil {
			log.Fatal("filepath.Walk():", err)
		}

	} else {
		if err := watcher.Watch(*flag_directory); err != nil {
			log.Fatal("watcher.Watch():", err)
		}
	}

	pattern := regexp.MustCompile(*flag_pattern)
	jobs := make(chan string)
	buildDone := make(chan struct{})

	go builder(jobs, buildDone)

	if *flag_command != "" {
		go runner(*flag_command, buildDone)
	} else {
		go flusher(buildDone)
	}

	for {
		select {
		case ev := <-watcher.Event:
			if ev.Name != "" {
				base := filepath.Base(ev.Name)

				if flag_includedFiles.Matches(base) || matchesPattern(pattern, ev.Name) {
					if !flag_excludedFiles.Matches(base) {
						jobs <- ev.Name
					}
				}
			}

		case err := <-watcher.Error:
			if v, ok := err.(*os.SyscallError); ok {
				if v.Err == syscall.EINTR {
					continue
				}
				log.Fatal("watcher.Error: SyscallError:", v)
			}
			log.Fatal("watcher.Error:", err)
		}
	}
}
<commit_msg>Introduced buildStarted channel to react when build is about to start to be able to stop process<commit_after>/*
CompileDaemon is a very simple compile daemon for Go.

CompileDaemon watches your .go files in a directory and invokes `go build`
if a file changes.

Examples

In its simplest form, the defaults will do. With the current working directory set
to the source directory you can simply…

    $ CompileDaemon

… and it will recompile your code whenever you save a source file.

If you want it to also run your program each time it builds you might add…

    $ CompileDaemon -command="./MyProgram -my-options"

… and it will also keep a copy of your program running. Killing the old one and
starting a new one each time you build.

You may find that you need to exclude some directories and files from
monitoring, such as a .git repository or emacs temporary files…

    $ CompileDaemon -exclude-dir=.git -exclude=".#*"

If you want to monitor files other than .go and .c files you might…

    $ CompileDaemon -include=Makefile -include="*.less" -include="*.tmpl"

Options

There are command line options.

	FILE SELECTION
	-directory=XXX    – which directory to monitor for changes
	-recursive=XXX    – look into subdirectories
	-exclude-dir=XXX  – exclude directories matching glob pattern XXX
	-exlude=XXX       – exclude files whose basename matches glob pattern XXX
	-include=XXX      – include files whose basename matches glob pattern XXX
	-pattern=XXX      – include files whose path matches regexp XXX

	MISC
	-color            - enable colorized output
	-log-prefix       - Enable/disable stdout/stderr labelling for the child process
	-graceful-kill    - On supported platforms, send the child process a SIGTERM to
	                    allow it to exit gracefully if possible.
	ACTIONS
	-build=CCC        – Execute CCC to rebuild when a file changes
	-command=CCC      – Run command CCC after a successful build, stops previous command first

*/
package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/fatih/color"
	"github.com/howeyc/fsnotify"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"syscall"
	"time"
)

// Milliseconds to wait for the next job to begin after a file change
const WorkDelay = 900

// Default pattern to match files which trigger a build
const FilePattern = `(.+\.go|.+\.c)$`

type globList []string

func (g *globList) String() string {
	return fmt.Sprint(*g)
}
func (g *globList) Set(value string) error {
	*g = append(*g, value)
	return nil
}
func (g *globList) Matches(value string) bool {
	for _, v := range *g {
		if match, err := filepath.Match(v, value); err != nil {
			log.Fatalf("Bad pattern \"%s\": %s", v, err.Error())
		} else if match {
			return true
		}
	}
	return false
}

var (
	flag_directory    = flag.String("directory", ".", "Directory to watch for changes")
	flag_pattern      = flag.String("pattern", FilePattern, "Pattern of watched files")
	flag_command      = flag.String("command", "", "Command to run and restart after build")
	flag_command_stop  = flag.Bool("command-stop", false, "Stop command before building")
	flag_recursive    = flag.Bool("recursive", true, "Watch all dirs. recursively")
	flag_build        = flag.String("build", "go build", "Command to rebuild after changes")
	flag_color        = flag.Bool("color", false, "Colorize output for CompileDaemon status messages")
	flag_logprefix    = flag.Bool("log-prefix", true, "Print log timestamps and subprocess stderr/stdout output")
	flag_gracefulkill = flag.Bool("graceful-kill", false, "Gracefully attempt to kill the child process by sending a SIGTERM first")

	// initialized in main() due to custom type.
	flag_excludedDirs globList
	flag_excludedFiles globList
	flag_includedFiles globList
)

func okColor(format string, args ...interface{}) string {
	if *flag_color {
		return color.GreenString(format, args...)
	} else {
		return fmt.Sprintf(format, args...)
	}
}

func failColor(format string, args ...interface{}) string {
	if *flag_color {
		return color.RedString(format, args...)
	} else {
		return fmt.Sprintf(format, args...)
	}
}

// Run `go build` and print the output if something's gone wrong.
func build() bool {
	log.Println(okColor("Running build command!"))

	args := strings.Split(*flag_build, " ")
	if len(args) == 0 {
		// If the user has specified and empty then we are done.
		return true
	}

	cmd := exec.Command(args[0], args[1:]...)

	cmd.Dir = *flag_directory

	output, err := cmd.CombinedOutput()

	if err == nil {
		log.Println(okColor("Build ok."))
	} else {
		log.Println(failColor("Error while building:\n"), failColor(string(output)))
	}

	return err == nil
}

func matchesPattern(pattern *regexp.Regexp, file string) bool {
	return pattern.MatchString(file)
}

// Accept build jobs and start building when there are no jobs rushing in.
// The inrush protection is WorkDelay milliseconds long, in this period
// every incoming job will reset the timer.
func builder(jobs <-chan string,buildStarted chan<- struct{}, buildDone chan<- struct{}) {
	createThreshold := func() <-chan time.Time {
		return time.After(time.Duration(WorkDelay * time.Millisecond))
	}

	threshold := createThreshold()

	for {
		select {
		case <-jobs:
			threshold = createThreshold()
		case <-threshold:
			buildStarted <- struct{}{}

			if build() {
				buildDone <- struct{}{}
			}
		}
	}
}

func logger(pipeChan <-chan io.ReadCloser) {
	dumper := func(pipe io.ReadCloser, prefix string) {
		reader := bufio.NewReader(pipe)

	readloop:
		for {
			line, err := reader.ReadString('\n')

			if err != nil {
				break readloop
			}

			if *flag_logprefix {
				log.Print(prefix, " ", line)
			} else {
				log.Print(line)
			}
		}
	}

	for {
		pipe := <-pipeChan
		go dumper(pipe, "stdout:")

		pipe = <-pipeChan
		go dumper(pipe, "stderr:")
	}
}

// Start the supplied command and return stdout and stderr pipes for logging.
func startCommand(command string) (cmd *exec.Cmd, stdout io.ReadCloser, stderr io.ReadCloser, err error) {
	args := strings.Split(command, " ")
	cmd = exec.Command(args[0], args[1:]...)

	if stdout, err = cmd.StdoutPipe(); err != nil {
		err = fmt.Errorf("can't get stdout pipe for command: %s", err)
		return
	}

	if stderr, err = cmd.StderrPipe(); err != nil {
		err = fmt.Errorf("can't get stderr pipe for command: %s", err)
		return
	}

	if err = cmd.Start(); err != nil {
		err = fmt.Errorf("can't start command: %s", err)
		return
	}

	return
}

// Run the command in the given string and restart it after
// a message was received on the buildDone channel.
func runner(command string,buildStarted <-chan struct{}, buildDone <-chan struct{}) {
	var currentProcess *os.Process
	pipeChan := make(chan io.ReadCloser)

	go logger(pipeChan)

	for {
		<-buildStarted
		if !*flag_command_stop {
			<-buildDone
		}

		if currentProcess != nil {
			killProcess(currentProcess)
		}
		if *flag_command_stop {
			log.Println(okColor("Command stopped. Waiting for build to complete."))
			<-buildDone
		}

		log.Println(okColor("Restarting the given command."))
		cmd, stdoutPipe, stderrPipe, err := startCommand(command)

		if err != nil {
			log.Fatal(failColor("Could not start command:", err))
		}

		pipeChan <- stdoutPipe
		pipeChan <- stderrPipe

		currentProcess = cmd.Process
	}
}

func killProcess(process *os.Process) {
	if *flag_gracefulkill {
		killProcessGracefully(process)
	} else {
		killProcessHard(process)
	}
}

func killProcessHard(process *os.Process) {
	log.Println(okColor("Hard stopping the current process.."))

	if err := process.Kill(); err != nil {
		log.Fatal(failColor("Could not kill child process. Aborting due to danger of infinite forks."))
	}

	if _, err := process.Wait(); err != nil {
		log.Fatal(failColor("Could not wait for child process. Aborting due to danger of infinite forks."))
	}
}

func killProcessGracefully(process *os.Process) {
	done := make(chan error, 1)
	go func() {
		log.Println(okColor("Gracefully stopping the current process.."))
		if err := terminateGracefully(process); err != nil {
			done <- err
			return
		}
		_, err := process.Wait()
		done <- err
	}()

	select {
	case <-time.After(3 * time.Second):
		log.Println(failColor("Could not gracefully stop the current process, proceeding to hard stop."))
		killProcessHard(process)
		<-done
	case err := <-done:
		if err != nil {
			log.Fatal(failColor("Could not kill child process. Aborting due to danger of infinite forks."))
		}
	}
}

func flusher(buildStarted <-chan struct{},buildDone <-chan struct{}) {
	for {
		<-buildStarted
		<-buildDone
	}
}

func main() {
	flag.Var(&flag_excludedDirs, "exclude-dir", " Don't watch directories matching this name")
	flag.Var(&flag_excludedFiles, "exclude", " Don't watch files matching this name")
	flag.Var(&flag_includedFiles, "include", " Watch files matching this name")

	flag.Parse()

	if !*flag_logprefix {
		log.SetFlags(0)
	}

	if *flag_directory == "" {
		fmt.Fprintf(os.Stderr, "-directory=... is required.\n")
		os.Exit(1)
	}

	if *flag_gracefulkill && !gracefulTerminationPossible() {
		log.Fatal("Graceful termination is not supported on your platform.")
	}

	watcher, err := fsnotify.NewWatcher()

	if err != nil {
		log.Fatal(err)
	}

	defer watcher.Close()

	if *flag_recursive == true {
		err = filepath.Walk(*flag_directory, func(path string, info os.FileInfo, err error) error {
			if err == nil && info.IsDir() {
				if flag_excludedDirs.Matches(info.Name()) {
					return filepath.SkipDir
				} else {
					return watcher.Watch(path)
				}
			}
			return err
		})

		if err != nil {
			log.Fatal("filepath.Walk():", err)
		}

	} else {
		if err := watcher.Watch(*flag_directory); err != nil {
			log.Fatal("watcher.Watch():", err)
		}
	}

	pattern := regexp.MustCompile(*flag_pattern)
	jobs := make(chan string)
	buildDone := make(chan struct{})
	buildStarted := make(chan struct{})

	go builder(jobs, buildStarted,buildDone)

	if *flag_command != "" {
		go runner(*flag_command, buildStarted,buildDone)
	} else {
		go flusher(buildStarted,buildDone)
	}

	for {
		select {
		case ev := <-watcher.Event:
			if ev.Name != "" {
				base := filepath.Base(ev.Name)

				if flag_includedFiles.Matches(base) || matchesPattern(pattern, ev.Name) {
					if !flag_excludedFiles.Matches(base) {
						jobs <- ev.Name
					}
				}
			}

		case err := <-watcher.Error:
			if v, ok := err.(*os.SyscallError); ok {
				if v.Err == syscall.EINTR {
					continue
				}
				log.Fatal("watcher.Error: SyscallError:", v)
			}
			log.Fatal("watcher.Error:", err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"

	"github.com/gow/dyconf"
)

type daemon struct {
	fileName string
	cm       dyconf.ConfigManager
	log      *log.Logger
}

func (d *daemon) start(logger *log.Logger, fileName string, host string, port string) error {
	d.log = logger
	if fileName == "" {
		return fmt.Errorf("Invalid config file. File path cannot be empty.")
	}
	if !filepath.IsAbs(fileName) {
		return fmt.Errorf("Invalid config file: [%s]. File path must be absolute path.", fileName)
	}
	dir := filepath.Dir(fileName)
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		return fmt.Errorf("Invalid config file: [%s]. The enclosing directory [%s] does not exit.", fileName, dir)
	}

	if _, err := os.Stat(fileName); os.IsNotExist(err) {
		d.log.Printf("[INFO]: [%s] doesn't exist. Going to create a new one.", fileName)
	}

	// Open and close the config file.
	var err error
	d.cm, err = dyconf.NewManager(fileName)
	if err != nil {
		return err
	}
	d.log.Println("Starting the daemon...")
	http.HandleFunc("/config/", d.configServer)
	d.log.Fatal(
		"Failed to start the daemon.",
		http.ListenAndServe(fmt.Sprintf("%s:%s", host, port), nil),
	)
	return nil
}

func (d *daemon) stop() error {
	return d.cm.Close()
}

func (d *daemon) configServer(w http.ResponseWriter, req *http.Request) {
	if req.URL.EscapedPath() != "/config/" {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	switch req.Method {
	case "PUT":
		d.putConfig(w, req)
	case "GET":
		d.getConfig(w, req)
	default:
		w.WriteHeader(http.StatusMethodNotAllowed)
	}
}

func (d *daemon) putConfig(w http.ResponseWriter, req *http.Request) {
	resp := json.NewEncoder(w)
	key := req.FormValue("key")
	if key == "" {
		w.WriteHeader(http.StatusBadRequest)
		resp.Encode(struct{ Error string }{"Invalid Key"})
		return
	}
	val := req.FormValue("value")

	if _, err := d.cm.Get(key); err == nil {
		resp.Encode(struct{ Error string }{"key already exists. Use a POST request to modify it"})
	}
	if err := d.cm.Set(key, []byte(val)); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		resp.Encode(err)
	}
	w.WriteHeader(http.StatusOK)
}

func (d *daemon) getConfig(w http.ResponseWriter, req *http.Request) {
	keys := req.URL.Query()["key"]
	resp := json.NewEncoder(w)

	type kvpair struct {
		Key string `json:"key"`
		Val []byte `json:"value"`
		Err string `json:"error",omitempty`
	}
	var kvPairs []kvpair
	for _, key := range keys {
		pair := kvpair{Key: key}
		val, err := d.cm.Get(key)
		if err != nil {
			pair.Err = err.Error()
		} else {
			pair.Val = val
		}
		kvPairs = append(kvPairs, pair)
	}
	w.WriteHeader(http.StatusOK)
	resp.Encode(kvPairs)
}
<commit_msg>Added POST & DELETE methods<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"

	"github.com/gow/dyconf"
)

type daemon struct {
	fileName string
	cm       dyconf.ConfigManager
	log      *log.Logger
}

func (d *daemon) start(logger *log.Logger, fileName string, host string, port string) error {
	d.log = logger
	if fileName == "" {
		return fmt.Errorf("Invalid config file. File path cannot be empty.")
	}
	if !filepath.IsAbs(fileName) {
		return fmt.Errorf("Invalid config file: [%s]. File path must be absolute path.", fileName)
	}
	dir := filepath.Dir(fileName)
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		return fmt.Errorf("Invalid config file: [%s]. The enclosing directory [%s] does not exit.", fileName, dir)
	}

	if _, err := os.Stat(fileName); os.IsNotExist(err) {
		d.log.Printf("[INFO]: [%s] doesn't exist. Going to create a new one.", fileName)
	}

	// Open and close the config file.
	var err error
	d.cm, err = dyconf.NewManager(fileName)
	if err != nil {
		return err
	}
	d.log.Println("Starting the daemon...")
	http.HandleFunc("/config/", d.configServer)
	http.HandleFunc("/config/getall/", d.getAllConfig)
	d.log.Fatal(
		"Failed to start the daemon.",
		http.ListenAndServe(fmt.Sprintf("%s:%s", host, port), nil),
	)
	return nil
}

func (d *daemon) stop() error {
	return d.cm.Close()
}

func (d *daemon) configServer(w http.ResponseWriter, req *http.Request) {
	if req.URL.EscapedPath() != "/config/" {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	switch req.Method {
	case "PUT":
		d.putConfig(w, req)
	case "GET":
		d.getConfig(w, req)
	case "POST":
		d.postConfig(w, req)
	case "DELETE":
		d.deleteConfig(w, req)
	default:
		w.WriteHeader(http.StatusMethodNotAllowed)
	}
}

func (d *daemon) putConfig(w http.ResponseWriter, req *http.Request) {
	resp := json.NewEncoder(w)
	key := req.FormValue("key")
	if key == "" {
		w.WriteHeader(http.StatusBadRequest)
		resp.Encode(struct{ Error string }{"Invalid Key"})
		return
	}
	val := req.FormValue("value")

	if _, err := d.cm.Get(key); err == nil {
		resp.Encode(struct{ Error string }{"key already exists. Use a POST request to modify it"})
		return
	}
	if err := d.cm.Set(key, []byte(val)); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		resp.Encode(err)
		return
	}
	w.WriteHeader(http.StatusOK)
	resp.Encode(true)
}

func (d *daemon) getConfig(w http.ResponseWriter, req *http.Request) {
	keys := req.URL.Query()["key"]
	resp := json.NewEncoder(w)

	type kvpair struct {
		Key string `json:"key"`
		Val []byte `json:"value"`
		Err string `json:"error",omitempty`
	}
	var kvPairs []kvpair
	for _, key := range keys {
		pair := kvpair{Key: key}
		val, err := d.cm.Get(key)
		if err != nil {
			pair.Err = err.Error()
		} else {
			pair.Val = val
		}
		kvPairs = append(kvPairs, pair)
	}
	w.WriteHeader(http.StatusOK)
	resp.Encode(kvPairs)
}

func (d *daemon) getAllConfig(w http.ResponseWriter, req *http.Request) {
	if req.URL.EscapedPath() != "/config/getall/" {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	if req.Method != "GET" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	all, err := d.cm.Map()
	resp := json.NewEncoder(w)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		resp.Encode(err)
		return
	}
	w.WriteHeader(http.StatusOK)
	resp.Encode(all)

}
func (d *daemon) postConfig(w http.ResponseWriter, req *http.Request) {
	resp := json.NewEncoder(w)
	key := req.FormValue("key")
	if key == "" {
		w.WriteHeader(http.StatusBadRequest)
		resp.Encode(struct{ Error string }{"Invalid Key"})
		return
	}
	val := req.FormValue("value")

	if _, err := d.cm.Get(key); err != nil {
		resp.Encode(struct{ Error string }{"key doesn't exists. Use a PUT request to add it."})
		return
	}
	if err := d.cm.Set(key, []byte(val)); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		resp.Encode(err)
		return
	}
	w.WriteHeader(http.StatusOK)
	resp.Encode(true)
}

func (d *daemon) deleteConfig(w http.ResponseWriter, req *http.Request) {
	resp := json.NewEncoder(w)
	key := req.FormValue("key")
	if key == "" {
		w.WriteHeader(http.StatusBadRequest)
		resp.Encode(struct{ Error string }{"Invalid Key"})
		return
	}

	if _, err := d.cm.Get(key); err != nil {
		resp.Encode(struct{ Error string }{"key doesn't exists."})
		return
	}

	if err := d.cm.Delete(key); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		resp.Encode(err)
		return
	}
	w.WriteHeader(http.StatusOK)
	resp.Encode(true)
}
<|endoftext|>