<commit_before>package url

import (
	"bytes"
	"fmt"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/storage"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"strings"
	"time"
)

//Resource represents a URL based resource, with enriched meta info
type Resource struct {
	URL             string   `description:"resource URL or relative or absolute path" required:"true"` //URL of resource
	Credentials     string   `description:"credentials file"`                                          //name of credential file or credential key depending on implementation
	ParsedURL       *url.URL `json:"-"`                                                                //parsed URL resource
	Cache           string   `description:"local cache path"`                                          //Cache path for the resource, if specified resource will be cached in the specified path
	CacheExpiryMs   int      //CacheExpiryMs expiry time in ms
	modificationTag int64
	init            bool
}

//Clone creates a clone of the resource
func (r *Resource) Clone() *Resource {
	return &Resource{
		init:          r.init,
		URL:           r.URL,
		Credentials:   r.Credentials,
		ParsedURL:     r.ParsedURL,
		Cache:         r.Cache,
		CacheExpiryMs: r.CacheExpiryMs,
	}
}

var defaultSchemePorts = map[string]int{
	"ssh":   22,
	"scp":   22,
	"http":  80,
	"https": 443,
}

//Host returns url's host name with user name if user name is part of url
func (r *Resource) Host() string {
	result := r.ParsedURL.Hostname() + ":" + r.Port()
	if r.ParsedURL.User != nil {
		result = r.ParsedURL.User.Username() + "@" + result
	}
	return result
}

//CredentialURL returns url's with provided credential
func (r *Resource) CredentialURL(username, password string) string {
	var urlCredential = ""
	if username != "" {
		urlCredential = username
		if password != "" {
			urlCredential += ":" + password
		}
		urlCredential += "@"
	}
	result := r.ParsedURL.Scheme + "://" + urlCredential + r.ParsedURL.Hostname() + ":" + r.Port() + r.ParsedURL.Path
	if r.ParsedURL.RawQuery != "" {
		result += "?" + r.ParsedURL.RawQuery
	}

	return result
}

//Path returns url's path  directory, assumption is that directory does not have extension, if path ends with '/' it is being stripped.
func (r *Resource) DirectoryPath() string {
	if r.ParsedURL == nil {
		return ""
	}
	var result = r.ParsedURL.Path

	parent, name := path.Split(result)
	if path.Ext(name) != "" {
		result = parent
	}
	if strings.HasSuffix(result, "/") {
		result = string(result[:len(result)-1])
	}
	return result
}

//Port returns url's port
func (r *Resource) Port() string {
	port := r.ParsedURL.Port()
	if port == "" && r.ParsedURL != nil {
		if value, ok := defaultSchemePorts[r.ParsedURL.Scheme]; ok {
			port = toolbox.AsString(value)
		}
	}
	return port
}

//Download downloads data from URL, it returns data as []byte, or error, if resource is cacheable it first look into cache
func (r *Resource) Download() ([]byte, error) {
	if r == nil {
		return nil, fmt.Errorf("Fail to download content on empty resource")
	}
	if r.Cachable() {
		content := r.readFromCache()
		if content != nil {
			return content, nil
		}
	}
	service, err := storage.NewServiceForURL(r.URL, r.Credentials)
	if err != nil {
		return nil, err
	}
	object, err := service.StorageObject(r.URL)
	if err != nil {
		return nil, err
	}
	reader, err := service.Download(object)
	if err != nil {
		return nil, err
	}
	defer reader.Close()
	content, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}
	if r.Cachable() {
		_ = ioutil.WriteFile(r.Cache, content, 0666)
	}
	return content, err
}

//DownloadText returns a text downloaded from url
func (r *Resource) DownloadText() (string, error) {
	var result, err = r.Download()
	if err != nil {
		return "", err
	}
	return string(result), err
}

//Decode decodes url's data into target, it support JSON and YAML exp.
func (r *Resource) Decode(target interface{}) (err error) {
	defer func() {
		if err != nil {
			err = fmt.Errorf("failed to decode: %v, %v", r.URL, err)
		}
	}()
	if r.ParsedURL == nil {
		if r.ParsedURL, err = url.Parse(r.URL); err != nil {
			return err
		}
	}
	ext := path.Ext(r.ParsedURL.Path)
	switch ext {
	case ".yaml", ".yml":
		err = r.YAMLDecode(target)
	default:
		err = r.JSONDecode(target)
	}
	return err
}

//DecoderFactory returns new decoder factory for resource
func (r *Resource) DecoderFactory() toolbox.DecoderFactory {
	ext := path.Ext(r.ParsedURL.Path)
	switch ext {
	case ".yaml", ".yml":
		return toolbox.NewYamlDecoderFactory()
	default:
		return toolbox.NewJSONDecoderFactory()
	}
}

//Decode decodes url's data into target, it takes decoderFactory which decodes data into target
func (r *Resource) DecodeWith(target interface{}, decoderFactory toolbox.DecoderFactory) error {
	if r == nil {
		return fmt.Errorf("fail to %T decode on empty resource", decoderFactory)
	}
	if decoderFactory == nil {
		return fmt.Errorf("fail to decode %v, decoderFactory was empty", r.URL)
	}
	var content, err = r.Download()
	if err != nil {
		return err
	}

	text := string(content)
	if toolbox.IsNewLineDelimitedJSON(text) {
		if aSlice, err := toolbox.NewLineDelimitedJSON(text); err == nil {
			return toolbox.DefaultConverter.AssignConverted(target, aSlice)
		}
	}
	err = decoderFactory.Create(bytes.NewReader(content)).Decode(target)
	if err != nil {
		return fmt.Errorf("failed to decode: %v, payload: %s", err, content)
	}
	return err
}

//Rename renames URI name of this resource
func (r *Resource) Rename(name string) (err error) {
	var _, currentName = toolbox.URLSplit(r.URL)
	if currentName == "" && strings.HasSuffix(r.URL, "/") {
		_, currentName = toolbox.URLSplit(r.URL[:len(r.URL)-1])
		currentName += "/"
	}

	r.URL = strings.Replace(r.URL, currentName, name, 1)
	r.ParsedURL, err = url.Parse(r.URL)
	return err
}

//JSONDecode decodes json resource into target
func (r *Resource) JSONDecode(target interface{}) error {
	return r.DecodeWith(target, toolbox.NewJSONDecoderFactory())
}

//JSONDecode decodes yaml resource into target
func (r *Resource) YAMLDecode(target interface{}) error {
	if interfacePrt, ok := target.(*interface{}); ok {
		var data interface{}
		if err := r.DecodeWith(&data, toolbox.NewYamlDecoderFactory()); err != nil {
			return err
		}
		if toolbox.IsSlice(data) {
			*interfacePrt = data
			return nil
		}
	}
	var mapSlice = yaml.MapSlice{}
	if err := r.DecodeWith(&mapSlice, toolbox.NewYamlDecoderFactory()); err != nil {
		return err
	}
	if !toolbox.IsMap(target) {
		return toolbox.DefaultConverter.AssignConverted(target, mapSlice)
	}
	resultMap := toolbox.AsMap(target)
	for _, v := range mapSlice {
		resultMap[toolbox.AsString(v.Key)] = v.Value
	}
	return nil
}

func (r *Resource) readFromCache() []byte {
	if toolbox.FileExists(r.Cache) {
		info, err := os.Stat(r.Cache)
		var isExpired = false
		if err == nil && r.CacheExpiryMs > 0 {
			elapsed := time.Now().Sub(info.ModTime())
			isExpired = elapsed > time.Second*time.Duration(r.CacheExpiryMs)
		}
		content, err := ioutil.ReadFile(r.Cache)
		if err == nil && !isExpired {
			return content
		}
	}
	return nil
}

//Cachable returns true if resource is cachable
func (r *Resource) Cachable() bool {
	return r.Cache != ""
}

func computeResourceModificationTag(resource *Resource) (int64, error) {
	service, err := storage.NewServiceForURL(resource.URL, resource.Credentials)
	if err != nil {
		return 0, err
	}
	object, err := service.StorageObject(resource.URL)
	if err != nil {
		return 0, err
	}
	var fileInfo = object.FileInfo()

	if object.IsContent() {
		return fileInfo.Size() + fileInfo.ModTime().UnixNano(), nil
	}
	var result int64 = 0
	objects, err := service.List(resource.URL)
	if err != nil {
		return 0, err
	}
	for _, object := range objects {
		objectResource := NewResource(object.URL())
		if objectResource.ParsedURL.Path == resource.ParsedURL.Path {
			continue
		}
		modificationTag, err := computeResourceModificationTag(NewResource(object.URL(), resource.Credentials))
		if err != nil {
			return 0, err
		}
		result += modificationTag

	}
	return result, nil
}

func (r *Resource) HasChanged() (changed bool, err error) {
	if r.modificationTag == 0 {
		r.modificationTag, err = computeResourceModificationTag(r)
		return false, err
	}
	var recentModificationTag int64
	recentModificationTag, err = computeResourceModificationTag(r)
	if err != nil {
		return false, err
	}
	if recentModificationTag != r.modificationTag {
		changed = true
		r.modificationTag = recentModificationTag
	}
	return changed, err
}

func normalizeURL(URL string) string {
	if strings.Contains(URL, "://") {
		var protoPosition = strings.Index(URL, "://")
		if protoPosition != -1 {
			var urlSuffix = string(URL[protoPosition+3:])
			urlSuffix = strings.Replace(urlSuffix, "//", "/", len(urlSuffix))
			URL = string(URL[:protoPosition+3]) + urlSuffix
		}
		return URL
	}
	if !strings.HasPrefix(URL, "/") {
		currentDirectory, _ := os.Getwd()

		if strings.Contains(URL, "..") {
			fragments := strings.Split(URL, "/")
			var index = 0
			var offset = 0
			if fragments[0] == "." {
				offset = 1
			}

			for index = offset; index < len(fragments); index++ {
				var fragment = fragments[index]
				if fragment == ".." {
					currentDirectory, _ = path.Split(currentDirectory)
					if strings.HasSuffix(currentDirectory, "/") {
						currentDirectory = string(currentDirectory[:len(currentDirectory)-1])
					}
					continue
				}
				break
			}
			return toolbox.FileSchema + path.Join(currentDirectory, strings.Join(fragments[index:], "/"))
		}

		currentDirectory, err := os.Getwd()
		if err == nil {
			candidate := path.Join(currentDirectory, URL)
			URL = candidate
		}
	}
	return toolbox.FileSchema + URL
}

func (r *Resource) Init() (err error) {
	if r.init {
		return nil
	}
	r.init = true
	r.URL = normalizeURL(r.URL)
	r.ParsedURL, err = url.Parse(r.URL)
	return err
}

//NewResource returns a new resource for provided URL, followed by optional credential, cache and cache expiryMs.
func NewResource(Params ...interface{}) *Resource {
	if len(Params) == 0 {
		return nil
	}
	var URL = toolbox.AsString(Params[0])
	URL = normalizeURL(URL)

	var credential string
	if len(Params) > 1 {
		credential = toolbox.AsString(Params[1])
	}
	var cache string
	if len(Params) > 2 {
		cache = toolbox.AsString(Params[2])
	}
	var cacheExpiryMs int
	if len(Params) > 3 {
		cacheExpiryMs = toolbox.AsInt(Params[3])
	}
	parsedURL, _ := url.Parse(URL)
	return &Resource{
		init:          true,
		ParsedURL:     parsedURL,
		URL:           URL,
		Credentials:   credential,
		Cache:         cache,
		CacheExpiryMs: cacheExpiryMs,
	}
}
<commit_msg>added DownloadBase64<commit_after>package url

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/storage"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"strings"
	"time"
)

//Resource represents a URL based resource, with enriched meta info
type Resource struct {
	URL             string   `description:"resource URL or relative or absolute path" required:"true"` //URL of resource
	Credentials     string   `description:"credentials file"`                                          //name of credential file or credential key depending on implementation
	ParsedURL       *url.URL `json:"-"`                                                                //parsed URL resource
	Cache           string   `description:"local cache path"`                                          //Cache path for the resource, if specified resource will be cached in the specified path
	CacheExpiryMs   int      //CacheExpiryMs expiry time in ms
	modificationTag int64
	init            bool
}

//Clone creates a clone of the resource
func (r *Resource) Clone() *Resource {
	return &Resource{
		init:          r.init,
		URL:           r.URL,
		Credentials:   r.Credentials,
		ParsedURL:     r.ParsedURL,
		Cache:         r.Cache,
		CacheExpiryMs: r.CacheExpiryMs,
	}
}

var defaultSchemePorts = map[string]int{
	"ssh":   22,
	"scp":   22,
	"http":  80,
	"https": 443,
}

//Host returns url's host name with user name if user name is part of url
func (r *Resource) Host() string {
	result := r.ParsedURL.Hostname() + ":" + r.Port()
	if r.ParsedURL.User != nil {
		result = r.ParsedURL.User.Username() + "@" + result
	}
	return result
}

//CredentialURL returns url's with provided credential
func (r *Resource) CredentialURL(username, password string) string {
	var urlCredential = ""
	if username != "" {
		urlCredential = username
		if password != "" {
			urlCredential += ":" + password
		}
		urlCredential += "@"
	}
	result := r.ParsedURL.Scheme + "://" + urlCredential + r.ParsedURL.Hostname() + ":" + r.Port() + r.ParsedURL.Path
	if r.ParsedURL.RawQuery != "" {
		result += "?" + r.ParsedURL.RawQuery
	}

	return result
}

//Path returns url's path  directory, assumption is that directory does not have extension, if path ends with '/' it is being stripped.
func (r *Resource) DirectoryPath() string {
	if r.ParsedURL == nil {
		return ""
	}
	var result = r.ParsedURL.Path

	parent, name := path.Split(result)
	if path.Ext(name) != "" {
		result = parent
	}
	if strings.HasSuffix(result, "/") {
		result = string(result[:len(result)-1])
	}
	return result
}

//Port returns url's port
func (r *Resource) Port() string {
	port := r.ParsedURL.Port()
	if port == "" && r.ParsedURL != nil {
		if value, ok := defaultSchemePorts[r.ParsedURL.Scheme]; ok {
			port = toolbox.AsString(value)
		}
	}
	return port
}

//Download downloads data from URL, it returns data as []byte, or error, if resource is cacheable it first look into cache
func (r *Resource) Download() ([]byte, error) {
	if r == nil {
		return nil, fmt.Errorf("Fail to download content on empty resource")
	}
	if r.Cachable() {
		content := r.readFromCache()
		if content != nil {
			return content, nil
		}
	}
	service, err := storage.NewServiceForURL(r.URL, r.Credentials)
	if err != nil {
		return nil, err
	}
	object, err := service.StorageObject(r.URL)
	if err != nil {
		return nil, err
	}
	reader, err := service.Download(object)
	if err != nil {
		return nil, err
	}
	defer reader.Close()
	content, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}
	if r.Cachable() {
		_ = ioutil.WriteFile(r.Cache, content, 0666)
	}
	return content, err
}

//DownloadText returns a text downloaded from url
func (r *Resource) DownloadText() (string, error) {
	var result, err = r.Download()
	if err != nil {
		return "", err
	}
	return string(result), err
}

//Decode decodes url's data into target, it support JSON and YAML exp.
func (r *Resource) Decode(target interface{}) (err error) {
	defer func() {
		if err != nil {
			err = fmt.Errorf("failed to decode: %v, %v", r.URL, err)
		}
	}()
	if r.ParsedURL == nil {
		if r.ParsedURL, err = url.Parse(r.URL); err != nil {
			return err
		}
	}
	ext := path.Ext(r.ParsedURL.Path)
	switch ext {
	case ".yaml", ".yml":
		err = r.YAMLDecode(target)
	default:
		err = r.JSONDecode(target)
	}
	return err
}

//DecoderFactory returns new decoder factory for resource
func (r *Resource) DecoderFactory() toolbox.DecoderFactory {
	ext := path.Ext(r.ParsedURL.Path)
	switch ext {
	case ".yaml", ".yml":
		return toolbox.NewYamlDecoderFactory()
	default:
		return toolbox.NewJSONDecoderFactory()
	}
}

//Decode decodes url's data into target, it takes decoderFactory which decodes data into target
func (r *Resource) DecodeWith(target interface{}, decoderFactory toolbox.DecoderFactory) error {
	if r == nil {
		return fmt.Errorf("fail to %T decode on empty resource", decoderFactory)
	}
	if decoderFactory == nil {
		return fmt.Errorf("fail to decode %v, decoderFactory was empty", r.URL)
	}
	var content, err = r.Download()
	if err != nil {
		return err
	}

	text := string(content)
	if toolbox.IsNewLineDelimitedJSON(text) {
		if aSlice, err := toolbox.NewLineDelimitedJSON(text); err == nil {
			return toolbox.DefaultConverter.AssignConverted(target, aSlice)
		}
	}
	err = decoderFactory.Create(bytes.NewReader(content)).Decode(target)
	if err != nil {
		return fmt.Errorf("failed to decode: %v, payload: %s", err, content)
	}
	return err
}

//Rename renames URI name of this resource
func (r *Resource) Rename(name string) (err error) {
	var _, currentName = toolbox.URLSplit(r.URL)
	if currentName == "" && strings.HasSuffix(r.URL, "/") {
		_, currentName = toolbox.URLSplit(r.URL[:len(r.URL)-1])
		currentName += "/"
	}

	r.URL = strings.Replace(r.URL, currentName, name, 1)
	r.ParsedURL, err = url.Parse(r.URL)
	return err
}

//JSONDecode decodes json resource into target
func (r *Resource) JSONDecode(target interface{}) error {
	return r.DecodeWith(target, toolbox.NewJSONDecoderFactory())
}

//JSONDecode decodes yaml resource into target
func (r *Resource) YAMLDecode(target interface{}) error {
	if interfacePrt, ok := target.(*interface{}); ok {
		var data interface{}
		if err := r.DecodeWith(&data, toolbox.NewYamlDecoderFactory()); err != nil {
			return err
		}
		if toolbox.IsSlice(data) {
			*interfacePrt = data
			return nil
		}
	}
	var mapSlice = yaml.MapSlice{}
	if err := r.DecodeWith(&mapSlice, toolbox.NewYamlDecoderFactory()); err != nil {
		return err
	}
	if !toolbox.IsMap(target) {
		return toolbox.DefaultConverter.AssignConverted(target, mapSlice)
	}
	resultMap := toolbox.AsMap(target)
	for _, v := range mapSlice {
		resultMap[toolbox.AsString(v.Key)] = v.Value
	}
	return nil
}

func (r *Resource) readFromCache() []byte {
	if toolbox.FileExists(r.Cache) {
		info, err := os.Stat(r.Cache)
		var isExpired = false
		if err == nil && r.CacheExpiryMs > 0 {
			elapsed := time.Now().Sub(info.ModTime())
			isExpired = elapsed > time.Second*time.Duration(r.CacheExpiryMs)
		}
		content, err := ioutil.ReadFile(r.Cache)
		if err == nil && !isExpired {
			return content
		}
	}
	return nil
}

//Cachable returns true if resource is cachable
func (r *Resource) Cachable() bool {
	return r.Cache != ""
}

func computeResourceModificationTag(resource *Resource) (int64, error) {
	service, err := storage.NewServiceForURL(resource.URL, resource.Credentials)
	if err != nil {
		return 0, err
	}
	object, err := service.StorageObject(resource.URL)
	if err != nil {
		return 0, err
	}
	var fileInfo = object.FileInfo()

	if object.IsContent() {
		return fileInfo.Size() + fileInfo.ModTime().UnixNano(), nil
	}
	var result int64 = 0
	objects, err := service.List(resource.URL)
	if err != nil {
		return 0, err
	}
	for _, object := range objects {
		objectResource := NewResource(object.URL())
		if objectResource.ParsedURL.Path == resource.ParsedURL.Path {
			continue
		}
		modificationTag, err := computeResourceModificationTag(NewResource(object.URL(), resource.Credentials))
		if err != nil {
			return 0, err
		}
		result += modificationTag

	}
	return result, nil
}

func (r *Resource) HasChanged() (changed bool, err error) {
	if r.modificationTag == 0 {
		r.modificationTag, err = computeResourceModificationTag(r)
		return false, err
	}
	var recentModificationTag int64
	recentModificationTag, err = computeResourceModificationTag(r)
	if err != nil {
		return false, err
	}
	if recentModificationTag != r.modificationTag {
		changed = true
		r.modificationTag = recentModificationTag
	}
	return changed, err
}

func normalizeURL(URL string) string {
	if strings.Contains(URL, "://") {
		var protoPosition = strings.Index(URL, "://")
		if protoPosition != -1 {
			var urlSuffix = string(URL[protoPosition+3:])
			urlSuffix = strings.Replace(urlSuffix, "//", "/", len(urlSuffix))
			URL = string(URL[:protoPosition+3]) + urlSuffix
		}
		return URL
	}
	if !strings.HasPrefix(URL, "/") {
		currentDirectory, _ := os.Getwd()

		if strings.Contains(URL, "..") {
			fragments := strings.Split(URL, "/")
			var index = 0
			var offset = 0
			if fragments[0] == "." {
				offset = 1
			}

			for index = offset; index < len(fragments); index++ {
				var fragment = fragments[index]
				if fragment == ".." {
					currentDirectory, _ = path.Split(currentDirectory)
					if strings.HasSuffix(currentDirectory, "/") {
						currentDirectory = string(currentDirectory[:len(currentDirectory)-1])
					}
					continue
				}
				break
			}
			return toolbox.FileSchema + path.Join(currentDirectory, strings.Join(fragments[index:], "/"))
		}

		currentDirectory, err := os.Getwd()
		if err == nil {
			candidate := path.Join(currentDirectory, URL)
			URL = candidate
		}
	}
	return toolbox.FileSchema + URL
}

func (r *Resource) Init() (err error) {
	if r.init {
		return nil
	}
	r.init = true
	r.URL = normalizeURL(r.URL)
	r.ParsedURL, err = url.Parse(r.URL)
	return err
}

//DownloadBase64 loads base64 resource content
func (r *Resource) DownloadBase64() (string, error) {
	storageService, err := storage.NewServiceForURL(r.URL, r.Credentials)
	if err != nil {
		return "", err
	}
	reader, err := storage.Download(storageService, r.URL)
	if err != nil {
		return "", err
	}
	defer reader.Close()
	data, err := ioutil.ReadAll(reader)
	if err != nil {
		return "", err
	}
	_, err = base64.StdEncoding.DecodeString(string(data))
	if err == nil {
		return string(data), nil
	}
	return base64.StdEncoding.EncodeToString(data), nil
}

//NewResource returns a new resource for provided URL, followed by optional credential, cache and cache expiryMs.
func NewResource(Params ...interface{}) *Resource {
	if len(Params) == 0 {
		return nil
	}
	var URL = toolbox.AsString(Params[0])
	URL = normalizeURL(URL)

	var credential string
	if len(Params) > 1 {
		credential = toolbox.AsString(Params[1])
	}
	var cache string
	if len(Params) > 2 {
		cache = toolbox.AsString(Params[2])
	}
	var cacheExpiryMs int
	if len(Params) > 3 {
		cacheExpiryMs = toolbox.AsInt(Params[3])
	}
	parsedURL, _ := url.Parse(URL)
	return &Resource{
		init:          true,
		ParsedURL:     parsedURL,
		URL:           URL,
		Credentials:   credential,
		Cache:         cache,
		CacheExpiryMs: cacheExpiryMs,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path"

	"github.com/google/go-github/github"
)

var (
	user     *string
	upstream *string
	debug    *bool
	outdated *bool
)

func init() {
	user = flag.String("user", os.Getenv("USER"), "Github username")
	upstream = flag.String("upstream", "upstream", "Github user or org for upstream fork")
	debug = flag.Bool("debug", false, "Enable debug logging")
	outdated = flag.Bool("outdated", false, "Show outdated comments")
	flag.Parse()
}

func main() {
	remotes, _ := getRemotes()
	shas, _ := getSHA()
	if *debug {
		for _, sha := range shas {
			log.Println("SHA:", sha)
		}
	}

	var remote *Remote
	for _, r := range remotes {
		if r.User == *upstream {
			remote = r
			break
		}
	}
	if remote == nil {
		log.Fatalln("Couldn't find upstream remote named", *upstream)
	}
	client := github.NewClient(nil)
	data, _, _ := client.PullRequests.List(remote.User, remote.Repo, nil)
	for _, pr := range data {
		for _, sha := range shas {
			if *pr.Head.SHA == sha {
				if *debug {
					log.Println("Found the PR:", *pr.URL)
				}
				getCommentsForPullRequest(client, remote, pr)
				os.Exit(0)
			}
		}
	}
	fmt.Println("Couldn't find Pull Request matching this branch")
	os.Exit(1)
}

func getCommentsForPullRequest(client *github.Client, remote *Remote, pr github.PullRequest) {
	comments, _, _ := client.PullRequests.ListComments(remote.User, remote.Repo, *pr.Number, nil)
	diff := getDiffFromURL(*pr.URL)
	diffMap := processDiffIntoDiffMap(diff)
	basePath, _ := getTopLevelPath()
	for _, comment := range comments {
		p := basePath
		if comment.Path != nil {
			p = path.Join(basePath, *comment.Path)
		}
		pos := 0
		if comment.Position != nil && comment.Path != nil {
			diffLine := diffMap[*comment.Path][*comment.Position]
			fmt.Println(diffLine.Line)
			pos = diffLine.RightIndex
		} else if !*outdated {
			continue
		}
		fmt.Printf("%s:%d:%s (@%s)\n", p, pos, *comment.Body, *comment.User.Login)
	}
}
<commit_msg>use local name for upstream<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path"

	"github.com/google/go-github/github"
)

var (
	user     *string
	upstream *string
	debug    *bool
	outdated *bool
)

func init() {
	user = flag.String("user", os.Getenv("USER"), "Github username")
	upstream = flag.String("upstream", "upstream", "Github user or org for upstream fork")
	debug = flag.Bool("debug", false, "Enable debug logging")
	outdated = flag.Bool("outdated", false, "Show outdated comments")
	flag.Parse()
}

func main() {
	remotes, _ := getRemotes()
	shas, _ := getSHA()
	if *debug {
		for _, sha := range shas {
			log.Println("SHA:", sha)
		}
	}

	var remote *Remote
	for _, r := range remotes {
		if r.LocalName == *upstream {
			remote = r
			break
		}
	}
	if remote == nil {
		log.Fatalln("Couldn't find upstream remote named", *upstream)
	}
	client := github.NewClient(nil)
	data, _, _ := client.PullRequests.List(remote.User, remote.Repo, nil)
	for _, pr := range data {
		for _, sha := range shas {
			if *pr.Head.SHA == sha {
				if *debug {
					log.Println("Found the PR:", *pr.URL)
				}
				getCommentsForPullRequest(client, remote, pr)
				os.Exit(0)
			}
		}
	}
	fmt.Println("Couldn't find Pull Request matching this branch")
	os.Exit(1)
}

func getCommentsForPullRequest(client *github.Client, remote *Remote, pr github.PullRequest) {
	comments, _, _ := client.PullRequests.ListComments(remote.User, remote.Repo, *pr.Number, nil)
	diff := getDiffFromURL(*pr.URL)
	diffMap := processDiffIntoDiffMap(diff)
	basePath, _ := getTopLevelPath()
	for _, comment := range comments {
		p := basePath
		if comment.Path != nil {
			p = path.Join(basePath, *comment.Path)
		}
		pos := 0
		if comment.Position != nil && comment.Path != nil {
			diffLine := diffMap[*comment.Path][*comment.Position]
			fmt.Println(diffLine.Line)
			pos = diffLine.RightIndex
		} else if !*outdated {
			continue
		}
		fmt.Printf("%s:%d:%s (@%s)\n", p, pos, *comment.Body, *comment.User.Login)
	}
}
<|endoftext|>


<commit_before>package termite

import (
	"fmt"
	"path/filepath"
	"os"
	"log"
	"io/ioutil"
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/unionfs"
	"rpc"
	"sync"
)

// State associated with one master.
type Mirror struct {
	daemon        *WorkerDaemon
	fileServer    *rpc.Client
	rpcFs         *RpcFs
	writableRoot  string

	fuseFileSystemsMutex sync.Mutex
	fuseFileSystems      []*WorkerFuseFs
	workingFileSystems   map[*WorkerFuseFs]string
}

func (me *Mirror) ReturnFuse(wfs *WorkerFuseFs) {
	wfs.unionFs.DropBranchCache()
	wfs.unionFs.DropDeletionCache()

	me.fuseFileSystemsMutex.Lock()
	defer me.fuseFileSystemsMutex.Unlock()
	me.fuseFileSystems = append(me.fuseFileSystems, wfs)
	me.workingFileSystems[wfs] = "", false
}

func (me *Mirror) getWorkerFuseFs(name string) (f *WorkerFuseFs, err os.Error) {
	me.fuseFileSystemsMutex.Lock()
	defer me.fuseFileSystemsMutex.Unlock()
	l := len(me.fuseFileSystems)
	if l > 0 {
		f = me.fuseFileSystems[l-1]
		me.fuseFileSystems = me.fuseFileSystems[:l-1]
	}
	if f == nil {
		f, err = me.newWorkerFuseFs()
	}
	me.workingFileSystems[f] = name
	return f, err
}

func (me *Mirror) Update(req *UpdateRequest, rep *UpdateResponse) os.Error {
	return me.rpcFs.Update(req, rep)
}

func (me *Mirror) Run(req *WorkRequest, rep *WorkReply) os.Error {
	task, err := me.newWorkerTask(req, rep)

	err = task.Run()
	if err != nil {
		log.Println("Error", err)
		return err
	}

	updateReq := UpdateRequest{
	Files: rep.Files,
	}
	updateRep := UpdateResponse{}
	err = me.rpcFs.Update(&updateReq, &updateRep)
	if err != nil {
		// TODO - fatal?
		log.Println("Update failed.")
	}

	summary := rep
	// Trim output.

	summary.Stdout = trim(summary.Stdout)
	summary.Stderr = trim(summary.Stderr)

	log.Println("sending back", summary)
	return nil
}

const _DELETIONS = "DELETIONS"

func (me *Mirror) newWorkerFuseFs() (*WorkerFuseFs, os.Error) {
	w := WorkerFuseFs{}

	tmpDir, err := ioutil.TempDir("", "rpcfs-tmp")
	w.tmpDir = tmpDir
	type dirInit struct {
		dst *string
		val string
	}

	for _, v := range []dirInit{
		dirInit{&w.rwDir, "rw"},
		dirInit{&w.mount, "mnt"},
	} {
		*v.dst = filepath.Join(w.tmpDir, v.val)
		err = os.Mkdir(*v.dst, 0700)
		if err != nil {
			return nil, err
		}
	}

	rwFs := fuse.NewLoopbackFileSystem(w.rwDir)

	// High ttl, since all writes come through fuse.
	ttl := 100.0
	opts := unionfs.UnionFsOptions{
		BranchCacheTTLSecs:   ttl,
		DeletionCacheTTLSecs: ttl,
		DeletionDirName:      _DELETIONS,
	}
	mOpts := fuse.FileSystemOptions{
		EntryTimeout:    ttl,
		AttrTimeout:     ttl,
		NegativeTimeout: ttl,
	}

	w.unionFs = unionfs.NewUnionFs("ufs", []fuse.FileSystem{rwFs, me.rpcFs}, opts)
	swFs := []fuse.SwitchedFileSystem{
		{"dev", &DevNullFs{}, true},
		{"", me.rpcFs, false},

		// TODO - configurable.
		{"tmp", w.unionFs, false},
		{me.writableRoot, w.unionFs, false},
	}

	conn := fuse.NewFileSystemConnector(fuse.NewSwitchFileSystem(swFs), &mOpts)
	w.MountState = fuse.NewMountState(conn)
	w.MountState.Mount(w.mount, &fuse.MountOptions{AllowOther: true})
	if err != nil {
		return nil, err
	}

	go w.MountState.Loop(true)

	return &w, nil
}

func (me *Mirror) newWorkerTask(req *WorkRequest, rep *WorkReply) (*WorkerTask, os.Error) {
	fuseFs, err := me.getWorkerFuseFs(fmt.Sprintf("%v", req))
	if err != nil {
		return nil, err
	}
	stdin := me.daemon.pending.WaitConnection(req.StdinId)
	return &WorkerTask{
		WorkRequest:  req,
		WorkReply:    rep,
		stdinConn:    stdin,
		masterWorker: me,
		fuseFs:       fuseFs,
	}, nil
}
<commit_msg>Tune TTLs.<commit_after>package termite

import (
	"fmt"
	"path/filepath"
	"os"
	"log"
	"io/ioutil"
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/unionfs"
	"rpc"
	"sync"
)

// State associated with one master.
type Mirror struct {
	daemon        *WorkerDaemon
	fileServer    *rpc.Client
	rpcFs         *RpcFs
	writableRoot  string

	fuseFileSystemsMutex sync.Mutex
	fuseFileSystems      []*WorkerFuseFs
	workingFileSystems   map[*WorkerFuseFs]string
}

func (me *Mirror) ReturnFuse(wfs *WorkerFuseFs) {
	wfs.unionFs.DropBranchCache()
	wfs.unionFs.DropDeletionCache()

	me.fuseFileSystemsMutex.Lock()
	defer me.fuseFileSystemsMutex.Unlock()
	me.fuseFileSystems = append(me.fuseFileSystems, wfs)
	me.workingFileSystems[wfs] = "", false
}

func (me *Mirror) getWorkerFuseFs(name string) (f *WorkerFuseFs, err os.Error) {
	me.fuseFileSystemsMutex.Lock()
	defer me.fuseFileSystemsMutex.Unlock()
	l := len(me.fuseFileSystems)
	if l > 0 {
		f = me.fuseFileSystems[l-1]
		me.fuseFileSystems = me.fuseFileSystems[:l-1]
	}
	if f == nil {
		f, err = me.newWorkerFuseFs()
	}
	me.workingFileSystems[f] = name
	return f, err
}

func (me *Mirror) Update(req *UpdateRequest, rep *UpdateResponse) os.Error {
	return me.rpcFs.Update(req, rep)
}

func (me *Mirror) Run(req *WorkRequest, rep *WorkReply) os.Error {
	task, err := me.newWorkerTask(req, rep)

	err = task.Run()
	if err != nil {
		log.Println("Error", err)
		return err
	}

	updateReq := UpdateRequest{
	Files: rep.Files,
	}
	updateRep := UpdateResponse{}
	err = me.rpcFs.Update(&updateReq, &updateRep)
	if err != nil {
		// TODO - fatal?
		log.Println("Update failed.")
	}

	summary := rep
	// Trim output.

	summary.Stdout = trim(summary.Stdout)
	summary.Stderr = trim(summary.Stderr)

	log.Println("sending back", summary)
	return nil
}

const _DELETIONS = "DELETIONS"

func (me *Mirror) newWorkerFuseFs() (*WorkerFuseFs, os.Error) {
	w := WorkerFuseFs{}

	tmpDir, err := ioutil.TempDir("", "rpcfs-tmp")
	w.tmpDir = tmpDir
	type dirInit struct {
		dst *string
		val string
	}

	for _, v := range []dirInit{
		dirInit{&w.rwDir, "rw"},
		dirInit{&w.mount, "mnt"},
	} {
		*v.dst = filepath.Join(w.tmpDir, v.val)
		err = os.Mkdir(*v.dst, 0700)
		if err != nil {
			return nil, err
		}
	}

	rwFs := fuse.NewLoopbackFileSystem(w.rwDir)

	ttl := 5.0
	opts := unionfs.UnionFsOptions{
		BranchCacheTTLSecs:   ttl,
		DeletionCacheTTLSecs: ttl,
		DeletionDirName:      _DELETIONS,
	}
	mOpts := fuse.FileSystemOptions{
		EntryTimeout:    ttl,
		AttrTimeout:     ttl,
		NegativeTimeout: 0.01,
	}

	w.unionFs = unionfs.NewUnionFs("ufs", []fuse.FileSystem{rwFs, me.rpcFs}, opts)
	swFs := []fuse.SwitchedFileSystem{
		{"dev", &DevNullFs{}, true},
		{"", me.rpcFs, false},

		// TODO - configurable.
		{"tmp", w.unionFs, false},
		{me.writableRoot, w.unionFs, false},
	}

	conn := fuse.NewFileSystemConnector(fuse.NewSwitchFileSystem(swFs), &mOpts)
	w.MountState = fuse.NewMountState(conn)
	w.MountState.Mount(w.mount, &fuse.MountOptions{AllowOther: true})
	if err != nil {
		return nil, err
	}

	go w.MountState.Loop(true)

	return &w, nil
}

func (me *Mirror) newWorkerTask(req *WorkRequest, rep *WorkReply) (*WorkerTask, os.Error) {
	fuseFs, err := me.getWorkerFuseFs(fmt.Sprintf("%v", req))
	if err != nil {
		return nil, err
	}
	stdin := me.daemon.pending.WaitConnection(req.StdinId)
	return &WorkerTask{
		WorkRequest:  req,
		WorkReply:    rep,
		stdinConn:    stdin,
		masterWorker: me,
		fuseFs:       fuseFs,
	}, nil
}
<|endoftext|>


<commit_before>package application_test

import (
	testapi "github.com/cloudfoundry/cli/cf/api/fakes"
	"github.com/cloudfoundry/cli/cf/configuration"
	"github.com/cloudfoundry/cli/cf/models"
	testcmd "github.com/cloudfoundry/cli/testhelpers/commands"
	testconfig "github.com/cloudfoundry/cli/testhelpers/configuration"
	testreq "github.com/cloudfoundry/cli/testhelpers/requirements"
	testterm "github.com/cloudfoundry/cli/testhelpers/terminal"

	. "github.com/cloudfoundry/cli/cf/commands/application"
	. "github.com/cloudfoundry/cli/testhelpers/matchers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("files command", func() {
	var (
		ui                  *testterm.FakeUI
		configRepo          configuration.ReadWriter
		requirementsFactory *testreq.FakeReqFactory
		appFilesRepo        *testapi.FakeAppFilesRepo
	)

	BeforeEach(func() {
		ui = &testterm.FakeUI{}
		configRepo = testconfig.NewRepositoryWithDefaults()
		appFilesRepo = &testapi.FakeAppFilesRepo{}
		requirementsFactory = &testreq.FakeReqFactory{}
	})

	runCommand := func(args ...string) {
		testcmd.RunCommand(NewFiles(ui, configRepo, appFilesRepo), args, requirementsFactory)
	}

	Describe("requirements", func() {
		It("fails when not logged in", func() {
			requirementsFactory.TargetedSpaceSuccess = true
			runCommand("my-app", "/foo")
		})

		It("fails when a space is not targeted", func() {
			requirementsFactory.LoginSuccess = true
			runCommand("my-app", "/foo")
			Expect(testcmd.CommandDidPassRequirements).To(BeFalse())
		})

		It("fails with usage when not provided an app name", func() {
			requirementsFactory.LoginSuccess = true
			requirementsFactory.TargetedSpaceSuccess = true
			runCommand()

			Expect(ui.FailedWithUsage).To(BeTrue())
			Expect(testcmd.CommandDidPassRequirements).To(BeFalse())
		})
	})

	Context("when logged in, a space is targeted and a valid app name is provided", func() {
		BeforeEach(func() {
			app := models.Application{}
			app.Name = "my-found-app"
			app.Guid = "my-app-guid"

			requirementsFactory.Application = app
			requirementsFactory.LoginSuccess = true
			requirementsFactory.TargetedSpaceSuccess = true
			appFilesRepo.FileList = "file 1\nfile 2"
		})

		It("it lists files in a directory", func() {
			runCommand("my-app", "/foo")

			Expect(ui.Outputs).To(ContainSubstrings(
				[]string{"Getting files for app", "my-found-app", "my-org", "my-space", "my-user"},
				[]string{"OK"},
				[]string{"file 1"},
				[]string{"file 2"},
			))

			Expect(appFilesRepo.AppGuid).To(Equal("my-app-guid"))
			Expect(appFilesRepo.Path).To(Equal("/foo"))
		})

		It("does not interpolate or interpret special format characters as though it should be a format string", func() {
			appFilesRepo.FileList = "%s %d %i"
			runCommand("my-app", "/foo")

			Expect(ui.Outputs).To(ContainSubstrings([]string{"%s %d %i"}))
		})

		Context("checking for bad flags", func() {
			It("fails when non-positive value is given for instance", func() {
				runCommand("-i", "-1", "my-app")

				println(ui.Outputs)
				Expect(ui.Outputs).To(ContainSubstrings(
					[]string{"FAILED"},
					[]string{"Invalid instance"},
					[]string{"Instance must be a positive integer"},
				))
			})

			It("fails when instance is larger than instance count", func() {
				runCommand("-i", "5", "my-app")

				println(ui.Outputs)
				Expect(ui.Outputs).To(ContainSubstrings(
					[]string{"FAILED"},
					[]string{"Invalid instance"},
					[]string{"Instance must be less than"},
				))
			})

		})

	})
})
<commit_msg>Delete println from tests.<commit_after>package application_test

import (
	testapi "github.com/cloudfoundry/cli/cf/api/fakes"
	"github.com/cloudfoundry/cli/cf/configuration"
	"github.com/cloudfoundry/cli/cf/models"
	testcmd "github.com/cloudfoundry/cli/testhelpers/commands"
	testconfig "github.com/cloudfoundry/cli/testhelpers/configuration"
	testreq "github.com/cloudfoundry/cli/testhelpers/requirements"
	testterm "github.com/cloudfoundry/cli/testhelpers/terminal"

	. "github.com/cloudfoundry/cli/cf/commands/application"
	. "github.com/cloudfoundry/cli/testhelpers/matchers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("files command", func() {
	var (
		ui                  *testterm.FakeUI
		configRepo          configuration.ReadWriter
		requirementsFactory *testreq.FakeReqFactory
		appFilesRepo        *testapi.FakeAppFilesRepo
	)

	BeforeEach(func() {
		ui = &testterm.FakeUI{}
		configRepo = testconfig.NewRepositoryWithDefaults()
		appFilesRepo = &testapi.FakeAppFilesRepo{}
		requirementsFactory = &testreq.FakeReqFactory{}
	})

	runCommand := func(args ...string) {
		testcmd.RunCommand(NewFiles(ui, configRepo, appFilesRepo), args, requirementsFactory)
	}

	Describe("requirements", func() {
		It("fails when not logged in", func() {
			requirementsFactory.TargetedSpaceSuccess = true
			runCommand("my-app", "/foo")
		})

		It("fails when a space is not targeted", func() {
			requirementsFactory.LoginSuccess = true
			runCommand("my-app", "/foo")
			Expect(testcmd.CommandDidPassRequirements).To(BeFalse())
		})

		It("fails with usage when not provided an app name", func() {
			requirementsFactory.LoginSuccess = true
			requirementsFactory.TargetedSpaceSuccess = true
			runCommand()

			Expect(ui.FailedWithUsage).To(BeTrue())
			Expect(testcmd.CommandDidPassRequirements).To(BeFalse())
		})
	})

	Context("when logged in, a space is targeted and a valid app name is provided", func() {
		BeforeEach(func() {
			app := models.Application{}
			app.Name = "my-found-app"
			app.Guid = "my-app-guid"

			requirementsFactory.Application = app
			requirementsFactory.LoginSuccess = true
			requirementsFactory.TargetedSpaceSuccess = true
			appFilesRepo.FileList = "file 1\nfile 2"
		})

		It("it lists files in a directory", func() {
			runCommand("my-app", "/foo")

			Expect(ui.Outputs).To(ContainSubstrings(
				[]string{"Getting files for app", "my-found-app", "my-org", "my-space", "my-user"},
				[]string{"OK"},
				[]string{"file 1"},
				[]string{"file 2"},
			))

			Expect(appFilesRepo.AppGuid).To(Equal("my-app-guid"))
			Expect(appFilesRepo.Path).To(Equal("/foo"))
		})

		It("does not interpolate or interpret special format characters as though it should be a format string", func() {
			appFilesRepo.FileList = "%s %d %i"
			runCommand("my-app", "/foo")

			Expect(ui.Outputs).To(ContainSubstrings([]string{"%s %d %i"}))
		})

		Context("checking for bad flags", func() {
			It("fails when non-positive value is given for instance", func() {
				runCommand("-i", "-1", "my-app")

				Expect(ui.Outputs).To(ContainSubstrings(
					[]string{"FAILED"},
					[]string{"Invalid instance"},
					[]string{"Instance must be a positive integer"},
				))
			})

			It("fails when instance is larger than instance count", func() {
				runCommand("-i", "5", "my-app")

				Expect(ui.Outputs).To(ContainSubstrings(
					[]string{"FAILED"},
					[]string{"Invalid instance"},
					[]string{"Instance must be less than"},
				))
			})

		})

	})
})
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2016 Roy Xu

package redis

type respPubSub interface {
	// PSUBSCRIBE pattern [pattern ...] Listen for messages published to channels matching the given patterns
	// PUBSUB subcommand [argument [argument ...]] Inspect the state of the Pub/Sub subsystem
	// PUBLISH channel message Post a message to a channel
	// PUNSUBSCRIBE [pattern [pattern ...]] Stop listening for messages posted to channels matching the given patterns
	// SUBSCRIBE channel [channel ...] Listen for messages published to the given channels
	// UNSUBSCRIBE [channel [channel ...]] Stop listening for messages posted to the given channels
}
<commit_msg>Create a new branch PubSub<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2016 Roy Xu

package redis

type respPubSub interface {
	// PSUBSCRIBE pattern [pattern ...]
	// Listen for messages published to channels matching the given patterns

	// PUBSUB subcommand [argument [argument ...]]
	// Inspect the state of the Pub/Sub subsystem

	// PUBSUB CHANNELS [pattern]
	// Lists the currently active channels.
	// Array reply: a list of active channels, optionally matching the specified pattern.

	// PUBSUB NUMSUB [channel-1 ... channel-N]
	// Returns the number of subscribers (not counting clients subscribed to patterns) for the specified channels.
	// Array reply:
	//     a list of channels and number of subscribers for every channel.
	//     The format is channel, count, channel, count, ..., so the list is flat.
	//     The order in which the channels are listed is the same as the order of the channels specified in the command call.
	// Note that it is valid to call this command without channels. In this case it will just return an empty list.

	// PUBSUB NUMPAT
	// Returns the number of subscriptions to patterns (that are performed using the PSUBSCRIBE command).
	// Note that this is not just the count of clients subscribed to patterns but the total number of patterns all the clients are subscribed to.
	// Integer reply: the number of patterns all the clients are subscribed to.

	// PUBLISH channel message
	// Post a message to a channel
	// Integer reply: the number of clients that received the message.

	// PUNSUBSCRIBE [pattern [pattern ...]]
	// Stop listening for messages posted to channels matching the given patterns

	// SUBSCRIBE channel [channel ...]
	// Listen for messages published to the given channels

	// UNSUBSCRIBE [channel [channel ...]]
	// Stop listening for messages posted to the given channels
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"os"
	"sync"
)

// Hash bytes
func hashBytes(data []byte, checksums []*Checksum) []*Checksum {
	if checksums == nil {
		checksums = getChosen()
	}
	initHashes(checksums)
	// Do not use goroutines if only one algorith or no data
	if len(checksums) == 1 || len(data) == 0 {
		for i := range checksums {
			if n, err := checksums[i].Write(data); err != nil {
				logger.Print(err)
				return nil
			} else {
				checksums[i].written = int64(n)
				checksums[i].sum = checksums[i].Sum(nil)
			}
		}
		return checksums
	}

	// Use goroutines if more than one algorithm
	var wg sync.WaitGroup

	wg.Add(len(checksums))
	for _, h := range checksums {
		go func(h *Checksum) {
			defer wg.Done()
			if n, err := h.Write(data); err != nil {
				logger.Print(err)
			} else {
				h.written = int64(n)
				h.sum = h.Sum(nil)
			}
		}(h)
	}
	wg.Wait()
	return checksums
}

// Hash file
func hashF(f io.ReadCloser, checksums []*Checksum) []*Checksum {
	if checksums == nil {
		checksums = getChosen()
	}
	initHashes(checksums)
	if len(checksums) == 1 {
		if n, err := io.Copy(checksums[0], f); err != nil {
			logger.Print(err)
			return nil
		} else {
			checksums[0].written = n
			checksums[0].sum = checksums[0].Sum(nil)
		}
		return checksums
	}

	// Use goroutines if more than one algorithm

	var wg sync.WaitGroup
	var writers []io.Writer
	var pipeWriters []*io.PipeWriter

	wg.Add(len(checksums))
	for _, h := range checksums {
		pr, pw := io.Pipe()
		writers = append(writers, pw)
		pipeWriters = append(pipeWriters, pw)
		go func(h *Checksum) {
			defer wg.Done()
			if n, err := io.Copy(h, pr); err != nil {
				logger.Print(err)
			} else {
				h.written = n
				h.sum = h.Sum(nil)
			}
		}(h)
	}

	wg.Add(1)
	go func() {
		defer wg.Done()
		defer func() {
			for _, pw := range pipeWriters {
				pw.Close()
			}
		}()
		// build the multiwriter for all the pipes
		mw := io.MultiWriter(writers...)
		// copy the data into the multiwriter
		if _, err := io.Copy(mw, f); err != nil {
			logger.Print(err)
		}
	}()
	wg.Wait()
	return checksums
}

func hashFile(input *Checksums) *Checksums {
	file := input.file
	f, err := os.Open(file)
	if err != nil {
		stats.unreadable++
		if !opts.ignore {
			logger.Print(err)
		}
		return nil
	}
	defer f.Close()

	info, err := f.Stat()
	if err != nil {
		logger.Print(err)
		return nil
	}
	if info.IsDir() {
		logger.Printf("%s is a directory", file)
		return nil
	}

	var checksums []*Checksum

	if info.Size() < 1<<28 {
		data, err := io.ReadAll(f)
		if err != nil {
			logger.Print(err)
		} else {
			checksums = hashBytes(data, input.checksums)
		}
	}
	if checksums == nil { // io.ReadAll() failed or file is bigger
		checksums = hashF(f, input.checksums)
	}

	if info.Size() > 0 && checksums[0].written != info.Size() {
		logger.Printf("%s: written %d, expected: %d", file, checksums[0].written, info.Size())
		return nil
	} else {
		return &Checksums{
			file:      file,
			checksums: checksums,
		}
	}
}

func hashStdin(f io.ReadCloser) *Checksums {
	if f == nil {
		f = os.Stdin
	}
	return &Checksums{
		file:      "",
		checksums: hashF(f, nil),
	}
}

func hashString(str string) *Checksums {
	checksums := getChosen()
	for _, h := range checksums {
		initHash(h)
		h.Write([]byte(str))
		h.sum = h.Sum(nil)
	}
	return &Checksums{
		file:      `"` + str + `"`,
		checksums: checksums,
	}
}
<commit_msg>Drop the multiwriter approach for hashF()<commit_after>package main

import (
	"io"
	"os"
	"sync"
)

// Hash bytes
func hashBytes(data []byte, checksums []*Checksum) []*Checksum {
	if checksums == nil {
		checksums = getChosen()
	}
	initHashes(checksums)
	// Do not use goroutines if only one algorith or no data
	if len(checksums) == 1 || len(data) == 0 {  // TODO: Check best len(data)
		for i := range checksums {
			if n, err := checksums[i].Write(data); err != nil {
				logger.Print(err)
				return nil
			} else {
				checksums[i].written = int64(n)
				checksums[i].sum = checksums[i].Sum(nil)
			}
		}
		return checksums
	}

	// Use goroutines if more than one algorithm
	var wg sync.WaitGroup

	wg.Add(len(checksums))
	for _, h := range checksums {
		go func(h *Checksum) {
			defer wg.Done()
			if n, err := h.Write(data); err != nil {
				logger.Print(err)
			} else {
				h.written = int64(n)
				h.sum = h.Sum(nil)
			}
		}(h)
	}
	wg.Wait()
	return checksums
}

// Hash file
func hashF(f io.ReadCloser, checksums []*Checksum) []*Checksum {
	if checksums == nil {
		checksums = getChosen()
	}
	initHashes(checksums)
	if len(checksums) == 1 {
		if n, err := io.Copy(checksums[0], f); err != nil {
			logger.Print(err)
			return nil
		} else {
			checksums[0].written = n
			checksums[0].sum = checksums[0].Sum(nil)
		}
		return checksums
	}

	// Use goroutines if more than one algorithm

	var wg sync.WaitGroup
	wg.Add(len(checksums))

	dataChans := make([]chan []byte, len(checksums))

	for i, h := range checksums {
		dataChans[i] = make(chan []byte)
		go func(h *Checksum, dataChan <-chan []byte) {
			defer wg.Done()
			for data := range dataChan {
				n, err := h.Write(data)
				if err != nil {
					logger.Print(err)
				}
				h.written += int64(n)
			}
			h.sum = h.Sum(nil)
		}(h, dataChans[i])
	}

	wg.Add(1)
	go func() {
		defer wg.Done()
		b := make([]byte, 65536)  // TODO: Check best buffer size
		for {
			n, err := f.Read(b)
			if err != nil {
				if err != io.EOF {
					logger.Print(err)
				}
				break
			}
			for i := range dataChans {
				dataChans[i] <- b[:n]
			}
		}
		for i := range dataChans {
			close(dataChans[i])
		}
	}()
	wg.Wait()
	return checksums
}

func hashFile(input *Checksums) *Checksums {
	file := input.file
	f, err := os.Open(file)
	if err != nil {
		stats.unreadable++
		if !opts.ignore {
			logger.Print(err)
		}
		return nil
	}
	defer f.Close()

	info, err := f.Stat()
	if err != nil {
		logger.Print(err)
		return nil
	}
	if info.IsDir() {
		logger.Printf("%s is a directory", file)
		return nil
	}

	var checksums []*Checksum

	// If size is < 256M use io.ReadAll()
	if info.Size() < 1<<28 {  // TODO: Check best size
		data, err := io.ReadAll(f)
		if err != nil {
			logger.Print(err)
		} else {
			checksums = hashBytes(data, input.checksums)
		}
	}
	if checksums == nil { // io.ReadAll() failed or file is bigger
		checksums = hashF(f, input.checksums)
	}

	if info.Size() > 0 && checksums[0].written != info.Size() {
		logger.Printf("%s: written %d, expected: %d", file, checksums[0].written, info.Size())
		return nil
	} else {
		return &Checksums{
			file:      file,
			checksums: checksums,
		}
	}
}

func hashStdin(f io.ReadCloser) *Checksums {
	if f == nil {
		f = os.Stdin
	}
	return &Checksums{
		file:      "",
		checksums: hashF(f, nil),
	}
}

func hashString(str string) *Checksums {
	checksums := getChosen()
	for _, h := range checksums {
		initHash(h)
		h.Write([]byte(str))
		h.sum = h.Sum(nil)
	}
	return &Checksums{
		file:      `"` + str + `"`,
		checksums: checksums,
	}
}
<|endoftext|>


<commit_before>package registry

import (
	"errors"
	"fmt"
	"path"
	"sort"

	log "github.com/coreos/fleet/Godeps/_workspace/src/github.com/golang/glog"

	"github.com/coreos/fleet/etcd"
	"github.com/coreos/fleet/job"
	"github.com/coreos/fleet/unit"
)

const (
	jobPrefix = "job"
)

// Schedule returns all ScheduledUnits known by fleet, ordered by name
func (r *EtcdRegistry) Schedule() ([]job.ScheduledUnit, error) {
	req := etcd.Get{
		Key:       path.Join(r.keyPrefix, jobPrefix),
		Sorted:    true,
		Recursive: true,
	}

	res, err := r.etcd.Do(&req)
	if err != nil {
		if isKeyNotFound(err) {
			err = nil
		}
		return nil, err
	}

	heartbeats := make(map[string]string)
	uMap := make(map[string]*job.ScheduledUnit)

	for _, dir := range res.Node.Nodes {
		_, name := path.Split(dir.Key)
		u := &job.ScheduledUnit{
			Name:            name,
			TargetMachineID: dirToTargetMachineID(&dir),
		}
		heartbeats[name] = dirToHeartbeat(&dir)
		uMap[name] = u
	}

	states, err := r.statesByMUSKey()
	if err != nil {
		return nil, err
	}

	var sortable sort.StringSlice

	// Determine the JobState of each ScheduledUnit
	for name, su := range uMap {
		sortable = append(sortable, name)
		key := MUSKey{
			machID: su.TargetMachineID,
			name:   name,
		}
		us := states[key]
		js := determineJobState(heartbeats[name], su.TargetMachineID, us)
		su.State = &js
	}
	sortable.Sort()

	units := make([]job.ScheduledUnit, 0, len(sortable))
	for _, name := range sortable {
		units = append(units, *uMap[name])
	}
	return units, nil
}

// Units lists all Units stored in the Registry, ordered by name
func (r *EtcdRegistry) Units() ([]job.Unit, error) {
	return r.units()
}

// units retrieves a list of all units stored in the
// Registry, depending on the provided flag. The list is ordered by unit name.
func (r *EtcdRegistry) units() ([]job.Unit, error) {
	req := etcd.Get{
		Key:       path.Join(r.keyPrefix, jobPrefix),
		Sorted:    true,
		Recursive: true,
	}

	res, err := r.etcd.Do(&req)
	if err != nil {
		if isKeyNotFound(err) {
			err = nil
		}
		return nil, err
	}

	uMap := make(map[string]*job.Unit)
	for _, dir := range res.Node.Nodes {
		u, err := r.dirToUnit(&dir)
		if err != nil {
			log.Errorf("Error distilling Unit from node: %v", err)
			continue
		}
		if u == nil {
			continue
		}
		uMap[u.Name] = u
	}

	var sortable sort.StringSlice
	for name, _ := range uMap {
		sortable = append(sortable, name)
	}
	sortable.Sort()

	units := make([]job.Unit, 0, len(sortable))
	for _, name := range sortable {
		units = append(units, *uMap[name])
	}

	return units, nil
}

// Unit retrieves the Unit by the given name from the Registry. Returns nil if
// no such Unit exists, and any error encountered.
func (r *EtcdRegistry) Unit(name string) (*job.Unit, error) {
	req := etcd.Get{
		Key:       path.Join(r.keyPrefix, jobPrefix, name),
		Recursive: true,
	}

	res, err := r.etcd.Do(&req)
	if err != nil {
		if isKeyNotFound(err) {
			err = nil
		}
		return nil, err
	}

	return r.dirToUnit(res.Node)
}

// dirToUnit takes a Node containing a Job's constituent objects (in child
// nodes) and returns a *job.Unit, or any error encountered
func (r *EtcdRegistry) dirToUnit(dir *etcd.Node) (*job.Unit, error) {
	objKey := path.Join(dir.Key, "object")
	var objNode *etcd.Node
	for _, node := range dir.Nodes {
		node := node
		if node.Key == objKey {
			objNode = &node
		}
	}
	if objNode == nil {
		return nil, nil
	}
	u, err := r.getUnitFromObjectNode(objNode)
	if err != nil {
		return nil, err
	}
	if u == nil {
		return nil, fmt.Errorf("unable to parse Unit in Registry at key %s", objKey)
	}
	tgtstate := dirToTargetState(dir)
	if tgtstate == "" {
		return nil, fmt.Errorf("could not find target-state for Unit(%s)", u.Name)
	}
	ts, err := job.ParseJobState(tgtstate)
	if err != nil {
		return nil, fmt.Errorf("failed to parse Unit(%s) target-state: %v", u.Name, err)
	}
	u.TargetState = ts

	return u, nil
}

// ScheduledUnit retrieves the ScheduledUnit by the given name from the Registry.
// Returns nil if no such ScheduledUnit exists, and any error encountered.
func (r *EtcdRegistry) ScheduledUnit(name string) (*job.ScheduledUnit, error) {
	req := etcd.Get{
		Key:       path.Join(r.keyPrefix, jobPrefix, name),
		Recursive: true,
	}

	res, err := r.etcd.Do(&req)
	if err != nil {
		if isKeyNotFound(err) {
			err = nil
		}
		return nil, err
	}

	su := job.ScheduledUnit{
		Name:            name,
		TargetMachineID: dirToTargetMachineID(res.Node),
	}

	js := determineJobState(
		dirToHeartbeat(res.Node),
		su.TargetMachineID,
		r.getUnitState(name))
	su.State = &js

	return &su, nil
}

func (r *EtcdRegistry) UnscheduleUnit(name, machID string) error {
	req := etcd.Delete{
		Key:           r.jobTargetAgentPath(name),
		PreviousValue: machID,
	}

	_, err := r.etcd.Do(&req)
	if isKeyNotFound(err) {
		err = nil
	}

	return err
}

// getValueInDir takes a *etcd.Node containing a job, and returns the value of
// the given key within that directory (i.e. child node) as a string, or an
// empty string if the child node does not exist
func getValueInDir(dir *etcd.Node, key string) (value string) {
	valPath := path.Join(dir.Key, key)
	for _, node := range dir.Nodes {
		if node.Key == valPath {
			value = node.Value
			break
		}
	}
	return
}

func dirToTargetMachineID(dir *etcd.Node) (tgtMID string) {
	return getValueInDir(dir, "target")
}

func dirToTargetState(dir *etcd.Node) (tgtState string) {
	return getValueInDir(dir, "target-state")
}

func dirToHeartbeat(dir *etcd.Node) (heartbeat string) {
	return getValueInDir(dir, "job-state")
}

// getUnitFromObject takes a *etcd.Node containing a Unit's jobModel, and
// instantiates and returns a representative *job.Unit, transitively fetching the
// associated UnitFile as necessary
func (r *EtcdRegistry) getUnitFromObjectNode(node *etcd.Node) (*job.Unit, error) {
	var err error
	var jm jobModel
	if err = unmarshal(node.Value, &jm); err != nil {
		return nil, err
	}

	var unit *unit.UnitFile

	// New-style Jobs should have a populated UnitHash, and the contents of the Unit are stored separately in the Registry
	if !jm.UnitHash.Empty() {
		unit = r.getUnitByHash(jm.UnitHash)
		if unit == nil {
			log.Warningf("No Unit found in Registry for Job(%s)", jm.Name)
			return nil, nil
		}
	} else {
		// Old-style Jobs had "Payloads" instead of Units, also stored separately in the Registry
		unit, err = r.getUnitFromLegacyPayload(jm.Name)
		if err != nil {
			log.Errorf("Error retrieving legacy payload for Job(%s)", jm.Name)
			return nil, nil
		} else if unit == nil {
			log.Warningf("No Payload found in Registry for Job(%s)", jm.Name)
			return nil, nil
		}

		log.Infof("Migrating legacy Payload(%s)", jm.Name)
		if err := r.storeOrGetUnitFile(*unit); err != nil {
			log.Warningf("Unable to migrate legacy Payload: %v", err)
		}

		jm.UnitHash = unit.Hash()
		log.Infof("Updating Job(%s) with legacy payload Hash(%s)", jm.Name, jm.UnitHash)
		if err := r.updateJobObjectNode(&jm, node.ModifiedIndex); err != nil {
			log.Warningf("Unable to update Job(%s) with legacy payload Hash(%s): %v", jm.Name, jm.UnitHash, err)
		}
	}

	ju := &job.Unit{
		Name: jm.Name,
		Unit: *unit,
	}
	return ju, nil

}

// jobModel is used for serializing and deserializing Jobs stored in the Registry
type jobModel struct {
	Name     string
	UnitHash unit.Hash
}

// DestroyUnit removes a Job object from the repository, along with any legacy
// associated Payload and SignatureSet. It does not yet remove underlying
// Units from the repository.
func (r *EtcdRegistry) DestroyUnit(name string) error {
	req := etcd.Delete{
		Key:       path.Join(r.keyPrefix, jobPrefix, name),
		Recursive: true,
	}

	_, err := r.etcd.Do(&req)
	if err != nil {
		if isKeyNotFound(err) {
			err = errors.New("job does not exist")
		}

		return err
	}

	// TODO(jonboulle): add unit reference counting and actually destroying Units
	r.destroyLegacyPayload(name)
	// TODO(jonboulle): handle errors

	return nil
}

// destroyLegacyPayload removes an old-style Payload from the registry
func (r *EtcdRegistry) destroyLegacyPayload(payloadName string) {
	req := etcd.Delete{
		Key: path.Join(r.keyPrefix, payloadPrefix, payloadName),
	}
	r.etcd.Do(&req)
}

// CreateUnit attempts to store a Unit and its associated unit file in the registry
func (r *EtcdRegistry) CreateUnit(u *job.Unit) (err error) {
	if err := r.storeOrGetUnitFile(u.Unit); err != nil {
		return err
	}

	jm := jobModel{
		Name:     u.Name,
		UnitHash: u.Unit.Hash(),
	}
	json, err := marshal(jm)
	if err != nil {
		return
	}

	req := etcd.Create{
		Key:   path.Join(r.keyPrefix, jobPrefix, u.Name, "object"),
		Value: json,
	}

	_, err = r.etcd.Do(&req)
	if err != nil {
		if isNodeExist(err) {
			err = errors.New("job already exists")
		}
		return
	}

	return r.SetUnitTargetState(u.Name, u.TargetState)
}

func (r *EtcdRegistry) updateJobObjectNode(jm *jobModel, idx uint64) (err error) {
	json, err := marshal(jm)
	if err != nil {
		return
	}

	req := etcd.Set{
		Key:           path.Join(r.keyPrefix, jobPrefix, jm.Name, "object"),
		Value:         json,
		PreviousIndex: idx,
	}

	_, err = r.etcd.Do(&req)
	return
}

func (r *EtcdRegistry) SetUnitTargetState(name string, state job.JobState) error {
	req := etcd.Set{
		Key:   r.jobTargetStatePath(name),
		Value: string(state),
	}
	_, err := r.etcd.Do(&req)
	return err
}

func (r *EtcdRegistry) ScheduleUnit(name string, machID string) error {
	req := etcd.Create{
		Key:   r.jobTargetAgentPath(name),
		Value: machID,
	}
	_, err := r.etcd.Do(&req)
	return err
}

func (r *EtcdRegistry) jobTargetAgentPath(jobName string) string {
	return path.Join(r.keyPrefix, jobPrefix, jobName, "target")
}

func (r *EtcdRegistry) jobTargetStatePath(jobName string) string {
	return path.Join(r.keyPrefix, jobPrefix, jobName, "target-state")
}
<commit_msg>registry: make log message slightly clearer<commit_after>package registry

import (
	"errors"
	"fmt"
	"path"
	"sort"

	log "github.com/coreos/fleet/Godeps/_workspace/src/github.com/golang/glog"

	"github.com/coreos/fleet/etcd"
	"github.com/coreos/fleet/job"
	"github.com/coreos/fleet/unit"
)

const (
	jobPrefix = "job"
)

// Schedule returns all ScheduledUnits known by fleet, ordered by name
func (r *EtcdRegistry) Schedule() ([]job.ScheduledUnit, error) {
	req := etcd.Get{
		Key:       path.Join(r.keyPrefix, jobPrefix),
		Sorted:    true,
		Recursive: true,
	}

	res, err := r.etcd.Do(&req)
	if err != nil {
		if isKeyNotFound(err) {
			err = nil
		}
		return nil, err
	}

	heartbeats := make(map[string]string)
	uMap := make(map[string]*job.ScheduledUnit)

	for _, dir := range res.Node.Nodes {
		_, name := path.Split(dir.Key)
		u := &job.ScheduledUnit{
			Name:            name,
			TargetMachineID: dirToTargetMachineID(&dir),
		}
		heartbeats[name] = dirToHeartbeat(&dir)
		uMap[name] = u
	}

	states, err := r.statesByMUSKey()
	if err != nil {
		return nil, err
	}

	var sortable sort.StringSlice

	// Determine the JobState of each ScheduledUnit
	for name, su := range uMap {
		sortable = append(sortable, name)
		key := MUSKey{
			machID: su.TargetMachineID,
			name:   name,
		}
		us := states[key]
		js := determineJobState(heartbeats[name], su.TargetMachineID, us)
		su.State = &js
	}
	sortable.Sort()

	units := make([]job.ScheduledUnit, 0, len(sortable))
	for _, name := range sortable {
		units = append(units, *uMap[name])
	}
	return units, nil
}

// Units lists all Units stored in the Registry, ordered by name
func (r *EtcdRegistry) Units() ([]job.Unit, error) {
	return r.units()
}

// units retrieves a list of all units stored in the
// Registry, depending on the provided flag. The list is ordered by unit name.
func (r *EtcdRegistry) units() ([]job.Unit, error) {
	req := etcd.Get{
		Key:       path.Join(r.keyPrefix, jobPrefix),
		Sorted:    true,
		Recursive: true,
	}

	res, err := r.etcd.Do(&req)
	if err != nil {
		if isKeyNotFound(err) {
			err = nil
		}
		return nil, err
	}

	uMap := make(map[string]*job.Unit)
	for _, dir := range res.Node.Nodes {
		u, err := r.dirToUnit(&dir)
		if err != nil {
			log.Errorf("Failed to parse Unit from etcd: %v", err)
			continue
		}
		if u == nil {
			continue
		}
		uMap[u.Name] = u
	}

	var sortable sort.StringSlice
	for name, _ := range uMap {
		sortable = append(sortable, name)
	}
	sortable.Sort()

	units := make([]job.Unit, 0, len(sortable))
	for _, name := range sortable {
		units = append(units, *uMap[name])
	}

	return units, nil
}

// Unit retrieves the Unit by the given name from the Registry. Returns nil if
// no such Unit exists, and any error encountered.
func (r *EtcdRegistry) Unit(name string) (*job.Unit, error) {
	req := etcd.Get{
		Key:       path.Join(r.keyPrefix, jobPrefix, name),
		Recursive: true,
	}

	res, err := r.etcd.Do(&req)
	if err != nil {
		if isKeyNotFound(err) {
			err = nil
		}
		return nil, err
	}

	return r.dirToUnit(res.Node)
}

// dirToUnit takes a Node containing a Job's constituent objects (in child
// nodes) and returns a *job.Unit, or any error encountered
func (r *EtcdRegistry) dirToUnit(dir *etcd.Node) (*job.Unit, error) {
	objKey := path.Join(dir.Key, "object")
	var objNode *etcd.Node
	for _, node := range dir.Nodes {
		node := node
		if node.Key == objKey {
			objNode = &node
		}
	}
	if objNode == nil {
		return nil, nil
	}
	u, err := r.getUnitFromObjectNode(objNode)
	if err != nil {
		return nil, err
	}
	if u == nil {
		return nil, fmt.Errorf("unable to parse Unit in Registry at key %s", objKey)
	}
	tgtstate := dirToTargetState(dir)
	if tgtstate == "" {
		return nil, fmt.Errorf("could not find target-state for Unit(%s)", u.Name)
	}
	ts, err := job.ParseJobState(tgtstate)
	if err != nil {
		return nil, fmt.Errorf("failed to parse Unit(%s) target-state: %v", u.Name, err)
	}
	u.TargetState = ts

	return u, nil
}

// ScheduledUnit retrieves the ScheduledUnit by the given name from the Registry.
// Returns nil if no such ScheduledUnit exists, and any error encountered.
func (r *EtcdRegistry) ScheduledUnit(name string) (*job.ScheduledUnit, error) {
	req := etcd.Get{
		Key:       path.Join(r.keyPrefix, jobPrefix, name),
		Recursive: true,
	}

	res, err := r.etcd.Do(&req)
	if err != nil {
		if isKeyNotFound(err) {
			err = nil
		}
		return nil, err
	}

	su := job.ScheduledUnit{
		Name:            name,
		TargetMachineID: dirToTargetMachineID(res.Node),
	}

	js := determineJobState(
		dirToHeartbeat(res.Node),
		su.TargetMachineID,
		r.getUnitState(name))
	su.State = &js

	return &su, nil
}

func (r *EtcdRegistry) UnscheduleUnit(name, machID string) error {
	req := etcd.Delete{
		Key:           r.jobTargetAgentPath(name),
		PreviousValue: machID,
	}

	_, err := r.etcd.Do(&req)
	if isKeyNotFound(err) {
		err = nil
	}

	return err
}

// getValueInDir takes a *etcd.Node containing a job, and returns the value of
// the given key within that directory (i.e. child node) as a string, or an
// empty string if the child node does not exist
func getValueInDir(dir *etcd.Node, key string) (value string) {
	valPath := path.Join(dir.Key, key)
	for _, node := range dir.Nodes {
		if node.Key == valPath {
			value = node.Value
			break
		}
	}
	return
}

func dirToTargetMachineID(dir *etcd.Node) (tgtMID string) {
	return getValueInDir(dir, "target")
}

func dirToTargetState(dir *etcd.Node) (tgtState string) {
	return getValueInDir(dir, "target-state")
}

func dirToHeartbeat(dir *etcd.Node) (heartbeat string) {
	return getValueInDir(dir, "job-state")
}

// getUnitFromObject takes a *etcd.Node containing a Unit's jobModel, and
// instantiates and returns a representative *job.Unit, transitively fetching the
// associated UnitFile as necessary
func (r *EtcdRegistry) getUnitFromObjectNode(node *etcd.Node) (*job.Unit, error) {
	var err error
	var jm jobModel
	if err = unmarshal(node.Value, &jm); err != nil {
		return nil, err
	}

	var unit *unit.UnitFile

	// New-style Jobs should have a populated UnitHash, and the contents of the Unit are stored separately in the Registry
	if !jm.UnitHash.Empty() {
		unit = r.getUnitByHash(jm.UnitHash)
		if unit == nil {
			log.Warningf("No Unit found in Registry for Job(%s)", jm.Name)
			return nil, nil
		}
	} else {
		// Old-style Jobs had "Payloads" instead of Units, also stored separately in the Registry
		unit, err = r.getUnitFromLegacyPayload(jm.Name)
		if err != nil {
			log.Errorf("Error retrieving legacy payload for Job(%s)", jm.Name)
			return nil, nil
		} else if unit == nil {
			log.Warningf("No Payload found in Registry for Job(%s)", jm.Name)
			return nil, nil
		}

		log.Infof("Migrating legacy Payload(%s)", jm.Name)
		if err := r.storeOrGetUnitFile(*unit); err != nil {
			log.Warningf("Unable to migrate legacy Payload: %v", err)
		}

		jm.UnitHash = unit.Hash()
		log.Infof("Updating Job(%s) with legacy payload Hash(%s)", jm.Name, jm.UnitHash)
		if err := r.updateJobObjectNode(&jm, node.ModifiedIndex); err != nil {
			log.Warningf("Unable to update Job(%s) with legacy payload Hash(%s): %v", jm.Name, jm.UnitHash, err)
		}
	}

	ju := &job.Unit{
		Name: jm.Name,
		Unit: *unit,
	}
	return ju, nil

}

// jobModel is used for serializing and deserializing Jobs stored in the Registry
type jobModel struct {
	Name     string
	UnitHash unit.Hash
}

// DestroyUnit removes a Job object from the repository, along with any legacy
// associated Payload and SignatureSet. It does not yet remove underlying
// Units from the repository.
func (r *EtcdRegistry) DestroyUnit(name string) error {
	req := etcd.Delete{
		Key:       path.Join(r.keyPrefix, jobPrefix, name),
		Recursive: true,
	}

	_, err := r.etcd.Do(&req)
	if err != nil {
		if isKeyNotFound(err) {
			err = errors.New("job does not exist")
		}

		return err
	}

	// TODO(jonboulle): add unit reference counting and actually destroying Units
	r.destroyLegacyPayload(name)
	// TODO(jonboulle): handle errors

	return nil
}

// destroyLegacyPayload removes an old-style Payload from the registry
func (r *EtcdRegistry) destroyLegacyPayload(payloadName string) {
	req := etcd.Delete{
		Key: path.Join(r.keyPrefix, payloadPrefix, payloadName),
	}
	r.etcd.Do(&req)
}

// CreateUnit attempts to store a Unit and its associated unit file in the registry
func (r *EtcdRegistry) CreateUnit(u *job.Unit) (err error) {
	if err := r.storeOrGetUnitFile(u.Unit); err != nil {
		return err
	}

	jm := jobModel{
		Name:     u.Name,
		UnitHash: u.Unit.Hash(),
	}
	json, err := marshal(jm)
	if err != nil {
		return
	}

	req := etcd.Create{
		Key:   path.Join(r.keyPrefix, jobPrefix, u.Name, "object"),
		Value: json,
	}

	_, err = r.etcd.Do(&req)
	if err != nil {
		if isNodeExist(err) {
			err = errors.New("job already exists")
		}
		return
	}

	return r.SetUnitTargetState(u.Name, u.TargetState)
}

func (r *EtcdRegistry) updateJobObjectNode(jm *jobModel, idx uint64) (err error) {
	json, err := marshal(jm)
	if err != nil {
		return
	}

	req := etcd.Set{
		Key:           path.Join(r.keyPrefix, jobPrefix, jm.Name, "object"),
		Value:         json,
		PreviousIndex: idx,
	}

	_, err = r.etcd.Do(&req)
	return
}

func (r *EtcdRegistry) SetUnitTargetState(name string, state job.JobState) error {
	req := etcd.Set{
		Key:   r.jobTargetStatePath(name),
		Value: string(state),
	}
	_, err := r.etcd.Do(&req)
	return err
}

func (r *EtcdRegistry) ScheduleUnit(name string, machID string) error {
	req := etcd.Create{
		Key:   r.jobTargetAgentPath(name),
		Value: machID,
	}
	_, err := r.etcd.Do(&req)
	return err
}

func (r *EtcdRegistry) jobTargetAgentPath(jobName string) string {
	return path.Join(r.keyPrefix, jobPrefix, jobName, "target")
}

func (r *EtcdRegistry) jobTargetStatePath(jobName string) string {
	return path.Join(r.keyPrefix, jobPrefix, jobName, "target-state")
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"os"
	"text/template"

	"github.com/flynn/flynn-controller/client"
)

var cmdHelp = &Command{
	NoClient: true,
	Usage:    "help [topic]",
	Long:     `Help shows usage for a command or other topic.`,
}

func init() {
	cmdHelp.Run = runHelp // break init loop
}

func runHelp(cmd *Command, args []string, client *controller.Client) error {
	if len(args) == 0 {
		printUsage()
		return nil // not os.Exit(2); success
	}
	if len(args) != 1 {
		return errors.New("too many arguments")
	}

	for _, cmd := range commands {
		if cmd.Name() == args[0] {
			cmd.printUsage(false)
			return nil
		}
	}

	fmt.Fprintf(os.Stderr, "Unknown help topic: %q. Run 'flynn help'.\n", args[0])
	os.Exit(2)
	return nil
}

var cmdVersion = &Command{
	Run:      runVersion,
	NoClient: true,
	Usage:    "version",
	Short:    "show flynn version",
	Long:     `Version shows the flynn client version string.`,
}

func runVersion(cmd *Command, args []string, client *controller.Client) error {
	fmt.Println(Version)
	return nil
}

var usageTemplate = template.Must(template.New("usage").Parse(`
Usage: flynn [-a app] [command] [options] [arguments]

Commands:
{{range .Commands}}{{if .Runnable}}{{if .List}}
    {{.Name | printf (print "%-" $.MaxCommandWidth "s")}}  {{.Short}}{{end}}{{end}}{{end}}

Run 'flynn help [command]' for details.`[1:]))

func printUsage() {
	data := &struct {
		Commands        []*Command
		MaxCommandWidth int
	}{
		commands,
		0,
	}

	for _, cmd := range commands {
		if len(cmd.Name()) > data.MaxCommandWidth {
			data.MaxCommandWidth = len(cmd.Name())
		}
	}

	usageTemplate.Execute(os.Stdout, data)
}

func usage() {
	printUsage()
	os.Exit(2)
}
<commit_msg>cli: Add trailing newline to help<commit_after>package main

import (
	"errors"
	"fmt"
	"os"
	"text/template"

	"github.com/flynn/flynn-controller/client"
)

var cmdHelp = &Command{
	NoClient: true,
	Usage:    "help [topic]",
	Long:     `Help shows usage for a command or other topic.`,
}

func init() {
	cmdHelp.Run = runHelp // break init loop
}

func runHelp(cmd *Command, args []string, client *controller.Client) error {
	if len(args) == 0 {
		printUsage()
		return nil // not os.Exit(2); success
	}
	if len(args) != 1 {
		return errors.New("too many arguments")
	}

	for _, cmd := range commands {
		if cmd.Name() == args[0] {
			cmd.printUsage(false)
			return nil
		}
	}

	fmt.Fprintf(os.Stderr, "Unknown help topic: %q. Run 'flynn help'.\n", args[0])
	os.Exit(2)
	return nil
}

var cmdVersion = &Command{
	Run:      runVersion,
	NoClient: true,
	Usage:    "version",
	Short:    "show flynn version",
	Long:     `Version shows the flynn client version string.`,
}

func runVersion(cmd *Command, args []string, client *controller.Client) error {
	fmt.Println(Version)
	return nil
}

var usageTemplate = template.Must(template.New("usage").Parse(`
Usage: flynn [-a app] [command] [options] [arguments]

Commands:
{{range .Commands}}{{if .Runnable}}{{if .List}}
    {{.Name | printf (print "%-" $.MaxCommandWidth "s")}}  {{.Short}}{{end}}{{end}}{{end}}

Run 'flynn help [command]' for details.
`[1:]))

func printUsage() {
	data := &struct {
		Commands        []*Command
		MaxCommandWidth int
	}{
		commands,
		0,
	}

	for _, cmd := range commands {
		if len(cmd.Name()) > data.MaxCommandWidth {
			data.MaxCommandWidth = len(cmd.Name())
		}
	}

	usageTemplate.Execute(os.Stdout, data)
}

func usage() {
	printUsage()
	os.Exit(2)
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Che Wei, Lin
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tinynet

import (
	"net"
	"path/filepath"

	"github.com/containernetworking/plugins/pkg/ip"
	"github.com/containernetworking/plugins/pkg/ns"
	log "github.com/sirupsen/logrus"
)

// Host is a host instance
type Host struct {
	NodeType string
	Name     string
	VethName string
	IfName   string
	Sandbox  string
	IP       string
	MAC      string
}

// NewHost for creating a network namespace
func NewHost(name string) (*Host, error) {
	h := new(Host)
	h.NodeType = "Host"
	h.Name = name

	// Create a network namespace
	targetNs, err := ns.NewNS()
	if err != nil {
		log.Fatal("failed to open netns: ", err)
		return nil, err
	}
	// log.Info("netns mouted into the host: ", targetNs.Path())
	log.Infof("Adding a host: %s, namespace: %s", h.Name, filepath.Base(targetNs.Path()))
	h.Sandbox = targetNs.Path()
	return h, nil
}

// NewContainer for creating a docker container
func NewContainer(name string, imageRef string) (*Host, error) {
	h := new(Host)
	h.NodeType = "Host"
	h.Name = name
	_, sandboxKey, err := ensureDocker(imageRef)
	if err != nil {
		log.Fatal("failed to start container: ", err)
		return nil, err
	}
	h.Sandbox = sandboxKey
	log.Info("netns mouted into the host: ", h.Sandbox)
	return h, nil
}

func (h *Host) setupVeth(ifName string, mtu int) (*Host, error) {
	// Get network namespace object
	netns, err := ns.GetNS(h.Sandbox)
	if err != nil {
		log.Fatal("failed to open netns: ", err)
	}
	defer netns.Close()

	// attach network namespace and setup veth pair
	err = netns.Do(func(hostNS ns.NetNS) error {
		// create the veth pair in the container and move host end into host netns
		hostVeth, containerVeth, err := ip.SetupVeth(ifName, mtu, hostNS)
		if err != nil {
			return err
		}
		// Host interface name
		h.IfName = containerVeth.Name

		// h.mac = containerVeth.HardwareAddr.String()

		// Host veth name
		h.VethName = hostVeth.Name

		// ip link set lo up
		_, err = ifaceUp("lo")
		if err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	return h, nil
}

func (h *Host) setIfaceIP(address string) error {
	// Get network namespace object
	netns, err := ns.GetNS(h.Sandbox)
	if err != nil {
		log.Fatal("failed to open netns: ", err)
	}
	defer netns.Close()

	ipv4Addr, _, err := net.ParseCIDR(address)
	if err != nil {
		return err
	}
	h.IP = ipv4Addr.String()
	err = netns.Do(func(hostNS ns.NetNS) error {
		if err := setIP(h.IfName, address); err != nil {
			return err
		}
		// ip link set ifName up
		_, err := ifaceUp(h.IfName)
		if err != nil {
			return err
		}
		// get Host mac address
		hostIface, err := net.InterfaceByName(h.IfName)
		if err != nil {
			return err
		}
		h.MAC = hostIface.HardwareAddr.String()
		return nil
	})
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>enable host can use docker container as a container<commit_after>// Copyright (c) 2017 Che Wei, Lin
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tinynet

import (
	"net"
	"path/filepath"

	"github.com/containernetworking/plugins/pkg/ip"
	"github.com/containernetworking/plugins/pkg/ns"
	log "github.com/sirupsen/logrus"
)

// Host is a host instance
type Host struct {
	NodeType string
	Name     string
	VethName string
	IfName   string
	Sandbox  string
	IP       string
	MAC      string
}

// NewHost for creating a network namespace
func NewHost(name string, docker bool) (*Host, error) {
	h := new(Host)
	h.NodeType = "Host"

	if docker {
		name, sandbox := initDocker("busybox")
		h.Name = name
		h.Sandbox = sandbox
		log.Info("netns mouted into the host: ", h.Sandbox)
		return h, nil
	}

	h.Name = name

	// Create a network namespace
	targetNs, err := ns.NewNS()
	if err != nil {
		log.Fatal("failed to open netns: ", err)
		return nil, err
	}
	// log.Info("netns mouted into the host: ", targetNs.Path())
	log.Infof("Adding a host: %s, namespace: %s", h.Name, filepath.Base(targetNs.Path()))
	h.Sandbox = targetNs.Path()
	return h, nil
}

// NewContainer for creating a docker container
func NewContainer(name string, imageRef string) (*Host, error) {
	h := new(Host)
	h.NodeType = "Host"
	h.Name = name
	_, sandboxKey, err := ensureDocker(imageRef)
	if err != nil {
		log.Fatal("failed to start container: ", err)
		return nil, err
	}
	h.Sandbox = sandboxKey
	log.Info("netns mouted into the host: ", h.Sandbox)
	return h, nil
}

func (h *Host) setupVeth(ifName string, mtu int) (*Host, error) {
	// Get network namespace object
	netns, err := ns.GetNS(h.Sandbox)
	if err != nil {
		log.Fatal("failed to open netns: ", err)
	}
	defer netns.Close()

	// attach network namespace and setup veth pair
	err = netns.Do(func(hostNS ns.NetNS) error {
		// create the veth pair in the container and move host end into host netns
		hostVeth, containerVeth, err := ip.SetupVeth(ifName, mtu, hostNS)
		if err != nil {
			return err
		}
		// Host interface name
		h.IfName = containerVeth.Name

		// h.mac = containerVeth.HardwareAddr.String()

		// Host veth name
		h.VethName = hostVeth.Name

		// ip link set lo up
		_, err = ifaceUp("lo")
		if err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	return h, nil
}

func (h *Host) setIfaceIP(address string) error {
	// Get network namespace object
	netns, err := ns.GetNS(h.Sandbox)
	if err != nil {
		log.Fatal("failed to open netns: ", err)
	}
	defer netns.Close()

	ipv4Addr, _, err := net.ParseCIDR(address)
	if err != nil {
		return err
	}
	h.IP = ipv4Addr.String()
	err = netns.Do(func(hostNS ns.NetNS) error {
		if err := setIP(h.IfName, address); err != nil {
			return err
		}
		// ip link set ifName up
		_, err := ifaceUp(h.IfName)
		if err != nil {
			return err
		}
		// get Host mac address
		hostIface, err := net.InterfaceByName(h.IfName)
		if err != nil {
			return err
		}
		h.MAC = hostIface.HardwareAddr.String()
		return nil
	})
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/pachyderm/pachyderm/src/server/cmd/pachctl/cmd"
	"github.com/pachyderm/pachyderm/src/server/pkg/cmdutil"

	"github.com/spf13/cobra"
	"github.com/spf13/cobra/doc"
)

type appEnv struct{}

func main() {
	cmdutil.Main(do, &appEnv{})
}

// Walk the command tree, wrap any examples in a block-quote with shell highlighting
func recursiveBlockQuoteExamples(parent *cobra.Command) {
	if parent.Example != "" {
		parent.Example = fmt.Sprintf("```sh\n%s\n```", parent.Example)
	}

	for _, cmd := range parent.Commands() {
		recursiveBlockQuoteExamples(cmd)
	}
}

func do(appEnvObj interface{}) error {
	// Set 'os.Args[0]' so that examples use the expected command name
	os.Args[0] = "pachctl"

	rootCmd := cmd.PachctlCmd()

	recursiveBlockQuoteExamples(rootCmd)
	return doc.GenMarkdownTree(rootCmd, "./doc/pachctl/")
}
<commit_msg>remove attempt to use shell syntax highlighting on doc examples as this breaks stuff (#3888)<commit_after>package main

import (
	"os"

	"github.com/pachyderm/pachyderm/src/server/cmd/pachctl/cmd"
	"github.com/pachyderm/pachyderm/src/server/pkg/cmdutil"

	"github.com/spf13/cobra/doc"
)

type appEnv struct{}

func main() {
	cmdutil.Main(do, &appEnv{})
}

func do(appEnvObj interface{}) error {
	// Set 'os.Args[0]' so that examples use the expected command name
	os.Args[0] = "pachctl"

	rootCmd := cmd.PachctlCmd()

	return doc.GenMarkdownTree(rootCmd, "./doc/pachctl/")
}
<|endoftext|>


<commit_before>// chris 072115

package rebnf

import (
	"bytes"
	"fmt"
	"html"

	"path/filepath"
)

// Markers around EBNF sections in HTML files.
var (
	ebnfopen  = []byte(`<pre class="ebnf">`)
	ebnfclose = []byte(`</pre>`)
)

func preserveNewlines(buf *bytes.Buffer, excl []byte) {
	// write as many newlines as found in the excluded text
	// to maintain correct line numbers in error messages
	for _, ch := range excl {
		if ch == '\n' {
			buf.WriteByte('\n')
		}
	}
}

// ExtractEBNF extracts the EBNF text from a file in which grammar
// productions are grouped in boxes demarcated by the following HTML
// elements.
//
//

//

//
// An example of such a file is the Go Programming Language
// Specification HTML page.
//
// ExtractEBNF was itself initially extracted from
// golang.org/x/exp/ebnflint.  Unfortunately, it's not exported there,
// so we duplicate it here and export it.
func ExtractEBNF(src []byte) []byte {
	buf := new(bytes.Buffer)

	for {
		// i = beginning of EBNF text
		i := bytes.Index(src, ebnfopen)
		if i < 0 {
			break // no EBNF found - we are done
		}
		i += len(ebnfopen)

		preserveNewlines(buf, src[:i])

		// j = end of EBNF text (or end of source)
		j := bytes.Index(src[i:], ebnfclose) // close marker
		if j < 0 {
			j = len(src) - i
		}
		j += i

		// copy EBNF text
		buf.Write(src[i:j])

		// advance
		src = src[j:]
	}

	return buf.Bytes()
}

// StripTag strips out the named tag from a slice of bytes.  The tag is
// expected to open with a string of the following form.
//
//	<tagname ...>
//
// Where ... can be anything.  If any newlines appear in the ..., they
// will be preserved.
//
// The tag is expected to close with a string of the following form.
//
//	</tagname>
//
func StripTag(tagname string, src []byte) []byte {
	buf := new(bytes.Buffer)

	var (
		openbegin = []byte(fmt.Sprintf("<%s ", tagname))
		openend   = []byte(`>`)
		close     = []byte(fmt.Sprintf("</%s>", tagname))
	)

	for {
		// i = beginning of the opening of the tag
		i := bytes.Index(src, openbegin)
		if i < 0 {
			// no tag found - copy rest, and we are done
			buf.Write(src)
			break
		}

		// j = end of the opening of the tag
		j := bytes.Index(src[i:], openend)
		if j < 0 {
			// tag doesn't close - we are done
			break
		}
		j += i // Since we sliced src after the beginning of the open.
		j += len(openend)

		preserveNewlines(buf, src[i:j])

		// elide opening tag
		buf.Write(src[:i])
		src = src[j:]

		// k = end of tag (or end of source)
		k := bytes.Index(src, close) // close marker
		if k < 0 {
			// Great, we want everything then, and we're
			// done.
			buf.Write(src)
			break
		}

		// Copy up to the close.
		buf.Write(src[:k])

		m := k + len(close)

		preserveNewlines(buf, src[k:m])

		// advance
		src = src[m:]
	}

	return buf.Bytes()
}

// CheckRead checks the input filename to see if it ends in ".html" or,
// if not, then checks if an opening token is present.  If either is
// true, then the source byte array is treated as an HTML document and
// EBNF text is extracted according to the following opening and closing
// tokens.
//
//

//

//
// In this case, the extracted byte slice is returned instead of the
// original.  Otherwise, the original byte slice is returned.
//
// The logic in CheckRead is extracted from golang.org/x/exp/ebnflint.
// Unfortunately, it's not exported there, so we duplicate it here and
// export it.
func CheckRead(filename string, src []byte) []byte {
	if filepath.Ext(filename) == ".html" || bytes.Index(src, ebnfopen) >= 0 {
		src = ExtractEBNF(src)

		// The Go Programming Language Specification HTML page
		// wraps production names with links.
		src = StripTag("a", src)

		// We also want to unescape HTML escape sequences.
		src = []byte(html.UnescapeString(string(src)))
	}
	return src
}
<commit_msg>Removes unnecessary newline preservation.<commit_after>// chris 072115

package rebnf

import (
	"bytes"
	"fmt"
	"html"

	"path/filepath"
)

// Markers around EBNF sections in HTML files.
var (
	ebnfopen  = []byte(`<pre class="ebnf">`)
	ebnfclose = []byte(`</pre>`)
)

func preserveNewlines(buf *bytes.Buffer, excl []byte) {
	// write as many newlines as found in the excluded text
	// to maintain correct line numbers in error messages
	for _, ch := range excl {
		if ch == '\n' {
			buf.WriteByte('\n')
		}
	}
}

// ExtractEBNF extracts the EBNF text from a file in which grammar
// productions are grouped in boxes demarcated by the following HTML
// elements.
//
//

//

//
// An example of such a file is the Go Programming Language
// Specification HTML page.
//
// ExtractEBNF was itself initially extracted from
// golang.org/x/exp/ebnflint.  Unfortunately, it's not exported there,
// so we duplicate it here and export it.
func ExtractEBNF(src []byte) []byte {
	buf := new(bytes.Buffer)

	for {
		// i = beginning of EBNF text
		i := bytes.Index(src, ebnfopen)
		if i < 0 {
			break // no EBNF found - we are done
		}
		i += len(ebnfopen)

		preserveNewlines(buf, src[:i])

		// j = end of EBNF text (or end of source)
		j := bytes.Index(src[i:], ebnfclose) // close marker
		if j < 0 {
			j = len(src) - i
		}
		j += i

		// copy EBNF text
		buf.Write(src[i:j])

		// advance
		src = src[j:]
	}

	return buf.Bytes()
}

// StripTag strips out the named tag from a slice of bytes.  The tag is
// expected to open with a string of the following form.
//
//	<tagname ...>
//
// Where ... can be anything.  If any newlines appear in the ..., they
// will be preserved.
//
// The tag is expected to close with a string of the following form.
//
//	</tagname>
//
func StripTag(tagname string, src []byte) []byte {
	buf := new(bytes.Buffer)

	var (
		openbegin = []byte(fmt.Sprintf("<%s ", tagname))
		openend   = []byte(`>`)
		close     = []byte(fmt.Sprintf("</%s>", tagname))
	)

	for {
		// i = beginning of the opening of the tag
		i := bytes.Index(src, openbegin)
		if i < 0 {
			// no tag found - copy rest, and we are done
			buf.Write(src)
			break
		}

		// j = end of the opening of the tag
		j := bytes.Index(src[i:], openend)
		if j < 0 {
			// tag doesn't close - we are done
			break
		}
		j += i // Since we sliced src after the beginning of the open.
		j += len(openend)

		preserveNewlines(buf, src[i:j])

		// elide opening tag
		buf.Write(src[:i])
		src = src[j:]

		// k = end of tag (or end of source)
		k := bytes.Index(src, close) // close marker
		if k < 0 {
			// Great, we want everything then, and we're
			// done.
			buf.Write(src)
			break
		}

		// Copy up to the close.
		buf.Write(src[:k])

		k += len(close)

		// advance
		src = src[k:]
	}

	return buf.Bytes()
}

// CheckRead checks the input filename to see if it ends in ".html" or,
// if not, then checks if an opening token is present.  If either is
// true, then the source byte array is treated as an HTML document and
// EBNF text is extracted according to the following opening and closing
// tokens.
//
//

//

//
// In this case, the extracted byte slice is returned instead of the
// original.  Otherwise, the original byte slice is returned.
//
// The logic in CheckRead is extracted from golang.org/x/exp/ebnflint.
// Unfortunately, it's not exported there, so we duplicate it here and
// export it.
func CheckRead(filename string, src []byte) []byte {
	if filepath.Ext(filename) == ".html" || bytes.Index(src, ebnfopen) >= 0 {
		src = ExtractEBNF(src)

		// The Go Programming Language Specification HTML page
		// wraps production names with links.
		src = StripTag("a", src)

		// We also want to unescape HTML escape sequences.
		src = []byte(html.UnescapeString(string(src)))
	}
	return src
}
<|endoftext|>


<commit_before>package proxy

import (
	"bufio"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
)

// msgReader is a helper that reads values from a PostgreSQL message.
type msgReader struct {
	reader            *bufio.Reader
	buf               [128]byte
	msgBytesRemaining int32
	err               error
}

// Err returns any error that the msgReader has experienced
func (r *msgReader) Err() error {
	return r.err
}

// fatal tells r that a Fatal error has occurred
func (r *msgReader) fatal(err error) {
	r.err = err
}

// rxMsg reads the type and size of the next message.
func (r *msgReader) rxMsg() (t byte, err error) {
	if r.err != nil {
		return 0, err
	}
	if r.msgBytesRemaining > 0 {
		io.CopyN(ioutil.Discard, r.reader, int64(r.msgBytesRemaining))
	}
	b := r.buf[0:5]
	fmt.Println("1")
	fmt.Println(len(b))
	fmt.Println("2")
	min := len(b)
	var n int
	for n < min && err == nil {
		var nn int
		nn, err = r.reader.Read(b[n:])
		n += nn
		fmt.Println("3")
	}
	fmt.Println("4")
	if n >= min {
		err = nil
	} else if n > 0 && err == io.EOF {
		err = io.ErrUnexpectedEOF
	}
	// a, err := io.ReadAtLeast(r.reader, b, len(b))
	// ReadFull(r.reader, b)
	fmt.Printf("a: %#v\n", n)
	fmt.Println(err)
	if r.err != nil {
		fmt.Println(err)
	}
	r.msgBytesRemaining = int32(binary.BigEndian.Uint32(b[1:])) - 4
	return b[0], err
}

func (r *msgReader) readByte() byte {
	if r.err != nil {
		return 0
	}

	r.msgBytesRemaining -= 1
	if r.msgBytesRemaining < 0 {
		r.fatal(errors.New("read past end of message"))
		return 0
	}

	b, err := r.reader.ReadByte()
	if err != nil {
		r.fatal(err)
		return 0
	}

	return b
}

func (r *msgReader) readInt16() int16 {
	if r.err != nil {
		return 0
	}

	r.msgBytesRemaining -= 2
	if r.msgBytesRemaining < 0 {
		r.fatal(errors.New("read past end of message"))
		return 0
	}

	b := r.buf[0:2]
	_, err := io.ReadFull(r.reader, b)
	if err != nil {
		r.fatal(err)
		return 0
	}

	return int16(binary.BigEndian.Uint16(b))
}

func (r *msgReader) readInt32() int32 {
	if r.err != nil {
		return 0
	}

	r.msgBytesRemaining -= 4
	if r.msgBytesRemaining < 0 {
		r.fatal(errors.New("read past end of message"))
		return 0
	}

	b := r.buf[0:4]
	_, err := io.ReadFull(r.reader, b)
	if err != nil {
		r.fatal(err)
		return 0
	}

	return int32(binary.BigEndian.Uint32(b))
}

func (r *msgReader) readInt64() int64 {
	if r.err != nil {
		return 0
	}

	r.msgBytesRemaining -= 8
	if r.msgBytesRemaining < 0 {
		r.fatal(errors.New("read past end of message"))
		return 0
	}

	b := r.buf[0:8]
	_, err := io.ReadFull(r.reader, b)
	if err != nil {
		r.fatal(err)
		return 0
	}

	return int64(binary.BigEndian.Uint64(b))
}

func (r *msgReader) readOid() Oid {
	return Oid(r.readInt32())
}

// readCString reads a null terminated string
func (r *msgReader) readCString() string {
	if r.err != nil {
		return ""
	}

	b, err := r.reader.ReadBytes(0)
	if err != nil {
		r.fatal(err)
		return ""
	}

	r.msgBytesRemaining -= int32(len(b))
	if r.msgBytesRemaining < 0 {
		r.fatal(errors.New("read past end of message"))
		return ""
	}

	return string(b[0 : len(b)-1])
}

// readString reads count bytes and returns as string
func (r *msgReader) readString(count int32) string {
	if r.err != nil {
		return ""
	}

	r.msgBytesRemaining -= count
	if r.msgBytesRemaining < 0 {
		r.fatal(errors.New("read past end of message"))
		return ""
	}

	var b []byte
	if count <= int32(len(r.buf)) {
		b = r.buf[0:int(count)]
	} else {
		b = make([]byte, int(count))
	}

	_, err := io.ReadFull(r.reader, b)
	if err != nil {
		r.fatal(err)
		return ""
	}

	return string(b)
}

// readBytes reads count bytes and returns as []byte
func (r *msgReader) readBytes(count int32) []byte {
	if r.err != nil {
		return nil
	}

	r.msgBytesRemaining -= count
	if r.msgBytesRemaining < 0 {
		r.fatal(errors.New("read past end of message"))
		return nil
	}

	b := make([]byte, int(count))

	_, err := io.ReadFull(r.reader, b)
	if err != nil {
		r.fatal(err)
		return nil
	}

	return b
}
<commit_msg>Update<commit_after>package proxy

import (
	"bufio"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
)

// msgReader is a helper that reads values from a PostgreSQL message.
type msgReader struct {
	reader            *bufio.Reader
	buf               [128]byte
	msgBytesRemaining int32
	err               error
}

// Err returns any error that the msgReader has experienced
func (r *msgReader) Err() error {
	return r.err
}

// fatal tells r that a Fatal error has occurred
func (r *msgReader) fatal(err error) {
	r.err = err
}

// rxMsg reads the type and size of the next message.
func (r *msgReader) rxMsg() (t byte, err error) {
	if r.err != nil {
		return 0, err
	}
	if r.msgBytesRemaining > 0 {
		io.CopyN(ioutil.Discard, r.reader, int64(r.msgBytesRemaining))
	}
	b := r.buf[0:5]
	fmt.Println("1")
	min := len(b)
	fmt.Println(len(b))
	fmt.Println("2")
	var n int
	for n < min && err == nil {
		var nn int
		nn, err = r.reader.Read(b[n:])
		n += nn
		fmt.Println("3")
	}
	fmt.Println("4")
	if n >= min {
		err = nil
	} else if n > 0 && err == io.EOF {
		err = io.ErrUnexpectedEOF
	}
	// a, err := io.ReadAtLeast(r.reader, b, len(b))
	// ReadFull(r.reader, b)
	fmt.Printf("a: %#v\n", n)
	fmt.Println(err)
	if r.err != nil {
		fmt.Println(err)
	}
	r.msgBytesRemaining = int32(binary.BigEndian.Uint32(b[1:])) - 4
	return b[0], err
}

func (r *msgReader) readByte() byte {
	if r.err != nil {
		return 0
	}

	r.msgBytesRemaining -= 1
	if r.msgBytesRemaining < 0 {
		r.fatal(errors.New("read past end of message"))
		return 0
	}

	b, err := r.reader.ReadByte()
	if err != nil {
		r.fatal(err)
		return 0
	}

	return b
}

func (r *msgReader) readInt16() int16 {
	if r.err != nil {
		return 0
	}

	r.msgBytesRemaining -= 2
	if r.msgBytesRemaining < 0 {
		r.fatal(errors.New("read past end of message"))
		return 0
	}

	b := r.buf[0:2]
	_, err := io.ReadFull(r.reader, b)
	if err != nil {
		r.fatal(err)
		return 0
	}

	return int16(binary.BigEndian.Uint16(b))
}

func (r *msgReader) readInt32() int32 {
	if r.err != nil {
		return 0
	}

	r.msgBytesRemaining -= 4
	if r.msgBytesRemaining < 0 {
		r.fatal(errors.New("read past end of message"))
		return 0
	}

	b := r.buf[0:4]
	_, err := io.ReadFull(r.reader, b)
	if err != nil {
		r.fatal(err)
		return 0
	}

	return int32(binary.BigEndian.Uint32(b))
}

func (r *msgReader) readInt64() int64 {
	if r.err != nil {
		return 0
	}

	r.msgBytesRemaining -= 8
	if r.msgBytesRemaining < 0 {
		r.fatal(errors.New("read past end of message"))
		return 0
	}

	b := r.buf[0:8]
	_, err := io.ReadFull(r.reader, b)
	if err != nil {
		r.fatal(err)
		return 0
	}

	return int64(binary.BigEndian.Uint64(b))
}

func (r *msgReader) readOid() Oid {
	return Oid(r.readInt32())
}

// readCString reads a null terminated string
func (r *msgReader) readCString() string {
	if r.err != nil {
		return ""
	}

	b, err := r.reader.ReadBytes(0)
	if err != nil {
		r.fatal(err)
		return ""
	}

	r.msgBytesRemaining -= int32(len(b))
	if r.msgBytesRemaining < 0 {
		r.fatal(errors.New("read past end of message"))
		return ""
	}

	return string(b[0 : len(b)-1])
}

// readString reads count bytes and returns as string
func (r *msgReader) readString(count int32) string {
	if r.err != nil {
		return ""
	}

	r.msgBytesRemaining -= count
	if r.msgBytesRemaining < 0 {
		r.fatal(errors.New("read past end of message"))
		return ""
	}

	var b []byte
	if count <= int32(len(r.buf)) {
		b = r.buf[0:int(count)]
	} else {
		b = make([]byte, int(count))
	}

	_, err := io.ReadFull(r.reader, b)
	if err != nil {
		r.fatal(err)
		return ""
	}

	return string(b)
}

// readBytes reads count bytes and returns as []byte
func (r *msgReader) readBytes(count int32) []byte {
	if r.err != nil {
		return nil
	}

	r.msgBytesRemaining -= count
	if r.msgBytesRemaining < 0 {
		r.fatal(errors.New("read past end of message"))
		return nil
	}

	b := make([]byte, int(count))

	_, err := io.ReadFull(r.reader, b)
	if err != nil {
		r.fatal(err)
		return nil
	}

	return b
}
<|endoftext|>