<commit_before>package main

import (
  "log"
  "os"
  "path/filepath"
  "time"
)

type ProspectorResume struct {
  files  map[string]*FileState
  resave chan *FileState
}

type ProspectorInfo struct {
  fileinfo  os.FileInfo /* the file info */
  harvester chan int64  /* the harvester will send an event with its offset when it closes */
  last_seen uint32      /* int number of the last iterations in which we saw this file */
}

type Prospector struct {
  FileConfig     FileConfig
  prospectorinfo map[string]ProspectorInfo
  iteration      uint32
  lastscan       time.Time
}

func (p *Prospector) Prospect(resumelist *ProspectorResume, registrar_chan chan []*FileEvent, output chan *FileEvent) {
  p.prospectorinfo = make(map[string]ProspectorInfo)

  // Handle any "-" (stdin) paths
  for i, path := range p.FileConfig.Paths {
    if path == "-" {
      // Offset and Initial never get used when path is "-"
      harvester := Harvester{Path: path, FileConfig: p.FileConfig}
      go harvester.Harvest(registrar_chan, output)

      // Remove it from the file list
      p.FileConfig.Paths = append(p.FileConfig.Paths[:i], p.FileConfig.Paths[i+1:]...)
    }
  }

  // Seed last scan time
  p.lastscan = time.Now()

  // Now let's do one quick scan to pick up new files - flag true so new files obey from-beginning
  for _, path := range p.FileConfig.Paths {
    p.scan(path, registrar_chan, output, resumelist)
  }

  // This signals we finished considering the previous state
  event := &FileState{
    Source: nil,
  }
  resumelist.resave <- event

  for {
    newlastscan := time.Now()

    for _, path := range p.FileConfig.Paths {
      // Scan - flag false so new files always start at beginning
      p.scan(path, registrar_chan, output, nil)
    }

    p.lastscan = newlastscan

    // Defer next scan for a bit.
    time.Sleep(10 * time.Second) // Make this tunable

    // Clear out files that disappeared
    for file, lastinfo := range p.prospectorinfo {
      if lastinfo.last_seen < p.iteration {
        delete(p.prospectorinfo, file)
      }
    }

    p.iteration++ // Overflow is allowed
  }
} /* Prospect */

func (p *Prospector) scan(path string, registrar_chan chan []*FileEvent, output chan *FileEvent, resumelist *ProspectorResume) {
  //log.Printf("Prospecting %s\n", path)

  // Evaluate the path as a wildcards/shell glob
  matches, err := filepath.Glob(path)
  if err != nil {
    log.Printf("glob(%s) failed: %v\n", path, err)
    return
  }

  // To keep the old inode/dev reference if we see a file has renamed, in case it was also renamed prior
  missingfiles := make(map[string]os.FileInfo)

  // Check any matched files to see if we need to start a harvester
  for _, file := range matches {
    // Stat the file, following any symlinks.
    fileinfo, err := os.Stat(file)
    // TODO(sissel): check err
    if err != nil {
      log.Printf("stat(%s) failed: %s\n", file, err)
      continue
    }

    if fileinfo.IsDir() {
      log.Printf("Skipping directory: %s\n", file)
      continue
    }

    // Check the current info against p.prospectorinfo[file]
    lastinfo, is_known := p.prospectorinfo[file]
    newinfo := lastinfo

    // Conditions for starting a new harvester:
    // - file path hasn't been seen before
    // - the file's inode or device changed
    if !is_known {
      // Create a new prospector info with the stat info for comparison
      newinfo = ProspectorInfo{fileinfo: fileinfo, harvester: make(chan int64, 1), last_seen: p.iteration}

      // Check for dead time, but only if the file modification time is before the last scan started
      // This ensures we don't skip genuine creations with dead times less than 10s
      if fileinfo.ModTime().Before(p.lastscan) && time.Since(fileinfo.ModTime()) > p.FileConfig.deadtime {
        var offset int64 = 0
        var is_resuming bool = false

        if resumelist != nil {
          // Call the calculator - it will process resume state if there is one
          offset, is_resuming = p.calculate_resume(file, fileinfo, resumelist)
        }

        // Are we resuming a dead file? We have to resume even if dead so we catch any old updates to the file
        // This is safe as the harvester, once it hits the EOF and a timeout, will stop harvesting
        // Once we detect changes again we can resume another harvester again - this keeps number of go routines to a minimum
        if is_resuming {
          log.Printf("Resuming harvester on a previously harvested file: %s\n", file)
          harvester := &Harvester{Path: file, FileConfig: p.FileConfig, Offset: offset, FinishChan: newinfo.harvester}
          go harvester.Harvest(registrar_chan, output)
        } else {
          // Old file, skip it, but push offset of file size so we start from the end if this file changes and needs picking up
          log.Printf("Skipping file (older than dead time of %v): %s\n", p.FileConfig.deadtime, file)
          newinfo.harvester <- fileinfo.Size()
        }
      } else if previous := lookup_file_ids(file, fileinfo, p.prospectorinfo, missingfiles); previous != "" {
        // This file was simply renamed (known inode+dev) - link the same harvester channel as the old file
        log.Printf("File rename was detected: %s -> %s\n", previous, file)

        newinfo.harvester = p.prospectorinfo[previous].harvester
      } else {
        var initial bool = false
        var offset int64 = 0
        var is_resuming bool = false

        if resumelist != nil {
          // Call the calculator - it will process resume state if there is one
          offset, is_resuming = p.calculate_resume(file, fileinfo, resumelist)
          initial = true
        }

        // Are we resuming a file or is this a completely new file?
        if is_resuming {
          log.Printf("Resuming harvester on a previously harvested file: %s\n", file)

          // By setting initial to false we ensure that offset is always obeyed, even on first scan, which is necessary for resume
          initial = false
        } else {
          log.Printf("Launching harvester on new file: %s\n", file)
        }

        // Launch the harvester - if initial is true it means ignore offset and choose end if this is first scan, and choose beginning if subsequence scan
        // This ensures we always pick up new logs from start - and only skip to end if we've just started up
        harvester := &Harvester{Path: file, FileConfig: p.FileConfig, Offset: offset, FinishChan: newinfo.harvester, Initial: initial}
        go harvester.Harvest(registrar_chan, output)
      }
    } else {
      // Update the fileinfo information used for future comparisons, and the last_seen counter
      newinfo.fileinfo = fileinfo
      newinfo.last_seen = p.iteration

      if !os.SameFile(lastinfo.fileinfo, fileinfo) {
        if previous := lookup_file_ids(file, fileinfo, p.prospectorinfo, missingfiles); previous != "" {
          // This file was renamed from another file we know - link the same harvester channel as the old file
          log.Printf("File rename was detected: %s -> %s\n", previous, file)

          newinfo.harvester = p.prospectorinfo[previous].harvester
        } else {
          // File is not the same file we saw previously, it must have rotated and is a new file
          log.Printf("Launching harvester on rotated file: %s\n", file)

          // Forget about the previous harvester and let it continue on the old file - so start a new channel to use with the new harvester
          newinfo.harvester = make(chan int64, 1)

          // Start a harvester on the path
          harvester := &Harvester{Path: file, FileConfig: p.FileConfig, FinishChan: newinfo.harvester, Initial: (resumelist != nil)}
          go harvester.Harvest(registrar_chan, output)
        }

        // Keep the old file in missingfiles so we don't rescan it if it was renamed and we've not yet reached the new filename
        // We only need to keep it for the remainder of this iteration then we can assume it was deleted and forget about it
        missingfiles[file] = lastinfo.fileinfo
      } else if len(newinfo.harvester) != 0 && lastinfo.fileinfo.ModTime() != fileinfo.ModTime() {
        // Resume harvesting of an old file we've stopped harvesting from
        log.Printf("Resuming harvester on an old file that was just modified: %s\n", file)

        // Start a harvester on the path; an old file was just modified and it doesn't have a harvester
        // The offset to continue from will be stored in the harvester channel - so take that to use and also clear the channel
        // Don't pass initial, we are resuming based on last offset
        harvester := &Harvester{Path: file, FileConfig: p.FileConfig, Offset: <-newinfo.harvester, FinishChan: newinfo.harvester}
        go harvester.Harvest(registrar_chan, output)
      }
    }

    // Track the stat data for this file for later comparison to check for
    // rotation/etc
    p.prospectorinfo[file] = newinfo
  } // for each file matched by the glob
}

func (p *Prospector) calculate_resume(file string, fileinfo os.FileInfo, resumelist *ProspectorResume) (int64, bool) {
  last_state, is_found := resumelist.files[file]

  if is_found && is_filestate_same(file, fileinfo, last_state) {
    // We're resuming - throw the last state back downstream so we resave it
    // And return the offset - also force harvest in case the file is old and we're about to skip it
    resumelist.resave <- last_state
    return last_state.Offset, true
  }

  if previous := lookup_file_ids_resumelist(file, fileinfo, resumelist.files); previous != "" {
    // File has rotated between shutdown and startup
    // We return last state downstream, with a modified event source with the new file name
    // And return the offset - also force harvest in case the file is old and we're about to skip it
    log.Printf("Detected rename of a previously harvested file: %s -> %s\n", previous, file)
    event := resumelist.files[previous]
    event.Source = &file
    resumelist.resave <- event
    return event.Offset, true
  }

  if is_found {
    log.Printf("Not resuming rotated file: %s\n", file)
  }

  // New file so just start from an automatic position
  return 0, false
}
<commit_msg>Only clean out prospector info of files we've stopped harvesting<commit_after>package main

import (
  "log"
  "os"
  "path/filepath"
  "time"
)

type ProspectorResume struct {
  files  map[string]*FileState
  resave chan *FileState
}

type ProspectorInfo struct {
  fileinfo  os.FileInfo /* the file info */
  harvester chan int64  /* the harvester will send an event with its offset when it closes */
  last_seen uint32      /* int number of the last iterations in which we saw this file */
}

type Prospector struct {
  FileConfig     FileConfig
  prospectorinfo map[string]ProspectorInfo
  iteration      uint32
  lastscan       time.Time
}

func (p *Prospector) Prospect(resumelist *ProspectorResume, registrar_chan chan []*FileEvent, output chan *FileEvent) {
  p.prospectorinfo = make(map[string]ProspectorInfo)

  // Handle any "-" (stdin) paths
  for i, path := range p.FileConfig.Paths {
    if path == "-" {
      // Offset and Initial never get used when path is "-"
      harvester := Harvester{Path: path, FileConfig: p.FileConfig}
      go harvester.Harvest(registrar_chan, output)

      // Remove it from the file list
      p.FileConfig.Paths = append(p.FileConfig.Paths[:i], p.FileConfig.Paths[i+1:]...)
    }
  }

  // Seed last scan time
  p.lastscan = time.Now()

  // Now let's do one quick scan to pick up new files - flag true so new files obey from-beginning
  for _, path := range p.FileConfig.Paths {
    p.scan(path, registrar_chan, output, resumelist)
  }

  // This signals we finished considering the previous state
  event := &FileState{
    Source: nil,
  }
  resumelist.resave <- event

  for {
    newlastscan := time.Now()

    for _, path := range p.FileConfig.Paths {
      // Scan - flag false so new files always start at beginning
      p.scan(path, registrar_chan, output, nil)
    }

    p.lastscan = newlastscan

    // Defer next scan for a bit.
    time.Sleep(10 * time.Second) // Make this tunable

    // Clear out files that disappeared and we've stopped harvesting
    for file, lastinfo := range p.prospectorinfo {
      if len(lastinfo.harvester) != 0 && lastinfo.last_seen < p.iteration {
        delete(p.prospectorinfo, file)
      }
    }

    p.iteration++ // Overflow is allowed
  }
} /* Prospect */

func (p *Prospector) scan(path string, registrar_chan chan []*FileEvent, output chan *FileEvent, resumelist *ProspectorResume) {
  //log.Printf("Prospecting %s\n", path)

  // Evaluate the path as a wildcards/shell glob
  matches, err := filepath.Glob(path)
  if err != nil {
    log.Printf("glob(%s) failed: %v\n", path, err)
    return
  }

  // To keep the old inode/dev reference if we see a file has renamed, in case it was also renamed prior
  missingfiles := make(map[string]os.FileInfo)

  // Check any matched files to see if we need to start a harvester
  for _, file := range matches {
    // Stat the file, following any symlinks.
    fileinfo, err := os.Stat(file)
    // TODO(sissel): check err
    if err != nil {
      log.Printf("stat(%s) failed: %s\n", file, err)
      continue
    }

    if fileinfo.IsDir() {
      log.Printf("Skipping directory: %s\n", file)
      continue
    }

    // Check the current info against p.prospectorinfo[file]
    lastinfo, is_known := p.prospectorinfo[file]
    newinfo := lastinfo

    // Conditions for starting a new harvester:
    // - file path hasn't been seen before
    // - the file's inode or device changed
    if !is_known {
      // Create a new prospector info with the stat info for comparison
      newinfo = ProspectorInfo{fileinfo: fileinfo, harvester: make(chan int64, 1), last_seen: p.iteration}

      // Check for dead time, but only if the file modification time is before the last scan started
      // This ensures we don't skip genuine creations with dead times less than 10s
      if fileinfo.ModTime().Before(p.lastscan) && time.Since(fileinfo.ModTime()) > p.FileConfig.deadtime {
        var offset int64 = 0
        var is_resuming bool = false

        if resumelist != nil {
          // Call the calculator - it will process resume state if there is one
          offset, is_resuming = p.calculate_resume(file, fileinfo, resumelist)
        }

        // Are we resuming a dead file? We have to resume even if dead so we catch any old updates to the file
        // This is safe as the harvester, once it hits the EOF and a timeout, will stop harvesting
        // Once we detect changes again we can resume another harvester again - this keeps number of go routines to a minimum
        if is_resuming {
          log.Printf("Resuming harvester on a previously harvested file: %s\n", file)
          harvester := &Harvester{Path: file, FileConfig: p.FileConfig, Offset: offset, FinishChan: newinfo.harvester}
          go harvester.Harvest(registrar_chan, output)
        } else {
          // Old file, skip it, but push offset of file size so we start from the end if this file changes and needs picking up
          log.Printf("Skipping file (older than dead time of %v): %s\n", p.FileConfig.deadtime, file)
          newinfo.harvester <- fileinfo.Size()
        }
      } else if previous := lookup_file_ids(file, fileinfo, p.prospectorinfo, missingfiles); previous != "" {
        // This file was simply renamed (known inode+dev) - link the same harvester channel as the old file
        log.Printf("File rename was detected: %s -> %s\n", previous, file)

        newinfo.harvester = p.prospectorinfo[previous].harvester
      } else {
        var initial bool = false
        var offset int64 = 0
        var is_resuming bool = false

        if resumelist != nil {
          // Call the calculator - it will process resume state if there is one
          offset, is_resuming = p.calculate_resume(file, fileinfo, resumelist)
          initial = true
        }

        // Are we resuming a file or is this a completely new file?
        if is_resuming {
          log.Printf("Resuming harvester on a previously harvested file: %s\n", file)

          // By setting initial to false we ensure that offset is always obeyed, even on first scan, which is necessary for resume
          initial = false
        } else {
          log.Printf("Launching harvester on new file: %s\n", file)
        }

        // Launch the harvester - if initial is true it means ignore offset and choose end if this is first scan, and choose beginning if subsequence scan
        // This ensures we always pick up new logs from start - and only skip to end if we've just started up
        harvester := &Harvester{Path: file, FileConfig: p.FileConfig, Offset: offset, FinishChan: newinfo.harvester, Initial: initial}
        go harvester.Harvest(registrar_chan, output)
      }
    } else {
      // Update the fileinfo information used for future comparisons, and the last_seen counter
      newinfo.fileinfo = fileinfo
      newinfo.last_seen = p.iteration

      if !os.SameFile(lastinfo.fileinfo, fileinfo) {
        if previous := lookup_file_ids(file, fileinfo, p.prospectorinfo, missingfiles); previous != "" {
          // This file was renamed from another file we know - link the same harvester channel as the old file
          log.Printf("File rename was detected: %s -> %s\n", previous, file)

          newinfo.harvester = p.prospectorinfo[previous].harvester
        } else {
          // File is not the same file we saw previously, it must have rotated and is a new file
          log.Printf("Launching harvester on rotated file: %s\n", file)

          // Forget about the previous harvester and let it continue on the old file - so start a new channel to use with the new harvester
          newinfo.harvester = make(chan int64, 1)

          // Start a harvester on the path
          harvester := &Harvester{Path: file, FileConfig: p.FileConfig, FinishChan: newinfo.harvester, Initial: (resumelist != nil)}
          go harvester.Harvest(registrar_chan, output)
        }

        // Keep the old file in missingfiles so we don't rescan it if it was renamed and we've not yet reached the new filename
        // We only need to keep it for the remainder of this iteration then we can assume it was deleted and forget about it
        missingfiles[file] = lastinfo.fileinfo
      } else if len(newinfo.harvester) != 0 && lastinfo.fileinfo.ModTime() != fileinfo.ModTime() {
        // Resume harvesting of an old file we've stopped harvesting from
        log.Printf("Resuming harvester on an old file that was just modified: %s\n", file)

        // Start a harvester on the path; an old file was just modified and it doesn't have a harvester
        // The offset to continue from will be stored in the harvester channel - so take that to use and also clear the channel
        // Don't pass initial, we are resuming based on last offset
        harvester := &Harvester{Path: file, FileConfig: p.FileConfig, Offset: <-newinfo.harvester, FinishChan: newinfo.harvester}
        go harvester.Harvest(registrar_chan, output)
      }
    }

    // Track the stat data for this file for later comparison to check for
    // rotation/etc
    p.prospectorinfo[file] = newinfo
  } // for each file matched by the glob
}

func (p *Prospector) calculate_resume(file string, fileinfo os.FileInfo, resumelist *ProspectorResume) (int64, bool) {
  last_state, is_found := resumelist.files[file]

  if is_found && is_filestate_same(file, fileinfo, last_state) {
    // We're resuming - throw the last state back downstream so we resave it
    // And return the offset - also force harvest in case the file is old and we're about to skip it
    resumelist.resave <- last_state
    return last_state.Offset, true
  }

  if previous := lookup_file_ids_resumelist(file, fileinfo, resumelist.files); previous != "" {
    // File has rotated between shutdown and startup
    // We return last state downstream, with a modified event source with the new file name
    // And return the offset - also force harvest in case the file is old and we're about to skip it
    log.Printf("Detected rename of a previously harvested file: %s -> %s\n", previous, file)
    event := resumelist.files[previous]
    event.Source = &file
    resumelist.resave <- event
    return event.Offset, true
  }

  if is_found {
    log.Printf("Not resuming rotated file: %s\n", file)
  }

  // New file so just start from an automatic position
  return 0, false
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcs

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"net/url"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/api/googleapi"
)

// A bucket that wraps another, calling its methods in a retry loop with
// randomized exponential backoff.
type retryBucket struct {
	maxSleep time.Duration
	wrapped  Bucket
}

func newRetryBucket(
	maxSleep time.Duration,
	wrapped Bucket) (b Bucket) {
	b = &retryBucket{
		maxSleep: maxSleep,
		wrapped:  wrapped,
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func shouldRetry(err error) (b bool) {
	// HTTP 50x errors.
	if typed, ok := err.(*googleapi.Error); ok {
		if typed.Code >= 500 && typed.Code < 600 {
			b = true
			return
		}
	}

	// HTTP 429 errors (GCS uses these for rate limiting).
	if typed, ok := err.(*googleapi.Error); ok {
		if typed.Code == 429 {
			b = true
			return
		}
	}

	// Network errors, which tend to show up transiently when doing lots of
	// operations in parallel. For example:
	//
	//     dial tcp 74.125.203.95:443: too many open files
	//
	if _, ok := err.(*net.OpError); ok {
		b = true
		return
	}

	// Sometimes the HTTP package helpfully encapsulates the real error in a URL
	// error.
	if urlErr, ok := err.(*url.Error); ok {
		b = shouldRetry(urlErr.Err)
		return
	}

	return
}

// Choose an appropriate delay for exponential backoff, given that we have
// already slept the given number of times for this logical request.
func chooseDelay(prevSleepCount uint) (d time.Duration) {
	const baseDelay = time.Millisecond

	// Choose a a delay in [0, 2^prevSleepCount * baseDelay).
	d = (1 << prevSleepCount) * baseDelay
	d = time.Duration(float64(d) * rand.Float64())

	return
}

// Exponential backoff for a function that might fail.
//
// This is essentially what is described in the "Best practices" section of the
// "Upload Objects" docs:
//
//     https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload
//
// with the following exceptions:
//
//  *  We perform backoff for all operations.
//
//  *  The random component scales with the delay, so that the first sleep
//     cannot be as long as one second. The algorithm used matches the
//     description at http://en.wikipedia.org/wiki/Exponential_backoff.
//
//  *  We retry more types of errors; see shouldRetry above.
//
func expBackoff(
	ctx context.Context,
	maxSleep time.Duration,
	f func() error) (err error) {
	var totalSleep time.Duration

	for n := uint(0); ; n++ {
		// Make an attempt. Stop if successful.
		err = f()
		if err == nil {
			return
		}

		// Do we want to retry?
		if !shouldRetry(err) {
			log.Printf(
				"Not retrying error of type %T (%q): %#v",
				err,
				err.Error(),
				err)

			return
		}

		// Choose a a delay.
		d := chooseDelay(n)

		// Are we out of credit?
		if totalSleep+d > maxSleep {
			// Return the most recent error.
			return
		}

		// Sleep, returning early if cancelled.
		log.Printf("Retrying after error of type %T (%q) in %v", err, err, d)

		select {
		case <-ctx.Done():
			err = ctx.Err()
			return

		case <-time.After(d):
			totalSleep += d
			continue
		}
	}
}

////////////////////////////////////////////////////////////////////////
// Read support
////////////////////////////////////////////////////////////////////////

type retryObjectReader struct {
	// The context we should watch when sleeping for retries.
	ctx context.Context

	// The object name and generation we want to read.
	name       string
	generation int64

	// nil when we start or have seen a permanent error.
	wrapped io.ReadCloser

	// If we've seen an error that we shouldn't retry for, this will be non-nil
	// and should be returned permanently.
	permanentErr error

	// How many bytes we've already passed on to the user.
	bytesRead uint64

	// The number of times we've slept so far, and the total amount of time we've
	// spent sleeping.
	sleepCount    uint
	sleepDuration time.Duration
}

// Set up the wrapped reader.
func (rc *retryObjectReader) setUpWrapped() (err error)

// Set up the wrapped reader if necessary, and make one attempt to read through
// it.
//
// Clears the wrapped reader on error.
func (rc *retryObjectReader) readOnce(p []byte) (n int, err error) {
	// Set up the wrapped reader if it's not already around.
	if rc.wrapped == nil {
		err = rc.setUpWrapped()
		if err != nil {
			return
		}
	}

	// Attempt to read from it.
	n, err = rc.wrapped.Read(p)
	if err != nil {
		rc.wrapped = nil
		return
	}

	return
}

// Invariant: we never return an error from this function unless we've given up
// on retrying. In particular, we won't return a short read because the wrapped
// reader returned a short read and an error.
func (rc *retryObjectReader) Read(p []byte) (n int, err error) {
	// Whatever we do, accumulate the bytes that we're returning to the user.
	defer func() {
		if n < 0 {
			panic(fmt.Sprintf("Negative byte count: %d", n))
		}

		rc.bytesRead += uint64(n)
	}()

	// If we've already decided on a permanent error, return that.
	if rc.permanentErr != nil {
		err = rc.permanentErr
		rc.wrapped = nil
		return
	}

	// If we let an error escape below, it must be a permanent one.
	defer func() {
		if err != nil {
			rc.permanentErr = err
		}
	}()

	for {
		// Make one attempt. Make sure to accumulate the result.
		var bytesRead int
		bytesRead, err = rc.readOnce(p)
		n += bytesRead
		p = p[bytesRead:]

		// If we were successful, we're done.
		if err == nil {
			return
		}

		// Do we want to retry?
		if !shouldRetry(err) {
			log.Printf(
				"Not retrying read error of type %T (%q): %#v",
				err,
				err.Error(),
				err)

			return
		}

		// Choose a delay.
		d := chooseDelay(rc.sleepCount)
		rc.sleepCount++

		// Sleep, returning early if cancelled.
		log.Printf("Retrying after read error of type %T (%q) in %v", err, err, d)

		select {
		case <-rc.ctx.Done():
			err = rc.ctx.Err()
			return

		case <-time.After(d):
			rc.sleepDuration += d
		}
	}
}

func (rc *retryObjectReader) Close() (err error) {
	// If we don't have a wrapped reader, there is nothing useful that we can or
	// need to do here.
	if rc.wrapped == nil {
		return
	}

	// Call through.
	err = rc.wrapped.Close()

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (rb *retryBucket) Name() (name string) {
	name = rb.wrapped.Name()
	return
}

func (rb *retryBucket) NewReader(
	ctx context.Context,
	req *ReadObjectRequest) (rc io.ReadCloser, err error) {
	err = expBackoff(
		ctx,
		rb.maxSleep,
		func() (err error) {
			rc, err = rb.wrapped.NewReader(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) CreateObject(
	ctx context.Context,
	req *CreateObjectRequest) (o *Object, err error) {
	// We can't simply replay the request multiple times, because the first
	// attempt might exhaust some of the req.Contents reader, leaving missing
	// contents for the second attempt.
	//
	// So, copy out all contents and create a modified request that serves from
	// memory.
	contents, err := ioutil.ReadAll(req.Contents)
	if err != nil {
		err = fmt.Errorf("ioutil.ReadAll: %v", err)
		return
	}

	reqCopy := *req
	reqCopy.Contents = bytes.NewReader(contents)

	// Call through with that request.
	err = expBackoff(
		ctx,
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.CreateObject(ctx, &reqCopy)
			return
		})

	return
}

func (rb *retryBucket) CopyObject(
	ctx context.Context,
	req *CopyObjectRequest) (o *Object, err error) {
	err = expBackoff(
		ctx,
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.CopyObject(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) StatObject(
	ctx context.Context,
	req *StatObjectRequest) (o *Object, err error) {
	err = expBackoff(
		ctx,
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.StatObject(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) ListObjects(
	ctx context.Context,
	req *ListObjectsRequest) (listing *Listing, err error) {
	err = expBackoff(
		ctx,
		rb.maxSleep,
		func() (err error) {
			listing, err = rb.wrapped.ListObjects(ctx, req)
			return
		})
	return
}

func (rb *retryBucket) UpdateObject(
	ctx context.Context,
	req *UpdateObjectRequest) (o *Object, err error) {
	err = expBackoff(
		ctx,
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.UpdateObject(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) DeleteObject(
	ctx context.Context,
	name string) (err error) {
	err = expBackoff(
		ctx,
		rb.maxSleep,
		func() (err error) {
			err = rb.wrapped.DeleteObject(ctx, name)
			return
		})

	return
}
<commit_msg>Made expBackoff shareable.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcs

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"net/url"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/api/googleapi"
)

// A bucket that wraps another, calling its methods in a retry loop with
// randomized exponential backoff.
type retryBucket struct {
	maxSleep time.Duration
	wrapped  Bucket
}

func newRetryBucket(
	maxSleep time.Duration,
	wrapped Bucket) (b Bucket) {
	b = &retryBucket{
		maxSleep: maxSleep,
		wrapped:  wrapped,
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func shouldRetry(err error) (b bool) {
	// HTTP 50x errors.
	if typed, ok := err.(*googleapi.Error); ok {
		if typed.Code >= 500 && typed.Code < 600 {
			b = true
			return
		}
	}

	// HTTP 429 errors (GCS uses these for rate limiting).
	if typed, ok := err.(*googleapi.Error); ok {
		if typed.Code == 429 {
			b = true
			return
		}
	}

	// Network errors, which tend to show up transiently when doing lots of
	// operations in parallel. For example:
	//
	//     dial tcp 74.125.203.95:443: too many open files
	//
	if _, ok := err.(*net.OpError); ok {
		b = true
		return
	}

	// Sometimes the HTTP package helpfully encapsulates the real error in a URL
	// error.
	if urlErr, ok := err.(*url.Error); ok {
		b = shouldRetry(urlErr.Err)
		return
	}

	return
}

// Choose an appropriate delay for exponential backoff, given that we have
// already slept the given number of times for this logical request.
func chooseDelay(prevSleepCount uint) (d time.Duration) {
	const baseDelay = time.Millisecond

	// Choose a a delay in [0, 2^prevSleepCount * baseDelay).
	d = (1 << prevSleepCount) * baseDelay
	d = time.Duration(float64(d) * rand.Float64())

	return
}

// Exponential backoff for a function that might fail.
//
// This is essentially what is described in the "Best practices" section of the
// "Upload Objects" docs:
//
//     https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload
//
// with the following exceptions:
//
//  *  We perform backoff for all operations.
//
//  *  The random component scales with the delay, so that the first sleep
//     cannot be as long as one second. The algorithm used matches the
//     description at http://en.wikipedia.org/wiki/Exponential_backoff.
//
//  *  We retry more types of errors; see shouldRetry above.
//
// State for total sleep time and number of previous sleeps is housed outside
// of this function to allow it to be "resumed" by multiple invocations of
// retryObjectReader.Read.
func expBackoff(
	ctx context.Context,
	desc string,
	maxSleep time.Duration,
	f func() error,
	prevSleepCount *uint,
	prevSleepDuration *time.Duration) (err error) {
	for {
		// Make an attempt. Stop if successful.
		err = f()
		if err == nil {
			return
		}

		// Do we want to retry?
		if !shouldRetry(err) {
			log.Printf(
				"Not retrying %s after error of type %T (%q): %#v",
				desc,
				err,
				err.Error(),
				err)

			return
		}

		// Choose a a delay.
		d := chooseDelay(*prevSleepCount)
		*prevSleepCount++

		// Are we out of credit?
		if *prevSleepDuration+d > maxSleep {
			// Return the most recent error.
			return
		}

		// Sleep, returning early if cancelled.
		log.Printf(
			"Retrying %s after error of type %T (%q) in %v",
			desc,
			err,
			err,
			d)

		select {
		case <-ctx.Done():
			err = ctx.Err()
			return

		case <-time.After(d):
			*prevSleepDuration += d
			continue
		}
	}
}

// Like expBackoff, but assumes that we've never slept before (and won't need
// to sleep again).
func oneShotExpBackoff(
	ctx context.Context,
	desc string,
	maxSleep time.Duration,
	f func() error) (err error) {
	var prevSleepCount uint
	var prevSleepDuration time.Duration

	err = expBackoff(
		ctx,
		desc,
		maxSleep,
		f,
		&prevSleepCount,
		&prevSleepDuration)

	return
}

////////////////////////////////////////////////////////////////////////
// Read support
////////////////////////////////////////////////////////////////////////

type retryObjectReader struct {
	// The context we should watch when sleeping for retries.
	ctx context.Context

	// The object name and generation we want to read.
	name       string
	generation int64

	// nil when we start or have seen a permanent error.
	wrapped io.ReadCloser

	// If we've seen an error that we shouldn't retry for, this will be non-nil
	// and should be returned permanently.
	permanentErr error

	// How many bytes we've already passed on to the user.
	bytesRead uint64

	// The number of times we've slept so far, and the total amount of time we've
	// spent sleeping.
	sleepCount    uint
	sleepDuration time.Duration
}

// Set up the wrapped reader.
func (rc *retryObjectReader) setUpWrapped() (err error)

// Set up the wrapped reader if necessary, and make one attempt to read through
// it.
//
// Clears the wrapped reader on error.
func (rc *retryObjectReader) readOnce(p []byte) (n int, err error) {
	// Set up the wrapped reader if it's not already around.
	if rc.wrapped == nil {
		err = rc.setUpWrapped()
		if err != nil {
			return
		}
	}

	// Attempt to read from it.
	n, err = rc.wrapped.Read(p)
	if err != nil {
		rc.wrapped = nil
		return
	}

	return
}

// Invariant: we never return an error from this function unless we've given up
// on retrying. In particular, we won't return a short read because the wrapped
// reader returned a short read and an error.
func (rc *retryObjectReader) Read(p []byte) (n int, err error) {
	// Whatever we do, accumulate the bytes that we're returning to the user.
	defer func() {
		if n < 0 {
			panic(fmt.Sprintf("Negative byte count: %d", n))
		}

		rc.bytesRead += uint64(n)
	}()

	// If we've already decided on a permanent error, return that.
	if rc.permanentErr != nil {
		err = rc.permanentErr
		rc.wrapped = nil
		return
	}

	// If we let an error escape below, it must be a permanent one.
	defer func() {
		if err != nil {
			rc.permanentErr = err
		}
	}()

	for {
		// Make one attempt. Make sure to accumulate the result.
		var bytesRead int
		bytesRead, err = rc.readOnce(p)
		n += bytesRead
		p = p[bytesRead:]

		// If we were successful, we're done.
		if err == nil {
			return
		}

		// Do we want to retry?
		if !shouldRetry(err) {
			log.Printf(
				"Not retrying read error of type %T (%q): %#v",
				err,
				err.Error(),
				err)

			return
		}

		// Choose a delay.
		d := chooseDelay(rc.sleepCount)
		rc.sleepCount++

		// Sleep, returning early if cancelled.
		log.Printf("Retrying after read error of type %T (%q) in %v", err, err, d)

		select {
		case <-rc.ctx.Done():
			err = rc.ctx.Err()
			return

		case <-time.After(d):
			rc.sleepDuration += d
		}
	}
}

func (rc *retryObjectReader) Close() (err error) {
	// If we don't have a wrapped reader, there is nothing useful that we can or
	// need to do here.
	if rc.wrapped == nil {
		return
	}

	// Call through.
	err = rc.wrapped.Close()

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (rb *retryBucket) Name() (name string) {
	name = rb.wrapped.Name()
	return
}

func (rb *retryBucket) NewReader(
	ctx context.Context,
	req *ReadObjectRequest) (rc io.ReadCloser, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("NewReader(%q)", req.Name),
		rb.maxSleep,
		func() (err error) {
			rc, err = rb.wrapped.NewReader(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) CreateObject(
	ctx context.Context,
	req *CreateObjectRequest) (o *Object, err error) {
	// We can't simply replay the request multiple times, because the first
	// attempt might exhaust some of the req.Contents reader, leaving missing
	// contents for the second attempt.
	//
	// So, copy out all contents and create a modified request that serves from
	// memory.
	contents, err := ioutil.ReadAll(req.Contents)
	if err != nil {
		err = fmt.Errorf("ioutil.ReadAll: %v", err)
		return
	}

	reqCopy := *req
	reqCopy.Contents = bytes.NewReader(contents)

	// Call through with that request.
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("CreateObject(%q)", req.Name),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.CreateObject(ctx, &reqCopy)
			return
		})

	return
}

func (rb *retryBucket) CopyObject(
	ctx context.Context,
	req *CopyObjectRequest) (o *Object, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("CopyObject(%q, %q)", req.SrcName, req.DstName),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.CopyObject(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) StatObject(
	ctx context.Context,
	req *StatObjectRequest) (o *Object, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("StatObject(%q)", req.Name),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.StatObject(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) ListObjects(
	ctx context.Context,
	req *ListObjectsRequest) (listing *Listing, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("ListObjects(%q)", req.Prefix),
		rb.maxSleep,
		func() (err error) {
			listing, err = rb.wrapped.ListObjects(ctx, req)
			return
		})
	return
}

func (rb *retryBucket) UpdateObject(
	ctx context.Context,
	req *UpdateObjectRequest) (o *Object, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("UpdateObject(%q)", req.Name),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.UpdateObject(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) DeleteObject(
	ctx context.Context,
	name string) (err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("DeleteObject(%q)", name),
		rb.maxSleep,
		func() (err error) {
			err = rb.wrapped.DeleteObject(ctx, name)
			return
		})

	return
}
<|endoftext|>


<commit_before>// Package registry contains client primitives to interact with a remote Docker registry.
package registry

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/distribution/registry/api/errcode"
	"github.com/docker/distribution/registry/api/v2"
	"github.com/docker/distribution/registry/client"
	"github.com/docker/distribution/registry/client/transport"
	"github.com/docker/docker/dockerversion"
	"github.com/docker/docker/pkg/parsers/kernel"
	"github.com/docker/docker/pkg/useragent"
	"github.com/docker/go-connections/tlsconfig"
)

var (
	// ErrAlreadyExists is an error returned if an image being pushed
	// already exists on the remote side
	ErrAlreadyExists = errors.New("Image already exists")
	errLoginRequired = errors.New("Authentication is required.")
)

// dockerUserAgent is the User-Agent the Docker client uses to identify itself.
// It is populated on init(), comprising version information of different components.
var dockerUserAgent string

func init() {
	httpVersion := make([]useragent.VersionInfo, 0, 6)
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "docker", Version: dockerversion.Version})
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "go", Version: runtime.Version()})
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "git-commit", Version: dockerversion.GitCommit})
	if kernelVersion, err := kernel.GetKernelVersion(); err == nil {
		httpVersion = append(httpVersion, useragent.VersionInfo{Name: "kernel", Version: kernelVersion.String()})
	}
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "os", Version: runtime.GOOS})
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "arch", Version: runtime.GOARCH})

	dockerUserAgent = useragent.AppendVersions("", httpVersion...)

	if runtime.GOOS != "linux" {
		V2Only = true
	}
}

func newTLSConfig(hostname string, isSecure bool) (*tls.Config, error) {
	// PreferredServerCipherSuites should have no effect
	tlsConfig := tlsconfig.ServerDefault

	tlsConfig.InsecureSkipVerify = !isSecure

	if isSecure && CertsDir != "" {
		hostDir := filepath.Join(CertsDir, cleanPath(hostname))
		logrus.Debugf("hostDir: %s", hostDir)
		if err := ReadCertsDirectory(&tlsConfig, hostDir); err != nil {
			return nil, err
		}
	}

	return &tlsConfig, nil
}

func hasFile(files []os.FileInfo, name string) bool {
	for _, f := range files {
		if f.Name() == name {
			return true
		}
	}
	return false
}

// ReadCertsDirectory reads the directory for TLS certificates
// including roots and certificate pairs and updates the
// provided TLS configuration.
func ReadCertsDirectory(tlsConfig *tls.Config, directory string) error {
	fs, err := ioutil.ReadDir(directory)
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	for _, f := range fs {
		if strings.HasSuffix(f.Name(), ".crt") {
			if tlsConfig.RootCAs == nil {
				// TODO(dmcgowan): Copy system pool
				tlsConfig.RootCAs = x509.NewCertPool()
			}
			logrus.Debugf("crt: %s", filepath.Join(directory, f.Name()))
			data, err := ioutil.ReadFile(filepath.Join(directory, f.Name()))
			if err != nil {
				return err
			}
			tlsConfig.RootCAs.AppendCertsFromPEM(data)
		}
		if strings.HasSuffix(f.Name(), ".cert") {
			certName := f.Name()
			keyName := certName[:len(certName)-5] + ".key"
			logrus.Debugf("cert: %s", filepath.Join(directory, f.Name()))
			if !hasFile(fs, keyName) {
				return fmt.Errorf("Missing key %s for certificate %s", keyName, certName)
			}
			cert, err := tls.LoadX509KeyPair(filepath.Join(directory, certName), filepath.Join(directory, keyName))
			if err != nil {
				return err
			}
			tlsConfig.Certificates = append(tlsConfig.Certificates, cert)
		}
		if strings.HasSuffix(f.Name(), ".key") {
			keyName := f.Name()
			certName := keyName[:len(keyName)-4] + ".cert"
			logrus.Debugf("key: %s", filepath.Join(directory, f.Name()))
			if !hasFile(fs, certName) {
				return fmt.Errorf("Missing certificate %s for key %s", certName, keyName)
			}
		}
	}

	return nil
}

// DockerHeaders returns request modifiers that ensure requests have
// the User-Agent header set to dockerUserAgent and that metaHeaders
// are added.
func DockerHeaders(metaHeaders http.Header) []transport.RequestModifier {
	modifiers := []transport.RequestModifier{
		transport.NewHeaderRequestModifier(http.Header{"User-Agent": []string{dockerUserAgent}}),
	}
	if metaHeaders != nil {
		modifiers = append(modifiers, transport.NewHeaderRequestModifier(metaHeaders))
	}
	return modifiers
}

// HTTPClient returns a HTTP client structure which uses the given transport
// and contains the necessary headers for redirected requests
func HTTPClient(transport http.RoundTripper) *http.Client {
	return &http.Client{
		Transport:     transport,
		CheckRedirect: addRequiredHeadersToRedirectedRequests,
	}
}

func trustedLocation(req *http.Request) bool {
	var (
		trusteds = []string{"docker.com", "docker.io"}
		hostname = strings.SplitN(req.Host, ":", 2)[0]
	)
	if req.URL.Scheme != "https" {
		return false
	}

	for _, trusted := range trusteds {
		if hostname == trusted || strings.HasSuffix(hostname, "."+trusted) {
			return true
		}
	}
	return false
}

// addRequiredHeadersToRedirectedRequests adds the necessary redirection headers
// for redirected requests
func addRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Request) error {
	if via != nil && via[0] != nil {
		if trustedLocation(req) && trustedLocation(via[0]) {
			req.Header = via[0].Header
			return nil
		}
		for k, v := range via[0].Header {
			if k != "Authorization" {
				for _, vv := range v {
					req.Header.Add(k, vv)
				}
			}
		}
	}
	return nil
}

// ShouldV2Fallback returns true if this error is a reason to fall back to v1.
func ShouldV2Fallback(err errcode.Error) bool {
	switch err.Code {
	case errcode.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown, v2.ErrorCodeNameUnknown:
		return true
	}
	return false
}

// ErrNoSupport is an error type used for errors indicating that an operation
// is not supported. It encapsulates a more specific error.
type ErrNoSupport struct{ Err error }

func (e ErrNoSupport) Error() string {
	if e.Err == nil {
		return "not supported"
	}
	return e.Err.Error()
}

// ContinueOnError returns true if we should fallback to the next endpoint
// as a result of this error.
func ContinueOnError(err error) bool {
	switch v := err.(type) {
	case errcode.Errors:
		if len(v) == 0 {
			return true
		}
		return ContinueOnError(v[0])
	case ErrNoSupport:
		return ContinueOnError(v.Err)
	case errcode.Error:
		return ShouldV2Fallback(v)
	case *client.UnexpectedHTTPResponseError:
		return true
	case error:
		return !strings.Contains(err.Error(), strings.ToLower(syscall.ENOSPC.Error()))
	}
	// let's be nice and fallback if the error is a completely
	// unexpected one.
	// If new errors have to be handled in some way, please
	// add them to the switch above.
	return true
}

// NewTransport returns a new HTTP transport. If tlsConfig is nil, it uses the
// default TLS configuration.
func NewTransport(tlsConfig *tls.Config) *http.Transport {
	if tlsConfig == nil {
		var cfg = tlsconfig.ServerDefault
		tlsConfig = &cfg
	}
	return &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
			DualStack: true,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig:     tlsConfig,
		// TODO(dmcgowan): Call close idle connections when complete and use keep alive
		DisableKeepAlives: true,
	}
}
<commit_msg>Clarify error message when a .cert file is missing a corresponding key<commit_after>// Package registry contains client primitives to interact with a remote Docker registry.
package registry

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/distribution/registry/api/errcode"
	"github.com/docker/distribution/registry/api/v2"
	"github.com/docker/distribution/registry/client"
	"github.com/docker/distribution/registry/client/transport"
	"github.com/docker/docker/dockerversion"
	"github.com/docker/docker/pkg/parsers/kernel"
	"github.com/docker/docker/pkg/useragent"
	"github.com/docker/go-connections/tlsconfig"
)

var (
	// ErrAlreadyExists is an error returned if an image being pushed
	// already exists on the remote side
	ErrAlreadyExists = errors.New("Image already exists")
	errLoginRequired = errors.New("Authentication is required.")
)

// dockerUserAgent is the User-Agent the Docker client uses to identify itself.
// It is populated on init(), comprising version information of different components.
var dockerUserAgent string

func init() {
	httpVersion := make([]useragent.VersionInfo, 0, 6)
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "docker", Version: dockerversion.Version})
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "go", Version: runtime.Version()})
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "git-commit", Version: dockerversion.GitCommit})
	if kernelVersion, err := kernel.GetKernelVersion(); err == nil {
		httpVersion = append(httpVersion, useragent.VersionInfo{Name: "kernel", Version: kernelVersion.String()})
	}
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "os", Version: runtime.GOOS})
	httpVersion = append(httpVersion, useragent.VersionInfo{Name: "arch", Version: runtime.GOARCH})

	dockerUserAgent = useragent.AppendVersions("", httpVersion...)

	if runtime.GOOS != "linux" {
		V2Only = true
	}
}

func newTLSConfig(hostname string, isSecure bool) (*tls.Config, error) {
	// PreferredServerCipherSuites should have no effect
	tlsConfig := tlsconfig.ServerDefault

	tlsConfig.InsecureSkipVerify = !isSecure

	if isSecure && CertsDir != "" {
		hostDir := filepath.Join(CertsDir, cleanPath(hostname))
		logrus.Debugf("hostDir: %s", hostDir)
		if err := ReadCertsDirectory(&tlsConfig, hostDir); err != nil {
			return nil, err
		}
	}

	return &tlsConfig, nil
}

func hasFile(files []os.FileInfo, name string) bool {
	for _, f := range files {
		if f.Name() == name {
			return true
		}
	}
	return false
}

// ReadCertsDirectory reads the directory for TLS certificates
// including roots and certificate pairs and updates the
// provided TLS configuration.
func ReadCertsDirectory(tlsConfig *tls.Config, directory string) error {
	fs, err := ioutil.ReadDir(directory)
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	for _, f := range fs {
		if strings.HasSuffix(f.Name(), ".crt") {
			if tlsConfig.RootCAs == nil {
				// TODO(dmcgowan): Copy system pool
				tlsConfig.RootCAs = x509.NewCertPool()
			}
			logrus.Debugf("crt: %s", filepath.Join(directory, f.Name()))
			data, err := ioutil.ReadFile(filepath.Join(directory, f.Name()))
			if err != nil {
				return err
			}
			tlsConfig.RootCAs.AppendCertsFromPEM(data)
		}
		if strings.HasSuffix(f.Name(), ".cert") {
			certName := f.Name()
			keyName := certName[:len(certName)-5] + ".key"
			logrus.Debugf("cert: %s", filepath.Join(directory, f.Name()))
			if !hasFile(fs, keyName) {
				return fmt.Errorf("Missing key %s for client certificate %s. Note that CA certificates should use the extension .crt.", keyName, certName)
			}
			cert, err := tls.LoadX509KeyPair(filepath.Join(directory, certName), filepath.Join(directory, keyName))
			if err != nil {
				return err
			}
			tlsConfig.Certificates = append(tlsConfig.Certificates, cert)
		}
		if strings.HasSuffix(f.Name(), ".key") {
			keyName := f.Name()
			certName := keyName[:len(keyName)-4] + ".cert"
			logrus.Debugf("key: %s", filepath.Join(directory, f.Name()))
			if !hasFile(fs, certName) {
				return fmt.Errorf("Missing client certificate %s for key %s", certName, keyName)
			}
		}
	}

	return nil
}

// DockerHeaders returns request modifiers that ensure requests have
// the User-Agent header set to dockerUserAgent and that metaHeaders
// are added.
func DockerHeaders(metaHeaders http.Header) []transport.RequestModifier {
	modifiers := []transport.RequestModifier{
		transport.NewHeaderRequestModifier(http.Header{"User-Agent": []string{dockerUserAgent}}),
	}
	if metaHeaders != nil {
		modifiers = append(modifiers, transport.NewHeaderRequestModifier(metaHeaders))
	}
	return modifiers
}

// HTTPClient returns a HTTP client structure which uses the given transport
// and contains the necessary headers for redirected requests
func HTTPClient(transport http.RoundTripper) *http.Client {
	return &http.Client{
		Transport:     transport,
		CheckRedirect: addRequiredHeadersToRedirectedRequests,
	}
}

func trustedLocation(req *http.Request) bool {
	var (
		trusteds = []string{"docker.com", "docker.io"}
		hostname = strings.SplitN(req.Host, ":", 2)[0]
	)
	if req.URL.Scheme != "https" {
		return false
	}

	for _, trusted := range trusteds {
		if hostname == trusted || strings.HasSuffix(hostname, "."+trusted) {
			return true
		}
	}
	return false
}

// addRequiredHeadersToRedirectedRequests adds the necessary redirection headers
// for redirected requests
func addRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Request) error {
	if via != nil && via[0] != nil {
		if trustedLocation(req) && trustedLocation(via[0]) {
			req.Header = via[0].Header
			return nil
		}
		for k, v := range via[0].Header {
			if k != "Authorization" {
				for _, vv := range v {
					req.Header.Add(k, vv)
				}
			}
		}
	}
	return nil
}

// ShouldV2Fallback returns true if this error is a reason to fall back to v1.
func ShouldV2Fallback(err errcode.Error) bool {
	switch err.Code {
	case errcode.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown, v2.ErrorCodeNameUnknown:
		return true
	}
	return false
}

// ErrNoSupport is an error type used for errors indicating that an operation
// is not supported. It encapsulates a more specific error.
type ErrNoSupport struct{ Err error }

func (e ErrNoSupport) Error() string {
	if e.Err == nil {
		return "not supported"
	}
	return e.Err.Error()
}

// ContinueOnError returns true if we should fallback to the next endpoint
// as a result of this error.
func ContinueOnError(err error) bool {
	switch v := err.(type) {
	case errcode.Errors:
		if len(v) == 0 {
			return true
		}
		return ContinueOnError(v[0])
	case ErrNoSupport:
		return ContinueOnError(v.Err)
	case errcode.Error:
		return ShouldV2Fallback(v)
	case *client.UnexpectedHTTPResponseError:
		return true
	case error:
		return !strings.Contains(err.Error(), strings.ToLower(syscall.ENOSPC.Error()))
	}
	// let's be nice and fallback if the error is a completely
	// unexpected one.
	// If new errors have to be handled in some way, please
	// add them to the switch above.
	return true
}

// NewTransport returns a new HTTP transport. If tlsConfig is nil, it uses the
// default TLS configuration.
func NewTransport(tlsConfig *tls.Config) *http.Transport {
	if tlsConfig == nil {
		var cfg = tlsconfig.ServerDefault
		tlsConfig = &cfg
	}
	return &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
			DualStack: true,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig:     tlsConfig,
		// TODO(dmcgowan): Call close idle connections when complete and use keep alive
		DisableKeepAlives: true,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"syscall"

	"code.google.com/p/go.crypto/ssh/terminal"

	"veyron.io/veyron/veyron/lib/flags/consts"
	_ "veyron.io/veyron/veyron/profiles"
	vsecurity "veyron.io/veyron/veyron/security"
	"veyron.io/veyron/veyron/security/agent"
	"veyron.io/veyron/veyron/security/agent/server"

	"veyron.io/veyron/veyron2/options"
	"veyron.io/veyron/veyron2/rt"
	"veyron.io/veyron/veyron2/security"
	"veyron.io/veyron/veyron2/vlog"
)

var (
	keypath      = flag.String("additional_principals", "", "If non-empty, allow for the creation of new principals and save them in this directory.")
	noPassphrase = flag.Bool("no_passphrase", false, "If true, user will not be prompted for principal encryption passphrase.")
)

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `Usage: %s [agent options] command command_args...

Loads the private key specified in privatekey.pem in %v into memory, then
starts the specified command with access to the private key via the
agent protocol instead of directly reading from disk.

`, os.Args[0], consts.VeyronCredentials)
		flag.PrintDefaults()
	}
	flag.Parse()
	if len(flag.Args()) < 1 {
		fmt.Fprintln(os.Stderr, "Need at least one argument.")
		flag.Usage()
		os.Exit(1)
	}
	// TODO(ashankar,cnicolaou): Should flags.Parse be used instead? But that adds unnecessary
	// flags like "--veyron.namespace.root", which has no meaning for this binary.
	dir := os.Getenv(consts.VeyronCredentials)
	if len(dir) == 0 {
		vlog.Fatalf("The %v environment variable must be set to a directory", consts.VeyronCredentials)
	}

	p, passphrase, err := newPrincipalFromDir(dir)
	if err != nil {
		vlog.Fatalf("failed to create new principal from dir(%s): %v", dir, err)
	}

	runtime := rt.Init(options.RuntimePrincipal{p})
	log := runtime.Logger()

	if err = os.Setenv(agent.FdVarName, "3"); err != nil {
		log.Fatalf("setenv: %v", err)
	}
	if err = os.Setenv(consts.VeyronCredentials, ""); err != nil {
		log.Fatalf("setenv: %v", err)
	}

	if *keypath == "" && passphrase != nil {
		// If we're done with the passphrase, zero it out so it doesn't stay in memory
		for i := range passphrase {
			passphrase[i] = 0
		}
		passphrase = nil
	}

	// Start running our server.
	var sock, mgrSock *os.File
	if sock, err = server.RunAnonymousAgent(runtime, p); err != nil {
		log.Fatalf("RunAnonymousAgent: %v", err)
	}
	if *keypath != "" {
		if mgrSock, err = server.RunKeyManager(runtime, *keypath, passphrase); err != nil {
			log.Fatalf("RunKeyManager: %v", err)
		}
	}

	// Now run the client and wait for it to finish.
	cmd := exec.Command(flag.Args()[0], flag.Args()[1:]...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.ExtraFiles = []*os.File{sock}

	if mgrSock != nil {
		cmd.ExtraFiles = append(cmd.ExtraFiles, mgrSock)
	}

	err = cmd.Start()
	if err != nil {
		log.Fatalf("Error starting child: %v", err)
	}
	sock.Close()
	cmd.Wait()
	status := cmd.ProcessState.Sys().(syscall.WaitStatus)
	os.Exit(status.ExitStatus())
}

func newPrincipalFromDir(dir string) (security.Principal, []byte, error) {
	p, err := vsecurity.LoadPersistentPrincipal(dir, nil)
	if os.IsNotExist(err) {
		return handleDoesNotExist(dir)
	}
	if err == vsecurity.PassphraseErr {
		return handlePassphrase(dir)
	}
	return p, nil, err
}

func handleDoesNotExist(dir string) (security.Principal, []byte, error) {
	fmt.Println("Private key file does not exist. Creating new private key...")
	var pass []byte
	if !*noPassphrase {
		var err error
		if pass, err = getPassword("Enter passphrase (entering nothing will store unencrypted): "); err != nil {
			return nil, nil, fmt.Errorf("failed to read passphrase: %v", err)
		}
	}
	p, err := vsecurity.CreatePersistentPrincipal(dir, pass)
	if err != nil {
		return nil, pass, err
	}
	vsecurity.InitDefaultBlessings(p, "agent_principal")
	return p, pass, nil
}

func handlePassphrase(dir string) (security.Principal, []byte, error) {
	if *noPassphrase {
		return nil, nil, fmt.Errorf("Passphrase required for decrypting principal.")
	}
	pass, err := getPassword("Private key file is encrypted. Please enter passphrase.\nEnter passphrase: ")
	if err != nil {
		return nil, nil, fmt.Errorf("failed to read passphrase: %v", err)
	}
	p, err := vsecurity.LoadPersistentPrincipal(dir, pass)
	return p, pass, err
}

func getPassword(prompt string) ([]byte, error) {
	if !terminal.IsTerminal(int(os.Stdin.Fd())) {
		// If the standard input is not a terminal, the password is obtained by reading a line from it.
		return readPassword()
	}
	fmt.Printf(prompt)
	stop := make(chan bool)
	defer close(stop)
	state, err := terminal.GetState(int(os.Stdin.Fd()))
	if err != nil {
		return nil, err
	}
	go catchTerminationSignals(stop, state)
	defer fmt.Printf("\n")
	return terminal.ReadPassword(int(os.Stdin.Fd()))
}

// readPassword reads form Stdin until it sees '\n' or EOF.
func readPassword() ([]byte, error) {
	var pass []byte
	var total int
	for {
		b := make([]byte, 1)
		count, err := os.Stdin.Read(b)
		if err != nil && err != io.EOF {
			return nil, err
		}
		if err == io.EOF || b[0] == '\n' {
			return pass[:total], nil
		}
		total += count
		pass = secureAppend(pass, b)
	}
}

func secureAppend(s, t []byte) []byte {
	res := append(s, t...)
	if len(res) > cap(s) {
		// When append needs to allocate a new array, clear out the old one.
		for i := range s {
			s[i] = '0'
		}
	}
	// Clear out the second array.
	for i := range t {
		t[i] = '0'
	}
	return res
}

// catchTerminationSignals catches signals to allow us to turn terminal echo back on.
func catchTerminationSignals(stop <-chan bool, state *terminal.State) {
	var successErrno syscall.Errno
	sig := make(chan os.Signal, 4)
	// Catch the blockable termination signals.
	signal.Notify(sig, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGHUP)
	select {
	case <-sig:
		// Start on new line in terminal.
		fmt.Printf("\n")
		if err := terminal.Restore(int(os.Stdin.Fd()), state); err != successErrno {
			vlog.Errorf("Failed to restore terminal state (%v), you words may not show up when you type, enter 'stty echo' to fix this.", err)
		}
		os.Exit(-1)
	case <-stop:
		signal.Stop(sig)
	}
}
<commit_msg>veyron/security/agent/agentd: kill child command when the agent is killed<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"syscall"

	"code.google.com/p/go.crypto/ssh/terminal"

	"veyron.io/veyron/veyron/lib/flags/consts"
	vsignals "veyron.io/veyron/veyron/lib/signals"
	_ "veyron.io/veyron/veyron/profiles"
	vsecurity "veyron.io/veyron/veyron/security"
	"veyron.io/veyron/veyron/security/agent"
	"veyron.io/veyron/veyron/security/agent/server"

	"veyron.io/veyron/veyron2/options"
	"veyron.io/veyron/veyron2/rt"
	"veyron.io/veyron/veyron2/security"
	"veyron.io/veyron/veyron2/vlog"
)

var (
	keypath      = flag.String("additional_principals", "", "If non-empty, allow for the creation of new principals and save them in this directory.")
	noPassphrase = flag.Bool("no_passphrase", false, "If true, user will not be prompted for principal encryption passphrase.")
)

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `Usage: %s [agent options] command command_args...

Loads the private key specified in privatekey.pem in %v into memory, then
starts the specified command with access to the private key via the
agent protocol instead of directly reading from disk.

`, os.Args[0], consts.VeyronCredentials)
		flag.PrintDefaults()
	}
	flag.Parse()
	if len(flag.Args()) < 1 {
		fmt.Fprintln(os.Stderr, "Need at least one argument.")
		flag.Usage()
		os.Exit(1)
	}
	// TODO(ashankar,cnicolaou): Should flags.Parse be used instead? But that adds unnecessary
	// flags like "--veyron.namespace.root", which has no meaning for this binary.
	dir := os.Getenv(consts.VeyronCredentials)
	if len(dir) == 0 {
		vlog.Fatalf("The %v environment variable must be set to a directory", consts.VeyronCredentials)
	}

	p, passphrase, err := newPrincipalFromDir(dir)
	if err != nil {
		vlog.Fatalf("failed to create new principal from dir(%s): %v", dir, err)
	}

	runtime := rt.Init(options.RuntimePrincipal{p})
	log := runtime.Logger()

	if err = os.Setenv(agent.FdVarName, "3"); err != nil {
		log.Fatalf("setenv: %v", err)
	}
	if err = os.Setenv(consts.VeyronCredentials, ""); err != nil {
		log.Fatalf("setenv: %v", err)
	}

	if *keypath == "" && passphrase != nil {
		// If we're done with the passphrase, zero it out so it doesn't stay in memory
		for i := range passphrase {
			passphrase[i] = 0
		}
		passphrase = nil
	}

	// Start running our server.
	var sock, mgrSock *os.File
	if sock, err = server.RunAnonymousAgent(runtime, p); err != nil {
		log.Fatalf("RunAnonymousAgent: %v", err)
	}
	if *keypath != "" {
		if mgrSock, err = server.RunKeyManager(runtime, *keypath, passphrase); err != nil {
			log.Fatalf("RunKeyManager: %v", err)
		}
	}

	// Now run the client and wait for it to finish.
	cmd := exec.Command(flag.Args()[0], flag.Args()[1:]...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.ExtraFiles = []*os.File{sock}

	if mgrSock != nil {
		cmd.ExtraFiles = append(cmd.ExtraFiles, mgrSock)
	}

	err = cmd.Start()
	if err != nil {
		log.Fatalf("Error starting child: %v", err)
	}
	sock.Close()
	shutdown := make(chan struct{})
	go func() {
		select {
		case sig := <-vsignals.ShutdownOnSignals(runtime):
			// TODO(caprita): Should we also relay double signal to
			// the child?  That currently just force exits the
			// current process.
			if sig == vsignals.STOP {
				sig = syscall.SIGTERM
			}
			cmd.Process.Signal(sig)
		case <-shutdown:
		}
	}()
	cmd.Wait()
	close(shutdown)
	status := cmd.ProcessState.Sys().(syscall.WaitStatus)
	os.Exit(status.ExitStatus())
}

func newPrincipalFromDir(dir string) (security.Principal, []byte, error) {
	p, err := vsecurity.LoadPersistentPrincipal(dir, nil)
	if os.IsNotExist(err) {
		return handleDoesNotExist(dir)
	}
	if err == vsecurity.PassphraseErr {
		return handlePassphrase(dir)
	}
	return p, nil, err
}

func handleDoesNotExist(dir string) (security.Principal, []byte, error) {
	fmt.Println("Private key file does not exist. Creating new private key...")
	var pass []byte
	if !*noPassphrase {
		var err error
		if pass, err = getPassword("Enter passphrase (entering nothing will store unencrypted): "); err != nil {
			return nil, nil, fmt.Errorf("failed to read passphrase: %v", err)
		}
	}
	p, err := vsecurity.CreatePersistentPrincipal(dir, pass)
	if err != nil {
		return nil, pass, err
	}
	vsecurity.InitDefaultBlessings(p, "agent_principal")
	return p, pass, nil
}

func handlePassphrase(dir string) (security.Principal, []byte, error) {
	if *noPassphrase {
		return nil, nil, fmt.Errorf("Passphrase required for decrypting principal.")
	}
	pass, err := getPassword("Private key file is encrypted. Please enter passphrase.\nEnter passphrase: ")
	if err != nil {
		return nil, nil, fmt.Errorf("failed to read passphrase: %v", err)
	}
	p, err := vsecurity.LoadPersistentPrincipal(dir, pass)
	return p, pass, err
}

func getPassword(prompt string) ([]byte, error) {
	if !terminal.IsTerminal(int(os.Stdin.Fd())) {
		// If the standard input is not a terminal, the password is obtained by reading a line from it.
		return readPassword()
	}
	fmt.Printf(prompt)
	stop := make(chan bool)
	defer close(stop)
	state, err := terminal.GetState(int(os.Stdin.Fd()))
	if err != nil {
		return nil, err
	}
	go catchTerminationSignals(stop, state)
	defer fmt.Printf("\n")
	return terminal.ReadPassword(int(os.Stdin.Fd()))
}

// readPassword reads form Stdin until it sees '\n' or EOF.
func readPassword() ([]byte, error) {
	var pass []byte
	var total int
	for {
		b := make([]byte, 1)
		count, err := os.Stdin.Read(b)
		if err != nil && err != io.EOF {
			return nil, err
		}
		if err == io.EOF || b[0] == '\n' {
			return pass[:total], nil
		}
		total += count
		pass = secureAppend(pass, b)
	}
}

func secureAppend(s, t []byte) []byte {
	res := append(s, t...)
	if len(res) > cap(s) {
		// When append needs to allocate a new array, clear out the old one.
		for i := range s {
			s[i] = '0'
		}
	}
	// Clear out the second array.
	for i := range t {
		t[i] = '0'
	}
	return res
}

// catchTerminationSignals catches signals to allow us to turn terminal echo back on.
func catchTerminationSignals(stop <-chan bool, state *terminal.State) {
	var successErrno syscall.Errno
	sig := make(chan os.Signal, 4)
	// Catch the blockable termination signals.
	signal.Notify(sig, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGHUP)
	select {
	case <-sig:
		// Start on new line in terminal.
		fmt.Printf("\n")
		if err := terminal.Restore(int(os.Stdin.Fd()), state); err != successErrno {
			vlog.Errorf("Failed to restore terminal state (%v), you words may not show up when you type, enter 'stty echo' to fix this.", err)
		}
		os.Exit(-1)
	case <-stop:
		signal.Stop(sig)
	}
}
<|endoftext|>


<commit_before>// Copyright © 2016, The T Authors.

package editor

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"net/http"
	"strconv"
	"sync"

	"github.com/eaburns/T/edit"
	"github.com/gorilla/mux"
)

// Server implements http.Handler, serving an HTTP text editor.
// It provides an HTTP API for creating buffers of text
// and editors to read and modify those buffers.
//
// Buffers and editors
//
// A buffer is an un-bounded sequence of runes.
// Buffers can be viewed and modified using editors.
// A buffer can have multiple editors,
// but each editor edits only a single buffer.
//
// An editor can view and modify a buffer
// using the T edit language documented here:
// https://godoc.org/github.com/eaburns/T/edit#Ed.
// While multiple editors can edit the same buffer concurrently,
// each editor maintains its own local state.
type Server struct {
	sync.RWMutex
	buffers      map[int]*buffer
	nextBufferID int
}

// NewServer returns a new Server.
func NewServer() *Server { return &Server{buffers: make(map[int]*buffer)} }

// Close closes the server and all of its buffers.
func (s *Server) Close() error {
	var errs []error
	for _, b := range s.buffers {
		errs = append(errs, b.Buffer.Close())
	}
	for _, err := range errs {
		if err != nil {
			return err
		}
	}
	return nil
}

// RegisterHandlers registeres handlers for the following paths and methods:
//  /buffer is the list of opened buffers
//
// 	GET returns a BufferInfo list of the opened buffers.
// 	Returns:
// 	• OK on success.
// 	• Internal Server Error on internal error.
//
// 	PUT creates a new, empty buffer and returns its BufferInfo.
// 	Returns:
// 	• OK on success.
// 	• Internal Server Error on internal error.
//
//  /buffer/<N> is the buffer with ID N
//
// 	GET returns the buffer's BufferInfo
// 	Returns:
// 	• OK on success.
// 	• Internal Server Error on internal error.
// 	• Not Found if the buffer is not found.
// 	  The body is the path to the buffer.
//
// 	DELETE deletes the buffer and all of its editors.
// 	Returns:
// 	• OK on success.
// 	• Internal Server Error on internal error.
// 	• Not Found if the buffer is not found.
// 	  The body is the path to the buffer.
//
//  /buffer/<N>/editor is the list of the buffer's editors.
//
// 	GET returns an EditorInfo list of the buffer's opened editors.
// 	Returns:
// 	• OK on success.
// 	• Internal Server Error on internal error.
// 	• Not Found if the buffer is not found.
// 	  The body is the path to the buffer.
//
// 	PUT creates a new editor for the buffer and returns its EditorInfo.
// 	Returns:
// 	• OK on success.
// 	• Internal Server Error on internal error.
// 	• Not Found if the buffer is not found.
// 	  The body is the path to the buffer.
//
//  /buffer/<N>/editor/<M> is the editor M of buffer N.
//
// 	GET returns the editor's EditorInfo.
// 	Returns:
// 	• OK on success.
// 	• Internal Server Error on internal error.
// 	• Not Found if either the buffer or editor is not found.
// 	  The body is the path to the buffer or editor.
//
// 	DELETE deletes the editor.
// 	Returns:
// 	• OK on success.
// 	• Internal Server Error on internal error.
// 	• Not Found if either the buffer or editor is not found.
// 	  The body is the path to the buffer or editor.
//
// 	POST performs an atomic sequence of edits on the buffer.
// 	The body must be an ordered list of EditRequests.
// 	The response is an ordered list of EditResponses.
// 	Returns:
// 	• OK on success.
// 	• Internal Server Error on internal error.
// 	• Not Found if either the buffer or editor is not found.
// 	  The body is the path to the buffer or editor.
// 	• Bad Request if the EditRequest list is malformed.
//
// Unless otherwise stated, the body of all error responses is the error message.
func (s *Server) RegisterHandlers(r *mux.Router) {
	r.HandleFunc("/buffer", s.listBuffers).Methods(http.MethodGet)
	r.HandleFunc("/buffer", s.newBuffer).Methods(http.MethodPut)
	r.HandleFunc(`/buffer/{bid}`, s.bufferInfo).Methods(http.MethodGet)
	r.HandleFunc(`/buffer/{bid}`, s.closeBuffer).Methods(http.MethodDelete)
	r.HandleFunc(`/buffer/{bid}/editor`, s.listEditors).Methods(http.MethodGet)
	r.HandleFunc(`/buffer/{bid}/editor`, s.newEditor).Methods(http.MethodPut)
	r.HandleFunc(`/buffer/{bid}/editor/{eid}`, s.editorInfo).Methods(http.MethodGet)
	r.HandleFunc(`/buffer/{bid}/editor/{eid}`, s.closeEditor).Methods(http.MethodDelete)
	r.HandleFunc(`/buffer/{bid}/editor/{eid}`, s.edit).Methods(http.MethodPost)
}

func notFound(w http.ResponseWriter, err error) { http.Error(w, err.Error(), http.StatusNotFound) }

// getBufferRLocked returns the opened, read-locked buffer
// with the ID from the URL's bid variable,
// or an error if the buffer is not found.
// The buffer is opened and will not be closed until the lock is released.
func getBufferRLocked(s *Server, req *http.Request) (*buffer, error) {
	s.RLock()
	defer s.RUnlock()
	buf, err := getBufferUnsafe(s, req)
	if err == nil {
		buf.RLock()
	}
	return buf, err
}

// getBufferLocked returns the opened, write-locked buffer
// with the ID from the URL's bid variable,
// or an error if the buffer is not found.
// The buffer is opened and will not be closed until the lock is released.
func getBufferLocked(s *Server, req *http.Request) (*buffer, error) {
	s.RLock()
	defer s.RUnlock()
	buf, err := getBufferUnsafe(s, req)
	if err == nil {
		buf.Lock()
	}
	return buf, err
}

// GetBufferUnsafe is like GetBuffer,
// but requires the caller to take the server lock.
func getBufferUnsafe(s *Server, req *http.Request) (*buffer, error) {
	bid := mux.Vars(req)["bid"]
	id, err := strconv.Atoi(bid)
	if err != nil {
		return nil, errors.New("/buffer/" + bid)
	}
	buf, ok := s.buffers[id]
	if !ok {
		return nil, errors.New("/buffer/" + bid)
	}
	return buf, nil
}

// GetEditor returns the editor with the bid and eid from the URL
// or an error if the editor is not found.
func getEditor(s *Server, req *http.Request) (*editor, error) {
	buf, err := getBufferRLocked(s, req)
	if err != nil {
		return nil, err
	}
	defer buf.RUnlock()

	eid := mux.Vars(req)["eid"]
	id, err := strconv.Atoi(eid)
	if err != nil {
		return nil, errors.New("/buffer/" + strconv.Itoa(buf.ID) + "/editor/" + eid)
	}
	ed, ok := buf.editors[id]
	if !ok {
		return nil, errors.New("/buffer/" + strconv.Itoa(buf.ID) + "/editor/" + eid)
	}
	return ed, nil
}

func (s *Server) listBuffers(w http.ResponseWriter, req *http.Request) {
	s.RLock()
	var bufs []BufferInfo
	for _, b := range s.buffers {
		bufs = append(bufs, b.BufferInfo)
	}
	s.RUnlock()

	if err := json.NewEncoder(w).Encode(bufs); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func (s *Server) newBuffer(w http.ResponseWriter, req *http.Request) {
	s.Lock()
	buf := &buffer{
		BufferInfo: BufferInfo{ID: s.nextBufferID},
		Buffer:     edit.NewBuffer(),
		editors:    make(map[int]*editor),
	}
	s.buffers[buf.ID] = buf
	s.nextBufferID++
	s.Unlock()

	if err := json.NewEncoder(w).Encode(buf.BufferInfo); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func (s *Server) bufferInfo(w http.ResponseWriter, req *http.Request) {
	buf, err := getBufferRLocked(s, req)
	if err != nil {
		notFound(w, err)
		return
	}
	info := buf.BufferInfo
	buf.RUnlock()

	if err := json.NewEncoder(w).Encode(info); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func (s *Server) closeBuffer(w http.ResponseWriter, req *http.Request) {
	s.Lock()
	buf, err := getBufferUnsafe(s, req)
	if err != nil {
		s.Unlock()
		notFound(w, err)
		return
	}
	delete(s.buffers, buf.ID)
	s.Unlock()

	buf.Lock()
	defer buf.Unlock()
	if err := buf.Close(); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func (s *Server) listEditors(w http.ResponseWriter, req *http.Request) {
	buf, err := getBufferRLocked(s, req)
	if err != nil {
		notFound(w, err)
		return
	}
	var eds []EditorInfo
	for _, ed := range buf.editors {
		eds = append(eds, ed.EditorInfo)
	}
	buf.RUnlock()

	if err := json.NewEncoder(w).Encode(eds); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func (s *Server) newEditor(w http.ResponseWriter, req *http.Request) {
	buf, err := getBufferLocked(s, req)
	if err != nil {
		notFound(w, err)
		return
	}
	ed := &editor{
		EditorInfo: EditorInfo{ID: buf.nextEditorID, BufferID: buf.ID},
		buffer:     buf,
		marks:      make(map[rune]edit.Span),
	}
	buf.editors[ed.ID] = ed
	buf.nextEditorID++
	defer buf.Unlock()

	if err := json.NewEncoder(w).Encode(ed.EditorInfo); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func (s *Server) editorInfo(w http.ResponseWriter, req *http.Request) {
	ed, err := getEditor(s, req)
	if err != nil {
		notFound(w, err)
		return
	}
	ed.buffer.RLock()
	info := ed.EditorInfo
	ed.buffer.RUnlock()

	if err := json.NewEncoder(w).Encode(info); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func (s *Server) closeEditor(w http.ResponseWriter, req *http.Request) {
	ed, err := getEditor(s, req)
	if err != nil {
		notFound(w, err)
		return
	}
	ed.buffer.Lock()
	delete(ed.buffer.editors, ed.ID)
	ed.buffer.Unlock()
}

func (s *Server) edit(w http.ResponseWriter, req *http.Request) {
	var edits []EditRequest
	if err := json.NewDecoder(req.Body).Decode(&edits); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	ed, err := getEditor(s, req)
	if err != nil {
		notFound(w, err)
		return
	}
	ed.buffer.Lock()
	var resps []EditResponse
	print := bytes.NewBuffer(nil)
	for _, e := range edits {
		print.Reset()
		err := e.Do(ed, print)
		ed.buffer.Sequence++
		resp := EditResponse{
			Sequence: ed.buffer.Sequence,
			Print:    print.String(),
		}
		if err != nil {
			resp.Error = err.Error()
		}
		resps = append(resps, resp)
	}
	ed.buffer.Unlock()

	if err := json.NewEncoder(w).Encode(resps); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

type buffer struct {
	sync.RWMutex
	BufferInfo
	*edit.Buffer
	editors      map[int]*editor
	nextEditorID int
}

type editor struct {
	EditorInfo
	*buffer
	marks   map[rune]edit.Span
	pending []change
}

type change struct {
	span edit.Span
	size int64
}

func (ed *editor) Mark(m rune) edit.Span { return ed.marks[m] }

func (ed *editor) SetMark(m rune, s edit.Span) error {
	if size := ed.Size(); s[0] < 0 || s[1] < 0 || s[0] > size || s[1] > size {
		return edit.ErrInvalidArgument
	}
	ed.marks[m] = s
	return nil
}

func (ed *editor) Change(s edit.Span, r io.Reader) (int64, error) {
	n, err := ed.Buffer.Change(s, r)
	if err == nil {
		ed.pending = append(ed.pending, change{span: s, size: n})
	}
	return n, err
}

func (ed *editor) Apply() error {
	if err := ed.Buffer.Apply(); err != nil {
		return err
	}
	for _, c := range ed.pending {
		for _, e := range ed.buffer.editors {
			for m, s := range e.marks {
				if e == ed && m == '.' && c.span[0] == s[0] {
					// We handle dot of the current editor specially.
					// If the change has the same start, grow dot.
					// Otherwise, update would simply leave it
					// as a point address and move it.
					dot := e.marks[m]
					dot[1] = dot.Update(c.span, c.size)[1]
					e.marks[m] = dot
				} else {
					e.marks[m] = e.marks[m].Update(c.span, c.size)
				}
			}
		}
	}
	ed.pending = ed.pending[:0]
	return nil
}
<commit_msg>editor: fix buffer locking when getting editors.<commit_after>// Copyright © 2016, The T Authors.

package editor

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"net/http"
	"strconv"
	"sync"

	"github.com/eaburns/T/edit"
	"github.com/gorilla/mux"
)

// Server implements http.Handler, serving an HTTP text editor.
// It provides an HTTP API for creating buffers of text
// and editors to read and modify those buffers.
//
// Buffers and editors
//
// A buffer is an un-bounded sequence of runes.
// Buffers can be viewed and modified using editors.
// A buffer can have multiple editors,
// but each editor edits only a single buffer.
//
// An editor can view and modify a buffer
// using the T edit language documented here:
// https://godoc.org/github.com/eaburns/T/edit#Ed.
// While multiple editors can edit the same buffer concurrently,
// each editor maintains its own local state.
type Server struct {
	sync.RWMutex
	buffers      map[int]*buffer
	nextBufferID int
}

// NewServer returns a new Server.
func NewServer() *Server { return &Server{buffers: make(map[int]*buffer)} }

// Close closes the server and all of its buffers.
func (s *Server) Close() error {
	var errs []error
	for _, b := range s.buffers {
		errs = append(errs, b.Buffer.Close())
	}
	for _, err := range errs {
		if err != nil {
			return err
		}
	}
	return nil
}

// RegisterHandlers registeres handlers for the following paths and methods:
//  /buffer is the list of opened buffers
//
// 	GET returns a BufferInfo list of the opened buffers.
// 	Returns:
// 	• OK on success.
// 	• Internal Server Error on internal error.
//
// 	PUT creates a new, empty buffer and returns its BufferInfo.
// 	Returns:
// 	• OK on success.
// 	• Internal Server Error on internal error.
//
//  /buffer/<N> is the buffer with ID N
//
// 	GET returns the buffer's BufferInfo
// 	Returns:
// 	• OK on success.
// 	• Internal Server Error on internal error.
// 	• Not Found if the buffer is not found.
// 	  The body is the path to the buffer.
//
// 	DELETE deletes the buffer and all of its editors.
// 	Returns:
// 	• OK on success.
// 	• Internal Server Error on internal error.
// 	• Not Found if the buffer is not found.
// 	  The body is the path to the buffer.
//
//  /buffer/<N>/editor is the list of the buffer's editors.
//
// 	GET returns an EditorInfo list of the buffer's opened editors.
// 	Returns:
// 	• OK on success.
// 	• Internal Server Error on internal error.
// 	• Not Found if the buffer is not found.
// 	  The body is the path to the buffer.
//
// 	PUT creates a new editor for the buffer and returns its EditorInfo.
// 	Returns:
// 	• OK on success.
// 	• Internal Server Error on internal error.
// 	• Not Found if the buffer is not found.
// 	  The body is the path to the buffer.
//
//  /buffer/<N>/editor/<M> is the editor M of buffer N.
//
// 	GET returns the editor's EditorInfo.
// 	Returns:
// 	• OK on success.
// 	• Internal Server Error on internal error.
// 	• Not Found if either the buffer or editor is not found.
// 	  The body is the path to the buffer or editor.
//
// 	DELETE deletes the editor.
// 	Returns:
// 	• OK on success.
// 	• Internal Server Error on internal error.
// 	• Not Found if either the buffer or editor is not found.
// 	  The body is the path to the buffer or editor.
//
// 	POST performs an atomic sequence of edits on the buffer.
// 	The body must be an ordered list of EditRequests.
// 	The response is an ordered list of EditResponses.
// 	Returns:
// 	• OK on success.
// 	• Internal Server Error on internal error.
// 	• Not Found if either the buffer or editor is not found.
// 	  The body is the path to the buffer or editor.
// 	• Bad Request if the EditRequest list is malformed.
//
// Unless otherwise stated, the body of all error responses is the error message.
func (s *Server) RegisterHandlers(r *mux.Router) {
	r.HandleFunc("/buffer", s.listBuffers).Methods(http.MethodGet)
	r.HandleFunc("/buffer", s.newBuffer).Methods(http.MethodPut)
	r.HandleFunc(`/buffer/{bid}`, s.bufferInfo).Methods(http.MethodGet)
	r.HandleFunc(`/buffer/{bid}`, s.closeBuffer).Methods(http.MethodDelete)
	r.HandleFunc(`/buffer/{bid}/editor`, s.listEditors).Methods(http.MethodGet)
	r.HandleFunc(`/buffer/{bid}/editor`, s.newEditor).Methods(http.MethodPut)
	r.HandleFunc(`/buffer/{bid}/editor/{eid}`, s.editorInfo).Methods(http.MethodGet)
	r.HandleFunc(`/buffer/{bid}/editor/{eid}`, s.closeEditor).Methods(http.MethodDelete)
	r.HandleFunc(`/buffer/{bid}/editor/{eid}`, s.edit).Methods(http.MethodPost)
}

func notFound(w http.ResponseWriter, err error) { http.Error(w, err.Error(), http.StatusNotFound) }

// getBufferRLocked returns the opened, read-locked buffer
// with the ID from the URL's bid variable,
// or an error if the buffer is not found.
// The buffer is opened and will not be closed until the lock is released.
func getBufferRLocked(s *Server, req *http.Request) (*buffer, error) {
	s.RLock()
	defer s.RUnlock()
	buf, err := getBufferUnsafe(s, req)
	if err == nil {
		buf.RLock()
	}
	return buf, err
}

// getBufferLocked returns the opened, write-locked buffer
// with the ID from the URL's bid variable,
// or an error if the buffer is not found.
// The buffer is opened and will not be closed until the lock is released.
func getBufferLocked(s *Server, req *http.Request) (*buffer, error) {
	s.RLock()
	defer s.RUnlock()
	buf, err := getBufferUnsafe(s, req)
	if err == nil {
		buf.Lock()
	}
	return buf, err
}

// GetBufferUnsafe is like GetBuffer,
// but requires the caller to take the server lock.
func getBufferUnsafe(s *Server, req *http.Request) (*buffer, error) {
	bid := mux.Vars(req)["bid"]
	id, err := strconv.Atoi(bid)
	if err != nil {
		return nil, errors.New("/buffer/" + bid)
	}
	buf, ok := s.buffers[id]
	if !ok {
		return nil, errors.New("/buffer/" + bid)
	}
	return buf, nil
}

// getEditorRLocked returns the editor with the eid from the URL
// for the open, read-locked buffer with bid from the URL
// or an error if the editor is not found.
func getEditorRLocked(s *Server, req *http.Request) (*editor, error) {
	s.RLock()
	defer s.RUnlock()
	ed, err := getEditorUnsafe(s, req)
	if err == nil {
		ed.buffer.RLock()
	}
	return ed, err
}

// getEditorLocked returns the editor with the eid from the URL
// for the open, write-locked buffer with bid from the URL
// or an error if the editor is not found.
func getEditorLocked(s *Server, req *http.Request) (*editor, error) {
	s.RLock()
	defer s.RUnlock()
	ed, err := getEditorUnsafe(s, req)
	if err == nil {
		ed.buffer.Lock()
	}
	return ed, err
}

// getEditorUnsafe returns the editor with bid and eid from the URL
// or an error if the editor is not found.
// The caller must hold the buffer's Lock or RLock.
func getEditorUnsafe(s *Server, req *http.Request) (*editor, error) {
	buf, err := getBufferUnsafe(s, req)
	if err != nil {
		return nil, err
	}
	eid := mux.Vars(req)["eid"]
	id, err := strconv.Atoi(eid)
	if err != nil {
		return nil, errors.New("/buffer/" + strconv.Itoa(buf.ID) + "/editor/" + eid)
	}
	ed, ok := buf.editors[id]
	if !ok {
		return nil, errors.New("/buffer/" + strconv.Itoa(buf.ID) + "/editor/" + eid)
	}
	return ed, nil
}

func (s *Server) listBuffers(w http.ResponseWriter, req *http.Request) {
	s.RLock()
	var bufs []BufferInfo
	for _, b := range s.buffers {
		bufs = append(bufs, b.BufferInfo)
	}
	s.RUnlock()

	if err := json.NewEncoder(w).Encode(bufs); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func (s *Server) newBuffer(w http.ResponseWriter, req *http.Request) {
	s.Lock()
	buf := &buffer{
		BufferInfo: BufferInfo{ID: s.nextBufferID},
		Buffer:     edit.NewBuffer(),
		editors:    make(map[int]*editor),
	}
	s.buffers[buf.ID] = buf
	s.nextBufferID++
	s.Unlock()

	if err := json.NewEncoder(w).Encode(buf.BufferInfo); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func (s *Server) bufferInfo(w http.ResponseWriter, req *http.Request) {
	buf, err := getBufferRLocked(s, req)
	if err != nil {
		notFound(w, err)
		return
	}
	info := buf.BufferInfo
	buf.RUnlock()

	if err := json.NewEncoder(w).Encode(info); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func (s *Server) closeBuffer(w http.ResponseWriter, req *http.Request) {
	s.Lock()
	buf, err := getBufferUnsafe(s, req)
	if err != nil {
		s.Unlock()
		notFound(w, err)
		return
	}
	delete(s.buffers, buf.ID)
	s.Unlock()

	buf.Lock()
	defer buf.Unlock()
	if err := buf.Close(); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func (s *Server) listEditors(w http.ResponseWriter, req *http.Request) {
	buf, err := getBufferRLocked(s, req)
	if err != nil {
		notFound(w, err)
		return
	}
	var eds []EditorInfo
	for _, ed := range buf.editors {
		eds = append(eds, ed.EditorInfo)
	}
	buf.RUnlock()

	if err := json.NewEncoder(w).Encode(eds); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func (s *Server) newEditor(w http.ResponseWriter, req *http.Request) {
	buf, err := getBufferLocked(s, req)
	if err != nil {
		notFound(w, err)
		return
	}
	ed := &editor{
		EditorInfo: EditorInfo{ID: buf.nextEditorID, BufferID: buf.ID},
		buffer:     buf,
		marks:      make(map[rune]edit.Span),
	}
	buf.editors[ed.ID] = ed
	buf.nextEditorID++
	defer buf.Unlock()

	if err := json.NewEncoder(w).Encode(ed.EditorInfo); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func (s *Server) editorInfo(w http.ResponseWriter, req *http.Request) {
	ed, err := getEditorRLocked(s, req)
	if err != nil {
		notFound(w, err)
		return
	}
	info := ed.EditorInfo
	ed.buffer.RUnlock()

	if err := json.NewEncoder(w).Encode(info); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func (s *Server) closeEditor(w http.ResponseWriter, req *http.Request) {
	ed, err := getEditorLocked(s, req)
	if err != nil {
		notFound(w, err)
		return
	}
	delete(ed.buffer.editors, ed.ID)
	ed.buffer.Unlock()
}

func (s *Server) edit(w http.ResponseWriter, req *http.Request) {
	var edits []EditRequest
	if err := json.NewDecoder(req.Body).Decode(&edits); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	ed, err := getEditorLocked(s, req)
	if err != nil {
		notFound(w, err)
		return
	}
	var resps []EditResponse
	print := bytes.NewBuffer(nil)
	for _, e := range edits {
		print.Reset()
		err := e.Do(ed, print)
		ed.buffer.Sequence++
		resp := EditResponse{
			Sequence: ed.buffer.Sequence,
			Print:    print.String(),
		}
		if err != nil {
			resp.Error = err.Error()
		}
		resps = append(resps, resp)
	}
	ed.buffer.Unlock()

	if err := json.NewEncoder(w).Encode(resps); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

type buffer struct {
	sync.RWMutex
	BufferInfo
	*edit.Buffer
	editors      map[int]*editor
	nextEditorID int
}

type editor struct {
	EditorInfo
	*buffer
	marks   map[rune]edit.Span
	pending []change
}

type change struct {
	span edit.Span
	size int64
}

func (ed *editor) Mark(m rune) edit.Span { return ed.marks[m] }

func (ed *editor) SetMark(m rune, s edit.Span) error {
	if size := ed.Size(); s[0] < 0 || s[1] < 0 || s[0] > size || s[1] > size {
		return edit.ErrInvalidArgument
	}
	ed.marks[m] = s
	return nil
}

func (ed *editor) Change(s edit.Span, r io.Reader) (int64, error) {
	n, err := ed.Buffer.Change(s, r)
	if err == nil {
		ed.pending = append(ed.pending, change{span: s, size: n})
	}
	return n, err
}

func (ed *editor) Apply() error {
	if err := ed.Buffer.Apply(); err != nil {
		return err
	}
	for _, c := range ed.pending {
		for _, e := range ed.buffer.editors {
			for m, s := range e.marks {
				if e == ed && m == '.' && c.span[0] == s[0] {
					// We handle dot of the current editor specially.
					// If the change has the same start, grow dot.
					// Otherwise, update would simply leave it
					// as a point address and move it.
					dot := e.marks[m]
					dot[1] = dot.Update(c.span, c.size)[1]
					e.marks[m] = dot
				} else {
					e.marks[m] = e.marks[m].Update(c.span, c.size)
				}
			}
		}
	}
	ed.pending = ed.pending[:0]
	return nil
}
<|endoftext|>