<commit_before>// AppEngine-based server to sanity check byte arrays
// that are supposed to be random.
package randomsanity

import (
	"appengine"
	"encoding/hex"
	"fmt"
	"net/http"
	"strings"
	"time"
)

func init() {
	// Main API point, sanity check hex bytes
	http.HandleFunc("/v1/q/", submitBytesHandler)

	// Start an email loop to get an id token, to be
	// notified via email of failures:
	http.HandleFunc("/v1/registeremail/", registerEmailHandler)

	// Remove an id token
	http.HandleFunc("/v1/unregister/", unRegisterIDHandler)

	// Development/testing...
	http.HandleFunc("/v1/debug", debugHandler)

	// Redirect to www. home page
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/" {
			http.NotFound(w, r)
			return
		}
		http.Redirect(w, r, "http://www.randomsanity.org/", 301)
	})
}

func debugHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/plain")

	// Code useful for development/testing:

	//	fmt.Fprint(w, "***r.Header headers***\n")
	//	r.Header.Write(w)

	//	ctx := appengine.NewContext(r)
	//	fmt.Fprint(w, "Usage data:\n")
	//	for _, u := range GetUsage(ctx) {
	//		fmt.Fprintf(w, "%s,%d\n", u.Key, u.N)
	//	}
}

func submitBytesHandler(w http.ResponseWriter, r *http.Request) {
	parts := strings.Split(r.URL.Path, "/")
	if len(parts) != 4 {
		http.Error(w, "Invalid GET", http.StatusBadRequest)
		return
	}
	b, err := hex.DecodeString(parts[len(parts)-1])
	if err != nil {
		http.Error(w, "Invalid hex", http.StatusBadRequest)
		return
	}
	// Need at least 16 bytes to hit the 1-in-2^60 false positive rate
	if len(b) < 16 {
		http.Error(w, "Must provide 16 or more bytes", http.StatusBadRequest)
		return
	}

	ctx := appengine.NewContext(r)

	// Users that register can append id=....&tag=.... so
	// they're notified if somebody else submits
	// the same random bytes
	uID := r.FormValue("id")
	dbKey, _ := userID(ctx, uID)
	tag := ""
	if dbKey == nil {
		uID = ""
	} else {
		tag = r.FormValue("tag")
		if len(tag) > 64 {
			tag = "" // Tags must be short
		}
	}

	// Rate-limit by IP address, with a much higher limit for registered users
	// If more complicated logic is needed because of abuse a per-user limit
	// could be stored in the datastore, but running into the 600-per-hour-per-ip
	// limit should be rare (maybe a sysadmin has 200 virtual machines
	// behind the same IP address and restarts them more than three times in a hour....)
	var ratelimit uint64 = 60
	if len(uID) > 0 {
		ratelimit = 600
	}
	limited, err := RateLimitResponse(ctx, w, IPKey("q", r.RemoteAddr), ratelimit, time.Hour)
	if err != nil || limited {
		return
	}

	w.Header().Add("Content-Type", "application/json")

	// Returns some randomness caller can use to mix in to
	// their PRNG:
	addEntropyHeader(w)

	// First, some simple tests for non-random input:
	result, reason := LooksRandom(b)
	if !result {
		RecordUsage(ctx, "Fail_"+reason, 1)
		fmt.Fprint(w, "false")
		notify(ctx, uID, tag, b, reason)
		return
	}

	// Try to catch two machines with insufficient starting
	// entropy generating identical streams of random bytes.
	if len(b) > 64 {
		b = b[0:64] // Prevent DoS from excessive datastore lookups
	}
	unique, err := looksUnique(ctx, w, b, uID, tag)
	if err != nil {
		return
	}
	if unique {
		RecordUsage(ctx, "Success", 1)
		fmt.Fprint(w, "true")
	} else {
		RecordUsage(ctx, "Fail_Nonunique", 1)
		fmt.Fprint(w, "false")
	}
}
<commit_msg>Redirect to https version of website<commit_after>// AppEngine-based server to sanity check byte arrays
// that are supposed to be random.
package randomsanity

import (
	"appengine"
	"encoding/hex"
	"fmt"
	"net/http"
	"strings"
	"time"
)

func init() {
	// Main API point, sanity check hex bytes
	http.HandleFunc("/v1/q/", submitBytesHandler)

	// Start an email loop to get an id token, to be
	// notified via email of failures:
	http.HandleFunc("/v1/registeremail/", registerEmailHandler)

	// Remove an id token
	http.HandleFunc("/v1/unregister/", unRegisterIDHandler)

	// Development/testing...
	http.HandleFunc("/v1/debug", debugHandler)

	// Redirect to www. home page
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/" {
			http.NotFound(w, r)
			return
		}
		http.Redirect(w, r, "https://www.randomsanity.org/", 301)
	})
}

func debugHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/plain")

	// Code useful for development/testing:

	//	fmt.Fprint(w, "***r.Header headers***\n")
	//	r.Header.Write(w)

	//	ctx := appengine.NewContext(r)
	//	fmt.Fprint(w, "Usage data:\n")
	//	for _, u := range GetUsage(ctx) {
	//		fmt.Fprintf(w, "%s,%d\n", u.Key, u.N)
	//	}
}

func submitBytesHandler(w http.ResponseWriter, r *http.Request) {
	parts := strings.Split(r.URL.Path, "/")
	if len(parts) != 4 {
		http.Error(w, "Invalid GET", http.StatusBadRequest)
		return
	}
	b, err := hex.DecodeString(parts[len(parts)-1])
	if err != nil {
		http.Error(w, "Invalid hex", http.StatusBadRequest)
		return
	}
	// Need at least 16 bytes to hit the 1-in-2^60 false positive rate
	if len(b) < 16 {
		http.Error(w, "Must provide 16 or more bytes", http.StatusBadRequest)
		return
	}

	ctx := appengine.NewContext(r)

	// Users that register can append id=....&tag=.... so
	// they're notified if somebody else submits
	// the same random bytes
	uID := r.FormValue("id")
	dbKey, _ := userID(ctx, uID)
	tag := ""
	if dbKey == nil {
		uID = ""
	} else {
		tag = r.FormValue("tag")
		if len(tag) > 64 {
			tag = "" // Tags must be short
		}
	}

	// Rate-limit by IP address, with a much higher limit for registered users
	// If more complicated logic is needed because of abuse a per-user limit
	// could be stored in the datastore, but running into the 600-per-hour-per-ip
	// limit should be rare (maybe a sysadmin has 200 virtual machines
	// behind the same IP address and restarts them more than three times in a hour....)
	var ratelimit uint64 = 60
	if len(uID) > 0 {
		ratelimit = 600
	}
	limited, err := RateLimitResponse(ctx, w, IPKey("q", r.RemoteAddr), ratelimit, time.Hour)
	if err != nil || limited {
		return
	}

	w.Header().Add("Content-Type", "application/json")

	// Returns some randomness caller can use to mix in to
	// their PRNG:
	addEntropyHeader(w)

	// First, some simple tests for non-random input:
	result, reason := LooksRandom(b)
	if !result {
		RecordUsage(ctx, "Fail_"+reason, 1)
		fmt.Fprint(w, "false")
		notify(ctx, uID, tag, b, reason)
		return
	}

	// Try to catch two machines with insufficient starting
	// entropy generating identical streams of random bytes.
	if len(b) > 64 {
		b = b[0:64] // Prevent DoS from excessive datastore lookups
	}
	unique, err := looksUnique(ctx, w, b, uID, tag)
	if err != nil {
		return
	}
	if unique {
		RecordUsage(ctx, "Success", 1)
		fmt.Fprint(w, "true")
	} else {
		RecordUsage(ctx, "Fail_Nonunique", 1)
		fmt.Fprint(w, "false")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha256"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"

	"github.com/lxc/lxd/shared"

	log "gopkg.in/inconshreveable/log15.v2"
)

const (
	APPARMOR_CMD_LOAD   = "r"
	APPARMOR_CMD_UNLOAD = "R"
	APPARMOR_CMD_PARSE  = "Q"
)

var aaPath = shared.VarPath("security", "apparmor")

const NESTING_AA_PROFILE = `
  pivot_root,
  mount /var/lib/lxd/shmounts/ -> /var/lib/lxd/shmounts/,
  mount none -> /var/lib/lxd/shmounts/,
  mount fstype=proc -> /usr/lib/x86_64-linux-gnu/lxc/**,
  mount fstype=sysfs -> /usr/lib/x86_64-linux-gnu/lxc/**,
  mount options=(rw,bind),
  mount options=(rw,rbind),
  deny /dev/.lxd/proc/** rw,
  deny /dev/.lxd/sys/** rw,
  mount options=(rw,make-rshared),

  # there doesn't seem to be a way to ask for:
  # mount options=(ro,nosuid,nodev,noexec,remount,bind),
  # as we always get mount to $cdir/proc/sys with those flags denied
  # So allow all mounts until that is straightened out:
  mount,
  mount options=bind /var/lib/lxd/shmounts/** -> /var/lib/lxd/**,
  change_profile -> lxc-container-default,
  # lxc-container-default-with-nesting also inherited these
  # from start-container, and seems to need them.
  ptrace,
  signal,
`

const DEFAULT_AA_PROFILE = `
#include <tunables/global>
profile "%s" flags=(attach_disconnected,mediate_deleted) {
    #include <abstractions/lxc/container-base>

    # user input raw.apparmor below here
    %s

    # nesting support goes here if needed
    %s
    change_profile -> "%s",
}`

func AAProfileFull(c *containerLXD) string {
	lxddir := shared.VarPath("")
	if len(c.name)+len(lxddir)+7 >= 253 {
		hash := sha256.New()
		io.WriteString(hash, lxddir)
		lxddir = fmt.Sprintf("%x", hash.Sum(nil))
	}

	return fmt.Sprintf("lxd-%s_<%s>", c.name, lxddir)
}

func AAProfileShort(c *containerLXD) string {
	return fmt.Sprintf("lxd-%s", c.name)
}

// getProfileContent generates the apparmor profile template from the given
// container. This includes the stock lxc includes as well as stuff from
// raw.apparmor.
func getAAProfileContent(c *containerLXD) string {
	rawApparmor, ok := c.config["raw.apparmor"]
	if !ok {
		rawApparmor = ""
	}

	nesting := ""
	if c.IsNesting() {
		nesting = NESTING_AA_PROFILE
	}

	return fmt.Sprintf(DEFAULT_AA_PROFILE, AAProfileFull(c), rawApparmor, nesting, AAProfileFull(c))
}

func runApparmor(command string, c *containerLXD) error {
	if aaConfined() {
		shared.Log.Debug("Already apparmor-confined (nested?), skipping aa profile actions")
		return nil
	}

	cmd := exec.Command("apparmor_parser", []string{
		fmt.Sprintf("-%sWL", command),
		path.Join(aaPath, "cache"),
		path.Join(aaPath, "profiles", AAProfileShort(c)),
	}...)

	output, err := cmd.CombinedOutput()
	if err != nil {
		shared.Log.Error("Running apparmor",
			log.Ctx{"action": command, "output": string(output), "err": err})
	}

	return err
}

/*
 * lxd could be confined by some other profile, but we'll only support
 * running under lxc-container-default-with-nesting or lxd-*
 */
func aaConfined() bool {
	curProfile := aaProfile()

	switch {
	case strings.HasPrefix(curProfile, "lxc-container-default-with-nesting"):
		return true
	case strings.HasPrefix(curProfile, "lxd-"):
		return true
	}

	return false
}

// Ensure that the container's policy is loaded into the kernel so the
// container can boot.
func AALoadProfile(c *containerLXD) error {
	if !aaEnabled {
		shared.Log.Debug("Apparmor not enabled, skipping profile load")
		return nil
	}

	/* In order to avoid forcing a profile parse (potentially slow) on
	 * every container start, let's use apparmor's binary policy cache,
	 * which checks mtime of the files to figure out if the policy needs to
	 * be regenerated.
	 *
	 * Since it uses mtimes, we shouldn't just always write out our local
	 * apparmor template; instead we should check to see whether the
	 * template is the same as ours. If it isn't we should write our
	 * version out so that the new changes are reflected and we definitely
	 * force a recompile.
	 */
	profile := path.Join(aaPath, "profiles", AAProfileShort(c))
	content, err := ioutil.ReadFile(profile)
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	updated := getAAProfileContent(c)

	if string(content) != string(updated) {
		if err := os.MkdirAll(path.Join(aaPath, "profiles"), 0700); err != nil {
			return err
		}

		if err := ioutil.WriteFile(profile, []byte(updated), 0600); err != nil {
			return err
		}
	}

	return runApparmor(APPARMOR_CMD_LOAD, c)
}

// Ensure that the container's policy is unloaded to free kernel memory. This
// does not delete the policy from disk or cache.
func AAUnloadProfile(c *containerLXD) error {
	if !aaEnabled {
		shared.Log.Debug("Apparmor not enabled, skipping profile unload")
		return nil
	}

	return runApparmor(APPARMOR_CMD_UNLOAD, c)
}

// Parse the profile without loading it into the kernel.
func AAParseProfile(c *containerLXD) error {
	if !aaEnabled {
		shared.Log.Debug("Apparmor not enabled, skipping profile parse")
		return nil
	}

	return runApparmor(APPARMOR_CMD_PARSE, c)
}

// Delete the policy from cache/disk.
func AADeleteProfile(c *containerLXD) {
	if !aaEnabled {
		shared.Log.Debug("Apparmor not enabled, skipping profile delete")
		return
	}

	/* It's ok if these deletes fail: if the container was never started,
	 * we'll have never written a profile or cached it.
	 */
	os.Remove(path.Join(aaPath, "cache", AAProfileShort(c)))
	os.Remove(path.Join(aaPath, "profiles", AAProfileShort(c)))
}

// What's current apparmor profile
func aaProfile() string {
	contents, err := ioutil.ReadFile("/proc/self/attr/current")
	if err == nil {
		return strings.TrimSpace(string(contents))
	}
	return ""
}
<commit_msg>drop extra change_profile call in nesting case<commit_after>package main

import (
	"crypto/sha256"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"

	"github.com/lxc/lxd/shared"

	log "gopkg.in/inconshreveable/log15.v2"
)

const (
	APPARMOR_CMD_LOAD   = "r"
	APPARMOR_CMD_UNLOAD = "R"
	APPARMOR_CMD_PARSE  = "Q"
)

var aaPath = shared.VarPath("security", "apparmor")

const NESTING_AA_PROFILE = `
  pivot_root,
  mount /var/lib/lxd/shmounts/ -> /var/lib/lxd/shmounts/,
  mount none -> /var/lib/lxd/shmounts/,
  mount fstype=proc -> /usr/lib/x86_64-linux-gnu/lxc/**,
  mount fstype=sysfs -> /usr/lib/x86_64-linux-gnu/lxc/**,
  mount options=(rw,bind),
  mount options=(rw,rbind),
  deny /dev/.lxd/proc/** rw,
  deny /dev/.lxd/sys/** rw,
  mount options=(rw,make-rshared),

  # there doesn't seem to be a way to ask for:
  # mount options=(ro,nosuid,nodev,noexec,remount,bind),
  # as we always get mount to $cdir/proc/sys with those flags denied
  # So allow all mounts until that is straightened out:
  mount,
  mount options=bind /var/lib/lxd/shmounts/** -> /var/lib/lxd/**,
  # lxc-container-default-with-nesting also inherited these
  # from start-container, and seems to need them.
  ptrace,
  signal,
`

const DEFAULT_AA_PROFILE = `
#include <tunables/global>
profile "%s" flags=(attach_disconnected,mediate_deleted) {
    #include <abstractions/lxc/container-base>

    # user input raw.apparmor below here
    %s

    # nesting support goes here if needed
    %s
    change_profile -> "%s",
}`

func AAProfileFull(c *containerLXD) string {
	lxddir := shared.VarPath("")
	if len(c.name)+len(lxddir)+7 >= 253 {
		hash := sha256.New()
		io.WriteString(hash, lxddir)
		lxddir = fmt.Sprintf("%x", hash.Sum(nil))
	}

	return fmt.Sprintf("lxd-%s_<%s>", c.name, lxddir)
}

func AAProfileShort(c *containerLXD) string {
	return fmt.Sprintf("lxd-%s", c.name)
}

// getProfileContent generates the apparmor profile template from the given
// container. This includes the stock lxc includes as well as stuff from
// raw.apparmor.
func getAAProfileContent(c *containerLXD) string {
	rawApparmor, ok := c.config["raw.apparmor"]
	if !ok {
		rawApparmor = ""
	}

	nesting := ""
	if c.IsNesting() {
		nesting = NESTING_AA_PROFILE
	}

	return fmt.Sprintf(DEFAULT_AA_PROFILE, AAProfileFull(c), rawApparmor, nesting, AAProfileFull(c))
}

func runApparmor(command string, c *containerLXD) error {
	if aaConfined() {
		shared.Log.Debug("Already apparmor-confined (nested?), skipping aa profile actions")
		return nil
	}

	cmd := exec.Command("apparmor_parser", []string{
		fmt.Sprintf("-%sWL", command),
		path.Join(aaPath, "cache"),
		path.Join(aaPath, "profiles", AAProfileShort(c)),
	}...)

	output, err := cmd.CombinedOutput()
	if err != nil {
		shared.Log.Error("Running apparmor",
			log.Ctx{"action": command, "output": string(output), "err": err})
	}

	return err
}

/*
 * lxd could be confined by some other profile, but we'll only support
 * running under lxc-container-default-with-nesting or lxd-*
 */
func aaConfined() bool {
	curProfile := aaProfile()

	switch {
	case strings.HasPrefix(curProfile, "lxc-container-default-with-nesting"):
		return true
	case strings.HasPrefix(curProfile, "lxd-"):
		return true
	}

	return false
}

// Ensure that the container's policy is loaded into the kernel so the
// container can boot.
func AALoadProfile(c *containerLXD) error {
	if !aaEnabled {
		shared.Log.Debug("Apparmor not enabled, skipping profile load")
		return nil
	}

	/* In order to avoid forcing a profile parse (potentially slow) on
	 * every container start, let's use apparmor's binary policy cache,
	 * which checks mtime of the files to figure out if the policy needs to
	 * be regenerated.
	 *
	 * Since it uses mtimes, we shouldn't just always write out our local
	 * apparmor template; instead we should check to see whether the
	 * template is the same as ours. If it isn't we should write our
	 * version out so that the new changes are reflected and we definitely
	 * force a recompile.
	 */
	profile := path.Join(aaPath, "profiles", AAProfileShort(c))
	content, err := ioutil.ReadFile(profile)
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	updated := getAAProfileContent(c)

	if string(content) != string(updated) {
		if err := os.MkdirAll(path.Join(aaPath, "profiles"), 0700); err != nil {
			return err
		}

		if err := ioutil.WriteFile(profile, []byte(updated), 0600); err != nil {
			return err
		}
	}

	return runApparmor(APPARMOR_CMD_LOAD, c)
}

// Ensure that the container's policy is unloaded to free kernel memory. This
// does not delete the policy from disk or cache.
func AAUnloadProfile(c *containerLXD) error {
	if !aaEnabled {
		shared.Log.Debug("Apparmor not enabled, skipping profile unload")
		return nil
	}

	return runApparmor(APPARMOR_CMD_UNLOAD, c)
}

// Parse the profile without loading it into the kernel.
func AAParseProfile(c *containerLXD) error {
	if !aaEnabled {
		shared.Log.Debug("Apparmor not enabled, skipping profile parse")
		return nil
	}

	return runApparmor(APPARMOR_CMD_PARSE, c)
}

// Delete the policy from cache/disk.
func AADeleteProfile(c *containerLXD) {
	if !aaEnabled {
		shared.Log.Debug("Apparmor not enabled, skipping profile delete")
		return
	}

	/* It's ok if these deletes fail: if the container was never started,
	 * we'll have never written a profile or cached it.
	 */
	os.Remove(path.Join(aaPath, "cache", AAProfileShort(c)))
	os.Remove(path.Join(aaPath, "profiles", AAProfileShort(c)))
}

// What's current apparmor profile
func aaProfile() string {
	contents, err := ioutil.ReadFile("/proc/self/attr/current")
	if err == nil {
		return strings.TrimSpace(string(contents))
	}
	return ""
}
<|endoftext|>


<commit_before>// Package lzma2 provides a reader and a writer for the LZMA2 encoding.
// LZMA2 is a framing format for LZMA raw streams to support flushing,
// parallel compression and uncompressed segments.
//
// The Reader and Writer allows the reading and writing of LZMA2 chunk
// sequences. They can be used to parallel compress or decompress LZMA2
// streams.
package lzma2

import (
	"errors"
	"io"

	"github.com/ulikunitz/xz/lzma"
)

// ReaderParams defines the LZMA2 reader parameters.
type ReaderParams struct {
	DictCap int
}

// ReaderDefaults define the defaults for the reader parameters.
var ReaderDefaults = ReaderParams{
	DictCap: 8 * 1024 * 1024,
}

// Verify verifies the LZMA2 reader parameters for correctness.
func (p *ReaderParams) Verify() error {
	return verifyDictCap(p.DictCap)
}

// Reader supports the reading of LZMA2 chunk sequences. Note that the
// first chunk should have a dictionary reset and the first compressed
// chunk a properties reset. The chunk sequence may not be terminated by
// an end-of-stream chunk.
type Reader struct {
	r   io.Reader
	err error

	dict        *lzma.DecoderDict
	ur          *uncompressedReader
	decoder     *lzma.Decoder
	chunkReader io.Reader

	cstate chunkState
	ctype  chunkType
}

// NewReader creates a reader for an LZMA2 chunk sequence with the given
// dictionary capacity.
func NewReader(lzma2 io.Reader, dictCap int) (r *Reader, err error) {
	params := ReaderDefaults
	params.DictCap = dictCap
	return NewReaderParams(lzma2, &params)
}

// NewReaderParams creates a new LZMA2 reader using the given
// parameters.
func NewReaderParams(lzma2 io.Reader, params *ReaderParams) (r *Reader, err error) {
	if err = params.Verify(); err != nil {
		return nil, err
	}
	r = &Reader{
		r:      lzma2,
		cstate: start,
	}
	r.dict, err = lzma.NewDecoderDict(params.DictCap)
	if err != nil {
		return nil, err
	}
	if err = r.startChunk(); err != nil {
		return nil, err
	}
	return r, nil
}

// uncompressed tests whether the chunk type specifies an uncompressed
// chunk.
func uncompressed(ctype chunkType) bool {
	return ctype == cU || ctype == cUD
}

// startChunk parses a new chunk.
func (r *Reader) startChunk() error {
	r.chunkReader = nil
	header, err := readChunkHeader(r.r)
	if err != nil {
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
		}
		return err
	}
	if err = r.cstate.next(header.ctype); err != nil {
		return err
	}
	if r.cstate == stop {
		return io.EOF
	}
	if header.ctype == cUD || header.ctype == cLRND {
		r.dict.Reset()
	}
	size := int64(header.uncompressed) + 1
	if uncompressed(header.ctype) {
		if r.ur != nil {
			r.ur.Reopen(r.r, size)
		} else {
			r.ur = newUncompressedReader(r.r, r.dict, size)
		}
		r.chunkReader = r.ur
		return nil
	}
	br := lzma.ByteReader(io.LimitReader(r.r, int64(header.compressed)+1))
	if r.decoder == nil {
		state := lzma.NewState(header.props)
		r.decoder, err = lzma.NewDecoder(br, state, r.dict, size)
		if err != nil {
			return err
		}
		r.chunkReader = r.decoder
		return nil
	}
	switch header.ctype {
	case cLR:
		r.decoder.State.Reset()
	case cLRN, cLRND:
		r.decoder.State = lzma.NewState(header.props)
	}
	err = r.decoder.Reopen(br, size)
	if err != nil {
		return err
	}
	r.chunkReader = r.decoder
	return nil
}

// Read reads data from the LZMA2 chunk sequence.
func (r *Reader) Read(p []byte) (n int, err error) {
	if r.err != nil {
		return 0, r.err
	}
	for n < len(p) {
		var k int
		k, err = r.chunkReader.Read(p[n:])
		n += k
		if err == io.EOF {
			err = r.startChunk()
		}
		if err != nil {
			r.err = err
			return n, err
		}
		if k == 0 {
			r.err = errors.New("lzma2: Reader doesn't get data")
			return n, r.err
		}
	}
	return n, nil
}

// EOS returns whether the LZMA2 stream has been terminated by an
// end-of-stream chunk.
func (r *Reader) EOS() bool {
	return r.cstate == stop
}

// uncompressedReader is used to read uncompressed chunks.
type uncompressedReader struct {
	lr   io.LimitedReader
	Dict *lzma.DecoderDict
	eof  bool
	err  error
}

// newUncompressedReader initializes a new uncompressedReader.
func newUncompressedReader(r io.Reader, dict *lzma.DecoderDict, size int64) *uncompressedReader {
	ur := &uncompressedReader{
		lr:   io.LimitedReader{R: r, N: size},
		Dict: dict,
	}
	return ur
}

// Reopen reinitializes an uncompressed reader.
func (ur *uncompressedReader) Reopen(r io.Reader, size int64) {
	ur.eof = false
	ur.lr = io.LimitedReader{R: r, N: size}
}

// fill reads uncompressed data into the dictionary.
func (ur *uncompressedReader) fill() error {
	if !ur.eof {
		n, err := io.CopyN(ur.Dict, &ur.lr, int64(ur.Dict.Available()))
		if err != io.EOF {
			return err
		}
		ur.eof = true
		if n > 0 {
			return nil
		}
	}
	if ur.lr.N != 0 {
		return io.ErrUnexpectedEOF
	}
	return io.EOF
}

// Read reads uncompressed data from the limited reader.
func (ur *uncompressedReader) Read(p []byte) (n int, err error) {
	if ur.err != nil {
		return 0, ur.err
	}
	for {
		var k int
		k, err = ur.Dict.Read(p[n:])
		n += k
		if n >= len(p) {
			return n, nil
		}
		if err != nil {
			break
		}
		err = ur.fill()
		if err != nil {
			break
		}
	}
	ur.err = err
	return n, err
}
<commit_msg>lzma2: fixed a bug in the uncompressed reader<commit_after>// Package lzma2 provides a reader and a writer for the LZMA2 encoding.
// LZMA2 is a framing format for LZMA raw streams to support flushing,
// parallel compression and uncompressed segments.
//
// The Reader and Writer allows the reading and writing of LZMA2 chunk
// sequences. They can be used to parallel compress or decompress LZMA2
// streams.
package lzma2

import (
	"errors"
	"io"

	"github.com/ulikunitz/xz/lzma"
)

// ReaderParams defines the LZMA2 reader parameters.
type ReaderParams struct {
	DictCap int
}

// ReaderDefaults define the defaults for the reader parameters.
var ReaderDefaults = ReaderParams{
	DictCap: 8 * 1024 * 1024,
}

// Verify verifies the LZMA2 reader parameters for correctness.
func (p *ReaderParams) Verify() error {
	return verifyDictCap(p.DictCap)
}

// Reader supports the reading of LZMA2 chunk sequences. Note that the
// first chunk should have a dictionary reset and the first compressed
// chunk a properties reset. The chunk sequence may not be terminated by
// an end-of-stream chunk.
type Reader struct {
	r   io.Reader
	err error

	dict        *lzma.DecoderDict
	ur          *uncompressedReader
	decoder     *lzma.Decoder
	chunkReader io.Reader

	cstate chunkState
	ctype  chunkType
}

// NewReader creates a reader for an LZMA2 chunk sequence with the given
// dictionary capacity.
func NewReader(lzma2 io.Reader, dictCap int) (r *Reader, err error) {
	params := ReaderDefaults
	params.DictCap = dictCap
	return NewReaderParams(lzma2, &params)
}

// NewReaderParams creates a new LZMA2 reader using the given
// parameters.
func NewReaderParams(lzma2 io.Reader, params *ReaderParams) (r *Reader, err error) {
	if err = params.Verify(); err != nil {
		return nil, err
	}
	r = &Reader{
		r:      lzma2,
		cstate: start,
	}
	r.dict, err = lzma.NewDecoderDict(params.DictCap)
	if err != nil {
		return nil, err
	}
	if err = r.startChunk(); err != nil {
		return nil, err
	}
	return r, nil
}

// uncompressed tests whether the chunk type specifies an uncompressed
// chunk.
func uncompressed(ctype chunkType) bool {
	return ctype == cU || ctype == cUD
}

// startChunk parses a new chunk.
func (r *Reader) startChunk() error {
	r.chunkReader = nil
	header, err := readChunkHeader(r.r)
	if err != nil {
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
		}
		return err
	}
	if err = r.cstate.next(header.ctype); err != nil {
		return err
	}
	if r.cstate == stop {
		return io.EOF
	}
	if header.ctype == cUD || header.ctype == cLRND {
		r.dict.Reset()
	}
	size := int64(header.uncompressed) + 1
	if uncompressed(header.ctype) {
		if r.ur != nil {
			r.ur.Reopen(r.r, size)
		} else {
			r.ur = newUncompressedReader(r.r, r.dict, size)
		}
		r.chunkReader = r.ur
		return nil
	}
	br := lzma.ByteReader(io.LimitReader(r.r, int64(header.compressed)+1))
	if r.decoder == nil {
		state := lzma.NewState(header.props)
		r.decoder, err = lzma.NewDecoder(br, state, r.dict, size)
		if err != nil {
			return err
		}
		r.chunkReader = r.decoder
		return nil
	}
	switch header.ctype {
	case cLR:
		r.decoder.State.Reset()
	case cLRN, cLRND:
		r.decoder.State = lzma.NewState(header.props)
	}
	err = r.decoder.Reopen(br, size)
	if err != nil {
		return err
	}
	r.chunkReader = r.decoder
	return nil
}

// Read reads data from the LZMA2 chunk sequence.
func (r *Reader) Read(p []byte) (n int, err error) {
	if r.err != nil {
		return 0, r.err
	}
	for n < len(p) {
		var k int
		k, err = r.chunkReader.Read(p[n:])
		n += k
		if err == io.EOF {
			err = r.startChunk()
		}
		if err != nil {
			r.err = err
			return n, err
		}
		if k == 0 {
			r.err = errors.New("lzma2: Reader doesn't get data")
			return n, r.err
		}
	}
	return n, nil
}

// EOS returns whether the LZMA2 stream has been terminated by an
// end-of-stream chunk.
func (r *Reader) EOS() bool {
	return r.cstate == stop
}

// uncompressedReader is used to read uncompressed chunks.
type uncompressedReader struct {
	lr   io.LimitedReader
	Dict *lzma.DecoderDict
	eof  bool
	err  error
}

// newUncompressedReader initializes a new uncompressedReader.
func newUncompressedReader(r io.Reader, dict *lzma.DecoderDict, size int64) *uncompressedReader {
	ur := &uncompressedReader{
		lr:   io.LimitedReader{R: r, N: size},
		Dict: dict,
	}
	return ur
}

// Reopen reinitializes an uncompressed reader.
func (ur *uncompressedReader) Reopen(r io.Reader, size int64) {
	ur.err = nil
	ur.eof = false
	ur.lr = io.LimitedReader{R: r, N: size}
}

// fill reads uncompressed data into the dictionary.
func (ur *uncompressedReader) fill() error {
	if !ur.eof {
		n, err := io.CopyN(ur.Dict, &ur.lr, int64(ur.Dict.Available()))
		if err != io.EOF {
			return err
		}
		ur.eof = true
		if n > 0 {
			return nil
		}
	}
	if ur.lr.N != 0 {
		return io.ErrUnexpectedEOF
	}
	return io.EOF
}

// Read reads uncompressed data from the limited reader.
func (ur *uncompressedReader) Read(p []byte) (n int, err error) {
	if ur.err != nil {
		return 0, ur.err
	}
	for {
		var k int
		k, err = ur.Dict.Read(p[n:])
		n += k
		if n >= len(p) {
			return n, nil
		}
		if err != nil {
			break
		}
		err = ur.fill()
		if err != nil {
			break
		}
	}
	ur.err = err
	return n, err
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"context"
	"encoding/json"
	"fmt"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apiserver/pkg/authentication/serviceaccount"
	"k8s.io/kubernetes/pkg/api/v1"
	rbacv1beta1 "k8s.io/kubernetes/pkg/apis/rbac/v1beta1"
	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
)

// partially cloned from webserver.go
type State struct {
	Received map[string]int
}

func testPreStop(c clientset.Interface, ns string) {
	// This is the server that will receive the preStop notification
	podDescr := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: "server",
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:  "server",
					Image: "gcr.io/google_containers/nettest:1.7",
					Ports: []v1.ContainerPort{{ContainerPort: 8080}},
				},
			},
		},
	}
	By(fmt.Sprintf("Creating server pod %s in namespace %s", podDescr.Name, ns))
	podDescr, err := c.Core().Pods(ns).Create(podDescr)
	framework.ExpectNoError(err, fmt.Sprintf("creating pod %s", podDescr.Name))

	// At the end of the test, clean up by removing the pod.
	defer func() {
		By("Deleting the server pod")
		c.Core().Pods(ns).Delete(podDescr.Name, nil)
	}()

	By("Waiting for pods to come up.")
	err = framework.WaitForPodRunningInNamespace(c, podDescr)
	framework.ExpectNoError(err, "waiting for server pod to start")

	val := "{\"Source\": \"prestop\"}"

	podOut, err := c.Core().Pods(ns).Get(podDescr.Name, metav1.GetOptions{})
	framework.ExpectNoError(err, "getting pod info")

	preStopDescr := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: "tester",
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:    "tester",
					Image:   "gcr.io/google_containers/busybox:1.24",
					Command: []string{"sleep", "600"},
					Lifecycle: &v1.Lifecycle{
						PreStop: &v1.Handler{
							Exec: &v1.ExecAction{
								Command: []string{
									"wget", "-O-", "--post-data=" + val, fmt.Sprintf("http://%s:8080/write", podOut.Status.PodIP),
								},
							},
						},
					},
				},
			},
		},
	}

	By(fmt.Sprintf("Creating tester pod %s in namespace %s", preStopDescr.Name, ns))
	preStopDescr, err = c.Core().Pods(ns).Create(preStopDescr)
	framework.ExpectNoError(err, fmt.Sprintf("creating pod %s", preStopDescr.Name))
	deletePreStop := true

	// At the end of the test, clean up by removing the pod.
	defer func() {
		if deletePreStop {
			By("Deleting the tester pod")
			c.Core().Pods(ns).Delete(preStopDescr.Name, nil)
		}
	}()

	err = framework.WaitForPodRunningInNamespace(c, preStopDescr)
	framework.ExpectNoError(err, "waiting for tester pod to start")

	// Delete the pod with the preStop handler.
	By("Deleting pre-stop pod")
	if err := c.Core().Pods(ns).Delete(preStopDescr.Name, nil); err == nil {
		deletePreStop = false
	}
	framework.ExpectNoError(err, fmt.Sprintf("deleting pod: %s", preStopDescr.Name))

	// Validate that the server received the web poke.
	err = wait.Poll(time.Second*5, time.Second*60, func() (bool, error) {
		subResourceProxyAvailable, err := framework.ServerVersionGTE(framework.SubResourcePodProxyVersion, c.Discovery())
		if err != nil {
			return false, err
		}

		ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout)
		defer cancel()

		var body []byte
		if subResourceProxyAvailable {
			body, err = c.Core().RESTClient().Get().
				Context(ctx).
				Namespace(ns).
				Resource("pods").
				SubResource("proxy").
				Name(podDescr.Name).
				Suffix("read").
				DoRaw()
		} else {
			body, err = c.Core().RESTClient().Get().
				Context(ctx).
				Prefix("proxy").
				Namespace(ns).
				Resource("pods").
				Name(podDescr.Name).
				Suffix("read").
				DoRaw()
		}
		if err != nil {
			if ctx.Err() != nil {
				framework.Failf("Error validating prestop: %v", err)
				return true, err
			}
			By(fmt.Sprintf("Error validating prestop: %v", err))
		} else {
			framework.Logf("Saw: %s", string(body))
			state := State{}
			err := json.Unmarshal(body, &state)
			if err != nil {
				framework.Logf("Error parsing: %v", err)
				return false, nil
			}
			if state.Received["prestop"] != 0 {
				return true, nil
			}
		}
		return false, nil
	})
	framework.ExpectNoError(err, "validating pre-stop.")
}

var _ = framework.KubeDescribe("PreStop", func() {
	f := framework.NewDefaultFramework("prestop")

	BeforeEach(func() {
		// this test wants extra permissions.  Since the namespace names are unique, we can leave this
		// lying around so we don't have to race any caches
		framework.BindClusterRole(f.ClientSet.Rbac(), "cluster-admin", f.Namespace.Name,
			rbacv1beta1.Subject{Kind: rbacv1beta1.ServiceAccountKind, Namespace: f.Namespace.Name, Name: "default"})

		err := framework.WaitForAuthorizationUpdate(f.ClientSet.AuthorizationV1beta1(),
			serviceaccount.MakeUsername(f.Namespace.Name, "default"),
			"", "create", schema.GroupResource{Resource: "pods"}, true)
		framework.ExpectNoError(err)
	})

	It("should call prestop when killing a pod [Conformance]", func() {
		testPreStop(f.ClientSet, f.Namespace.Name)
	})
})
<commit_msg>e2e: Remove special serviceaccount permission requirement<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"context"
	"encoding/json"
	"fmt"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
)

// partially cloned from webserver.go
type State struct {
	Received map[string]int
}

func testPreStop(c clientset.Interface, ns string) {
	// This is the server that will receive the preStop notification
	podDescr := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: "server",
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:  "server",
					Image: "gcr.io/google_containers/nettest:1.7",
					Ports: []v1.ContainerPort{{ContainerPort: 8080}},
				},
			},
		},
	}
	By(fmt.Sprintf("Creating server pod %s in namespace %s", podDescr.Name, ns))
	podDescr, err := c.Core().Pods(ns).Create(podDescr)
	framework.ExpectNoError(err, fmt.Sprintf("creating pod %s", podDescr.Name))

	// At the end of the test, clean up by removing the pod.
	defer func() {
		By("Deleting the server pod")
		c.Core().Pods(ns).Delete(podDescr.Name, nil)
	}()

	By("Waiting for pods to come up.")
	err = framework.WaitForPodRunningInNamespace(c, podDescr)
	framework.ExpectNoError(err, "waiting for server pod to start")

	val := "{\"Source\": \"prestop\"}"

	podOut, err := c.Core().Pods(ns).Get(podDescr.Name, metav1.GetOptions{})
	framework.ExpectNoError(err, "getting pod info")

	preStopDescr := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: "tester",
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:    "tester",
					Image:   "gcr.io/google_containers/busybox:1.24",
					Command: []string{"sleep", "600"},
					Lifecycle: &v1.Lifecycle{
						PreStop: &v1.Handler{
							Exec: &v1.ExecAction{
								Command: []string{
									"wget", "-O-", "--post-data=" + val, fmt.Sprintf("http://%s:8080/write", podOut.Status.PodIP),
								},
							},
						},
					},
				},
			},
		},
	}

	By(fmt.Sprintf("Creating tester pod %s in namespace %s", preStopDescr.Name, ns))
	preStopDescr, err = c.Core().Pods(ns).Create(preStopDescr)
	framework.ExpectNoError(err, fmt.Sprintf("creating pod %s", preStopDescr.Name))
	deletePreStop := true

	// At the end of the test, clean up by removing the pod.
	defer func() {
		if deletePreStop {
			By("Deleting the tester pod")
			c.Core().Pods(ns).Delete(preStopDescr.Name, nil)
		}
	}()

	err = framework.WaitForPodRunningInNamespace(c, preStopDescr)
	framework.ExpectNoError(err, "waiting for tester pod to start")

	// Delete the pod with the preStop handler.
	By("Deleting pre-stop pod")
	if err := c.Core().Pods(ns).Delete(preStopDescr.Name, nil); err == nil {
		deletePreStop = false
	}
	framework.ExpectNoError(err, fmt.Sprintf("deleting pod: %s", preStopDescr.Name))

	// Validate that the server received the web poke.
	err = wait.Poll(time.Second*5, time.Second*60, func() (bool, error) {
		subResourceProxyAvailable, err := framework.ServerVersionGTE(framework.SubResourcePodProxyVersion, c.Discovery())
		if err != nil {
			return false, err
		}

		ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout)
		defer cancel()

		var body []byte
		if subResourceProxyAvailable {
			body, err = c.Core().RESTClient().Get().
				Context(ctx).
				Namespace(ns).
				Resource("pods").
				SubResource("proxy").
				Name(podDescr.Name).
				Suffix("read").
				DoRaw()
		} else {
			body, err = c.Core().RESTClient().Get().
				Context(ctx).
				Prefix("proxy").
				Namespace(ns).
				Resource("pods").
				Name(podDescr.Name).
				Suffix("read").
				DoRaw()
		}
		if err != nil {
			if ctx.Err() != nil {
				framework.Failf("Error validating prestop: %v", err)
				return true, err
			}
			By(fmt.Sprintf("Error validating prestop: %v", err))
		} else {
			framework.Logf("Saw: %s", string(body))
			state := State{}
			err := json.Unmarshal(body, &state)
			if err != nil {
				framework.Logf("Error parsing: %v", err)
				return false, nil
			}
			if state.Received["prestop"] != 0 {
				return true, nil
			}
		}
		return false, nil
	})
	framework.ExpectNoError(err, "validating pre-stop.")
}

var _ = framework.KubeDescribe("PreStop", func() {
	f := framework.NewDefaultFramework("prestop")

	It("should call prestop when killing a pod [Conformance]", func() {
		testPreStop(f.ClientSet, f.Namespace.Name)
	})
})
<|endoftext|>


<commit_before>// Copyright 2016 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"fmt"
	"os"
	"syscall"

	dockerclient "github.com/control-center/serviced/commons/docker"
	"github.com/control-center/serviced/domain/service"
	"github.com/control-center/serviced/utils"
)

// TODO: what to do about logging?

// GetServiceInstances returns all instances running on a service
func (a *api) GetServiceInstances(serviceID string) ([]service.Instance, error) {
	client, err := a.connectMaster()
	if err != nil {
		return nil, err
	}
	return client.GetServiceInstances(serviceID)
}

// StopServiceInstance stops a running instance of a service.
func (a *api) StopServiceInstance(serviceID string, instanceID int) error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}
	return client.StopServiceInstance(serviceID, instanceID)
}

// AttachServiceInstance locates and attaches to a running instance of a service
func (a *api) AttachServiceInstance(serviceID string, instanceID int, command string, args []string) error {
	var (
		targetHost      string
		targetIP        string
		targetContainer string
	)

	hostID, err := utils.HostID()
	if err != nil {
		return err
	}

	// Check to see if serviceID is actually a dockerID
	_, err = dockerclient.FindContainer(serviceID)
	if err == nil {
		targetHost = hostID
		targetContainer = serviceID
	} else {
		client, err := a.connectMaster()
		if err != nil {
			return err
		}

		// get the location of the running instance
		location, err := client.LocateServiceInstance(serviceID, instanceID)
		if err != nil {
			return err
		}

		targetHost = location.HostID
		targetIP = location.HostIP
		targetContainer = location.ContainerID
	}

	// attach to the container
	cmd := []string{}
	if targetHost != hostID {
		cmd := []string{
			"/usr/bin/ssh",
			"-t", targetIP, "--",
			"serviced", "--endpoint", GetOptionsRPCEndpoint(),
			"service", "attach", fmt.Sprintf("%s/%d", serviceID, instanceID),
		}
		cmd = append(cmd, command)
		cmd = append(cmd, args...)
		return syscall.Exec(cmd[0], cmd[0:], os.Environ())
	} else {
		if command == "" {
			cmd = append(cmd, "/bin/bash")
		} else {
			cmd = append(cmd, command)
			cmd = append(cmd, args...)
		}
		return utils.AttachAndExec(targetContainer, cmd)
	}
}

// LogsForServiceInstance returns the logs for the service instance
func (a *api) LogsForServiceInstance(serviceID string, instanceID int, command string, args []string) error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}

	// get the location of the running instance
	location, err := client.LocateServiceInstance(serviceID, instanceID)
	if err != nil {
		return err
	}

	// check to see if it is running on this host
	hostID, err := utils.HostID()
	if err != nil {
		return err
	}

	// report container logs
	cmd := []string{}
	if location.HostID != hostID {
		cmd := []string{
			"/usr/bin/ssh",
			"-t", location.HostIP, "--",
			"serviced", "--endpoint", GetOptionsRPCEndpoint(),
			"service", "logs", fmt.Sprintf("%s/%d", serviceID, instanceID),
		}
		if command != "" {
			cmd = append(cmd, command)
			cmd = append(cmd, args...)
		}
		return syscall.Exec(cmd[0], cmd[0:], os.Environ())
	} else {
		if command != "" {
			cmd = append(cmd, command)
			cmd = append(cmd, args...)
		}
		return dockerclient.Logs(location.ContainerID, cmd)
	}
}

// SendDockerAction submits an action to a running service instance
func (a *api) SendDockerAction(serviceID string, instanceID int, action string, args []string) error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}

	return client.SendDockerAction(serviceID, instanceID, action, args)
}
<commit_msg>Pass docker id to other host on attach<commit_after>// Copyright 2016 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"fmt"
	"os"
	"syscall"

	dockerclient "github.com/control-center/serviced/commons/docker"
	"github.com/control-center/serviced/domain/service"
	"github.com/control-center/serviced/utils"
)

// TODO: what to do about logging?

// GetServiceInstances returns all instances running on a service
func (a *api) GetServiceInstances(serviceID string) ([]service.Instance, error) {
	client, err := a.connectMaster()
	if err != nil {
		return nil, err
	}
	return client.GetServiceInstances(serviceID)
}

// StopServiceInstance stops a running instance of a service.
func (a *api) StopServiceInstance(serviceID string, instanceID int) error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}
	return client.StopServiceInstance(serviceID, instanceID)
}

// AttachServiceInstance locates and attaches to a running instance of a service
func (a *api) AttachServiceInstance(serviceID string, instanceID int, command string, args []string) error {
	var (
		targetHost      string
		targetIP        string
		targetContainer string
	)

	hostID, err := utils.HostID()
	if err != nil {
		return err
	}

	// Check to see if serviceID is actually a dockerID
	_, err = dockerclient.FindContainer(serviceID)
	if err == nil {
		targetHost = hostID
		targetContainer = serviceID
	} else {
		client, err := a.connectMaster()
		if err != nil {
			return err
		}

		// get the location of the running instance
		location, err := client.LocateServiceInstance(serviceID, instanceID)
		if err != nil {
			return err
		}

		targetHost = location.HostID
		targetIP = location.HostIP
		targetContainer = location.ContainerID
	}

	// attach to the container
	cmd := []string{}
	if targetHost != hostID {
		cmd := []string{
			"/usr/bin/ssh",
			"-t", targetIP, "--",
			"serviced", "--endpoint", GetOptionsRPCEndpoint(),
			"service", "attach", fmt.Sprintf("%s", targetContainer),
		}
		cmd = append(cmd, command)
		cmd = append(cmd, args...)
		return syscall.Exec(cmd[0], cmd[0:], os.Environ())
	} else {
		if command == "" {
			cmd = append(cmd, "/bin/bash")
		} else {
			cmd = append(cmd, command)
			cmd = append(cmd, args...)
		}
		return utils.AttachAndExec(targetContainer, cmd)
	}
}

// LogsForServiceInstance returns the logs for the service instance
func (a *api) LogsForServiceInstance(serviceID string, instanceID int, command string, args []string) error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}

	// get the location of the running instance
	location, err := client.LocateServiceInstance(serviceID, instanceID)
	if err != nil {
		return err
	}

	// check to see if it is running on this host
	hostID, err := utils.HostID()
	if err != nil {
		return err
	}

	// report container logs
	cmd := []string{}
	if location.HostID != hostID {
		cmd := []string{
			"/usr/bin/ssh",
			"-t", location.HostIP, "--",
			"serviced", "--endpoint", GetOptionsRPCEndpoint(),
			"service", "logs", fmt.Sprintf("%s/%d", serviceID, instanceID),
		}
		if command != "" {
			cmd = append(cmd, command)
			cmd = append(cmd, args...)
		}
		return syscall.Exec(cmd[0], cmd[0:], os.Environ())
	} else {
		if command != "" {
			cmd = append(cmd, command)
			cmd = append(cmd, args...)
		}
		return dockerclient.Logs(location.ContainerID, cmd)
	}
}

// SendDockerAction submits an action to a running service instance
func (a *api) SendDockerAction(serviceID string, instanceID int, action string, args []string) error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}

	return client.SendDockerAction(serviceID, instanceID, action, args)
}
<|endoftext|>


<commit_before>// Package client is a CT log client implementation and contains types and code
// for interacting with RFC6962-compliant CT Log instances.
// See http://tools.ietf.org/html/rfc6962 for details
package client

import (
	"crypto/sha256"
	"encoding/base64"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"

	ct "github.com/google/certificate-transparency/go"
	"github.com/google/certificate-transparency/go/jsonclient"
	"github.com/google/certificate-transparency/go/tls"
	"golang.org/x/net/context"
)

// LogClient represents a client for a given CT Log instance
type LogClient struct {
	jsonclient.JSONClient
}

// New constructs a new LogClient instance.
// |uri| is the base URI of the CT log instance to interact with, e.g.
// http://ct.googleapis.com/pilot
// |hc| is the underlying client to be used for HTTP requests to the CT log.
// |opts| can be used to provide a customer logger interface and a public key
// for signature verification.
func New(uri string, hc *http.Client, opts jsonclient.Options) (*LogClient, error) {
	logClient, err := jsonclient.New(uri, hc, opts)
	if err != nil {
		return nil, err
	}
	return &LogClient{*logClient}, err
}

// Attempts to add |chain| to the log, using the api end-point specified by
// |path|. If provided context expires before submission is complete an
// error will be returned.
func (c *LogClient) addChainWithRetry(ctx context.Context, ctype ct.LogEntryType, path string, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	var resp ct.AddChainResponse
	var req ct.AddChainRequest
	for _, link := range chain {
		req.Chain = append(req.Chain, link.Data)
	}

	_, err := c.PostAndParseWithRetry(ctx, path, &req, &resp)
	if err != nil {
		return nil, err
	}

	var ds ct.DigitallySigned
	if rest, err := tls.Unmarshal(resp.Signature, &ds); err != nil {
		return nil, err
	} else if len(rest) > 0 {
		return nil, fmt.Errorf("trailing data (%d bytes) after DigitallySigned", len(rest))
	}

	var logID ct.LogID
	copy(logID.KeyID[:], resp.ID)
	sct := &ct.SignedCertificateTimestamp{
		SCTVersion: resp.SCTVersion,
		LogID:      logID,
		Timestamp:  resp.Timestamp,
		Extensions: ct.CTExtensions(resp.Extensions),
		Signature:  ds,
	}
	err = c.VerifySCTSignature(*sct, ctype, chain)
	if err != nil {
		return nil, err
	}
	return sct, nil
}

// AddChain adds the (DER represented) X509 |chain| to the log.
func (c *LogClient) AddChain(ctx context.Context, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	return c.addChainWithRetry(ctx, ct.X509LogEntryType, ct.AddChainPath, chain)
}

// AddPreChain adds the (DER represented) Precertificate |chain| to the log.
func (c *LogClient) AddPreChain(ctx context.Context, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	return c.addChainWithRetry(ctx, ct.PrecertLogEntryType, ct.AddPreChainPath, chain)
}

// AddJSON submits arbitrary data to to XJSON server.
func (c *LogClient) AddJSON(ctx context.Context, data interface{}) (*ct.SignedCertificateTimestamp, error) {
	req := ct.AddJSONRequest{Data: data}
	var resp ct.AddChainResponse
	_, err := c.PostAndParse(ctx, ct.AddJSONPath, &req, &resp)
	if err != nil {
		return nil, err
	}
	var ds ct.DigitallySigned
	if rest, err := tls.Unmarshal(resp.Signature, &ds); err != nil {
		return nil, err
	} else if len(rest) > 0 {
		return nil, fmt.Errorf("trailing data (%d bytes) after DigitallySigned", len(rest))
	}
	var logID ct.LogID
	copy(logID.KeyID[:], resp.ID)
	return &ct.SignedCertificateTimestamp{
		SCTVersion: resp.SCTVersion,
		LogID:      logID,
		Timestamp:  resp.Timestamp,
		Extensions: ct.CTExtensions(resp.Extensions),
		Signature:  ds,
	}, nil
}

// GetSTH retrieves the current STH from the log.
// Returns a populated SignedTreeHead, or a non-nil error.
func (c *LogClient) GetSTH(ctx context.Context) (sth *ct.SignedTreeHead, err error) {
	var resp ct.GetSTHResponse
	_, err = c.GetAndParse(ctx, ct.GetSTHPath, nil, &resp)
	if err != nil {
		return
	}
	sth = &ct.SignedTreeHead{
		TreeSize:  resp.TreeSize,
		Timestamp: resp.Timestamp,
	}

	if len(resp.SHA256RootHash) != sha256.Size {
		return nil, fmt.Errorf("sha256_root_hash is invalid length, expected %d got %d", sha256.Size, len(resp.SHA256RootHash))
	}
	copy(sth.SHA256RootHash[:], resp.SHA256RootHash)

	var ds ct.DigitallySigned
	if rest, err := tls.Unmarshal(resp.TreeHeadSignature, &ds); err != nil {
		return nil, err
	} else if len(rest) > 0 {
		return nil, fmt.Errorf("trailing data (%d bytes) after DigitallySigned", len(rest))
	}
	sth.TreeHeadSignature = ds
	err = c.VerifySTHSignature(*sth)
	if err != nil {
		return nil, err
	}
	return
}

// VerifySTHSignature checks the signature in sth, returning any error encountered or nil if verification is
// successful.
func (c *LogClient) VerifySTHSignature(sth ct.SignedTreeHead) error {
	if c.Verifier == nil {
		// Can't verify signatures without a verifier
		return nil
	}
	return c.Verifier.VerifySTHSignature(sth)
}

// VerifySCTSignature checks the signature in sct for the given LogEntryType, with associated certificate chain.
func (c *LogClient) VerifySCTSignature(sct ct.SignedCertificateTimestamp, ctype ct.LogEntryType, certData []ct.ASN1Cert) error {
	if c.Verifier == nil {
		// Can't verify signatures without a verifier
		return nil
	}

	if ctype == ct.PrecertLogEntryType {
		// TODO(drysdale): cope with pre-certs, which need to have the
		// following fields set:
		//    leaf.PrecertEntry.TBSCertificate
		//    leaf.PrecertEntry.IssuerKeyHash  (SHA-256 of issuer's public key)
		return errors.New("SCT verification for pre-certificates unimplemented")
	}
	// Build enough of a Merkle tree leaf for the verifier to work on.
	leaf := ct.MerkleTreeLeaf{
		Version:  sct.SCTVersion,
		LeafType: ct.TimestampedEntryLeafType,
		TimestampedEntry: &ct.TimestampedEntry{
			Timestamp:  sct.Timestamp,
			EntryType:  ctype,
			X509Entry:  &certData[0],
			Extensions: sct.Extensions,
		},
	}
	entry := ct.LogEntry{Leaf: leaf}
	return c.Verifier.VerifySCTSignature(sct, entry)
}

// GetSTHConsistency retrieves the consistency proof between two snapshots.
func (c *LogClient) GetSTHConsistency(ctx context.Context, first, second uint64) ([][]byte, error) {
	base10 := 10
	params := map[string]string{
		"first":  strconv.FormatUint(first, base10),
		"second": strconv.FormatUint(second, base10),
	}
	var resp ct.GetSTHConsistencyResponse
	if _, err := c.GetAndParse(ctx, ct.GetSTHConsistencyPath, params, &resp); err != nil {
		return nil, err
	}
	return resp.Consistency, nil
}

// GetProofByHash returns an audit path for the hash of an SCT.
func (c *LogClient) GetProofByHash(ctx context.Context, hash []byte, treeSize uint64) (*ct.GetProofByHashResponse, error) {
	b64Hash := url.QueryEscape(base64.StdEncoding.EncodeToString(hash))
	base10 := 10
	params := map[string]string{
		"tree_size": strconv.FormatUint(treeSize, base10),
		"hash":      b64Hash,
	}
	var resp ct.GetProofByHashResponse
	if _, err := c.GetAndParse(ctx, ct.GetProofByHashPath, params, &resp); err != nil {
		return nil, err
	}
	return &resp, nil
}

// GetAcceptedRoots retrieves the set of acceptable root certificates for a log.
func (c *LogClient) GetAcceptedRoots(ctx context.Context) ([]ct.ASN1Cert, error) {
	var resp ct.GetRootsResponse
	if _, err := c.GetAndParse(ctx, ct.GetRootsPath, nil, &resp); err != nil {
		return nil, err
	}
	var roots []ct.ASN1Cert
	for _, cert64 := range resp.Certificates {
		cert, err := base64.StdEncoding.DecodeString(cert64)
		if err != nil {
			return nil, err
		}
		roots = append(roots, ct.ASN1Cert{Data: cert})
	}
	return roots, nil
}
<commit_msg>go/client: validate add-pre-chain SCTs<commit_after>// Package client is a CT log client implementation and contains types and code
// for interacting with RFC6962-compliant CT Log instances.
// See http://tools.ietf.org/html/rfc6962 for details
package client

import (
	"crypto/sha256"
	"encoding/base64"
	"fmt"
	"net/http"
	"net/url"
	"strconv"

	ct "github.com/google/certificate-transparency/go"
	"github.com/google/certificate-transparency/go/jsonclient"
	"github.com/google/certificate-transparency/go/tls"
	"github.com/google/certificate-transparency/go/x509"
	"golang.org/x/net/context"
)

// LogClient represents a client for a given CT Log instance
type LogClient struct {
	jsonclient.JSONClient
}

// New constructs a new LogClient instance.
// |uri| is the base URI of the CT log instance to interact with, e.g.
// http://ct.googleapis.com/pilot
// |hc| is the underlying client to be used for HTTP requests to the CT log.
// |opts| can be used to provide a customer logger interface and a public key
// for signature verification.
func New(uri string, hc *http.Client, opts jsonclient.Options) (*LogClient, error) {
	logClient, err := jsonclient.New(uri, hc, opts)
	if err != nil {
		return nil, err
	}
	return &LogClient{*logClient}, err
}

// Attempts to add |chain| to the log, using the api end-point specified by
// |path|. If provided context expires before submission is complete an
// error will be returned.
func (c *LogClient) addChainWithRetry(ctx context.Context, ctype ct.LogEntryType, path string, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	var resp ct.AddChainResponse
	var req ct.AddChainRequest
	for _, link := range chain {
		req.Chain = append(req.Chain, link.Data)
	}

	_, err := c.PostAndParseWithRetry(ctx, path, &req, &resp)
	if err != nil {
		return nil, err
	}

	var ds ct.DigitallySigned
	if rest, err := tls.Unmarshal(resp.Signature, &ds); err != nil {
		return nil, err
	} else if len(rest) > 0 {
		return nil, fmt.Errorf("trailing data (%d bytes) after DigitallySigned", len(rest))
	}

	var logID ct.LogID
	copy(logID.KeyID[:], resp.ID)
	sct := &ct.SignedCertificateTimestamp{
		SCTVersion: resp.SCTVersion,
		LogID:      logID,
		Timestamp:  resp.Timestamp,
		Extensions: ct.CTExtensions(resp.Extensions),
		Signature:  ds,
	}
	err = c.VerifySCTSignature(*sct, ctype, chain)
	if err != nil {
		return nil, err
	}
	return sct, nil
}

// AddChain adds the (DER represented) X509 |chain| to the log.
func (c *LogClient) AddChain(ctx context.Context, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	return c.addChainWithRetry(ctx, ct.X509LogEntryType, ct.AddChainPath, chain)
}

// AddPreChain adds the (DER represented) Precertificate |chain| to the log.
func (c *LogClient) AddPreChain(ctx context.Context, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	return c.addChainWithRetry(ctx, ct.PrecertLogEntryType, ct.AddPreChainPath, chain)
}

// AddJSON submits arbitrary data to to XJSON server.
func (c *LogClient) AddJSON(ctx context.Context, data interface{}) (*ct.SignedCertificateTimestamp, error) {
	req := ct.AddJSONRequest{Data: data}
	var resp ct.AddChainResponse
	_, err := c.PostAndParse(ctx, ct.AddJSONPath, &req, &resp)
	if err != nil {
		return nil, err
	}
	var ds ct.DigitallySigned
	if rest, err := tls.Unmarshal(resp.Signature, &ds); err != nil {
		return nil, err
	} else if len(rest) > 0 {
		return nil, fmt.Errorf("trailing data (%d bytes) after DigitallySigned", len(rest))
	}
	var logID ct.LogID
	copy(logID.KeyID[:], resp.ID)
	return &ct.SignedCertificateTimestamp{
		SCTVersion: resp.SCTVersion,
		LogID:      logID,
		Timestamp:  resp.Timestamp,
		Extensions: ct.CTExtensions(resp.Extensions),
		Signature:  ds,
	}, nil
}

// GetSTH retrieves the current STH from the log.
// Returns a populated SignedTreeHead, or a non-nil error.
func (c *LogClient) GetSTH(ctx context.Context) (sth *ct.SignedTreeHead, err error) {
	var resp ct.GetSTHResponse
	_, err = c.GetAndParse(ctx, ct.GetSTHPath, nil, &resp)
	if err != nil {
		return
	}
	sth = &ct.SignedTreeHead{
		TreeSize:  resp.TreeSize,
		Timestamp: resp.Timestamp,
	}

	if len(resp.SHA256RootHash) != sha256.Size {
		return nil, fmt.Errorf("sha256_root_hash is invalid length, expected %d got %d", sha256.Size, len(resp.SHA256RootHash))
	}
	copy(sth.SHA256RootHash[:], resp.SHA256RootHash)

	var ds ct.DigitallySigned
	if rest, err := tls.Unmarshal(resp.TreeHeadSignature, &ds); err != nil {
		return nil, err
	} else if len(rest) > 0 {
		return nil, fmt.Errorf("trailing data (%d bytes) after DigitallySigned", len(rest))
	}
	sth.TreeHeadSignature = ds
	err = c.VerifySTHSignature(*sth)
	if err != nil {
		return nil, err
	}
	return
}

// VerifySTHSignature checks the signature in sth, returning any error encountered or nil if verification is
// successful.
func (c *LogClient) VerifySTHSignature(sth ct.SignedTreeHead) error {
	if c.Verifier == nil {
		// Can't verify signatures without a verifier
		return nil
	}
	return c.Verifier.VerifySTHSignature(sth)
}

// VerifySCTSignature checks the signature in sct for the given LogEntryType, with associated certificate chain.
func (c *LogClient) VerifySCTSignature(sct ct.SignedCertificateTimestamp, ctype ct.LogEntryType, certData []ct.ASN1Cert) error {
	if c.Verifier == nil {
		// Can't verify signatures without a verifier
		return nil
	}

	// Build enough of a Merkle tree leaf for the verifier to work on.
	leaf := ct.MerkleTreeLeaf{
		Version:  sct.SCTVersion,
		LeafType: ct.TimestampedEntryLeafType,
		TimestampedEntry: &ct.TimestampedEntry{
			Timestamp:  sct.Timestamp,
			EntryType:  ctype,
			Extensions: sct.Extensions,
		},
	}
	if ctype == ct.X509LogEntryType {
		leaf.TimestampedEntry.X509Entry = &certData[0]
	} else {
		// Pre-certs are more complicated; we need the issuer key hash and the
		// DER-encoded TBSCertificate.  First, parse the issuer to get its
		// public key hash.
		if len(certData) < 2 {
			return fmt.Errorf("no issuer cert available for precert SCT validation")
		}
		issuer, err := x509.ParseCertificate(certData[1].Data)
		if err != nil {
			return fmt.Errorf("failed to parse issuer cert: %v", err)
		}
		issuerKeyHash := sha256.Sum256(issuer.RawSubjectPublicKeyInfo)

		// Second, parse the pre-certificate to extract its DER-encoded
		// TBSCertificate, then post-process this to remove the CT poison
		// extension.
		cert, err := x509.ParseCertificate(certData[0].Data)
		if err != nil {
			return fmt.Errorf("failed to parse leaf pre-cert: %v", err)
		}
		defangedTBS, err := x509.RemoveCTPoison(cert.RawTBSCertificate)
		if err != nil {
			return fmt.Errorf("failed to remove poison extension: %v", err)
		}
		leaf.TimestampedEntry.PrecertEntry = &ct.PreCert{
			IssuerKeyHash:  issuerKeyHash,
			TBSCertificate: defangedTBS,
		}
	}
	entry := ct.LogEntry{Leaf: leaf}
	return c.Verifier.VerifySCTSignature(sct, entry)
}

// GetSTHConsistency retrieves the consistency proof between two snapshots.
func (c *LogClient) GetSTHConsistency(ctx context.Context, first, second uint64) ([][]byte, error) {
	base10 := 10
	params := map[string]string{
		"first":  strconv.FormatUint(first, base10),
		"second": strconv.FormatUint(second, base10),
	}
	var resp ct.GetSTHConsistencyResponse
	if _, err := c.GetAndParse(ctx, ct.GetSTHConsistencyPath, params, &resp); err != nil {
		return nil, err
	}
	return resp.Consistency, nil
}

// GetProofByHash returns an audit path for the hash of an SCT.
func (c *LogClient) GetProofByHash(ctx context.Context, hash []byte, treeSize uint64) (*ct.GetProofByHashResponse, error) {
	b64Hash := url.QueryEscape(base64.StdEncoding.EncodeToString(hash))
	base10 := 10
	params := map[string]string{
		"tree_size": strconv.FormatUint(treeSize, base10),
		"hash":      b64Hash,
	}
	var resp ct.GetProofByHashResponse
	if _, err := c.GetAndParse(ctx, ct.GetProofByHashPath, params, &resp); err != nil {
		return nil, err
	}
	return &resp, nil
}

// GetAcceptedRoots retrieves the set of acceptable root certificates for a log.
func (c *LogClient) GetAcceptedRoots(ctx context.Context) ([]ct.ASN1Cert, error) {
	var resp ct.GetRootsResponse
	if _, err := c.GetAndParse(ctx, ct.GetRootsPath, nil, &resp); err != nil {
		return nil, err
	}
	var roots []ct.ASN1Cert
	for _, cert64 := range resp.Certificates {
		cert, err := base64.StdEncoding.DecodeString(cert64)
		if err != nil {
			return nil, err
		}
		roots = append(roots, ct.ASN1Cert{Data: cert})
	}
	return roots, nil
}
<|endoftext|>


<commit_before>package lazycache

import (
	"fmt"
	kitlog "github.com/go-kit/kit/log"
	"io"
	"net/http"
	"os"
	"path"
)

type LocalImageStore struct {
	LocalRoot string
	UrlRoot   string
	logger    kitlog.Logger

	Stats struct {
		cacheRequests int
		cacheMisses   int
	}
}

func (store LocalImageStore) Has(key string) bool {
	filename := store.LocalRoot + key
	DefaultLogger.Log("level", "debug", "msg", fmt.Sprintf("Checking for \"%s\"", filename))
	_, err := os.Stat(filename)
	return err == nil
}

func (store *LocalImageStore) Url(key string) (string, bool) {

	store.Stats.cacheRequests++

	if store.Has(key) {
		return store.UrlRoot + key, true
	} else {
		store.Stats.cacheMisses++
		return "", false
	}
}

func RecursiveMkdir(dir string) {
	_, err := os.Stat(dir)
	if err != nil {
		RecursiveMkdir(path.Dir(dir))

		os.Mkdir(dir, 0755)
	}
}

func (store LocalImageStore) Store(key string, data io.Reader) {
	filename := store.LocalRoot + key
	RecursiveMkdir(path.Dir(filename))

	f, err := os.Create(filename)
	if err != nil {
		DefaultLogger.Log("msg", err.Error(), "type", "error")
	}

	io.Copy(f, data)
}

func (store LocalImageStore) Retrieve(key string) (io.Reader, error) {

	f, err := os.Open(store.LocalRoot + key)

	return f, err
}

func (store LocalImageStore) Statistics() interface{} {
	return struct {
		Type          string
		CacheRequests int `json: "cache_requests"`
		CacheMisses   int `json: "cache_misses"`
	}{
		Type:          "local_storage",
		CacheRequests: store.Stats.cacheRequests,
		CacheMisses:   store.Stats.cacheMisses,
	}
}

func (store LocalImageStore) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	localPath := path.Join(store.LocalRoot, r.URL.Path)

	if _, err := os.Stat(localPath); err != nil {
		http.Error(w, fmt.Sprintf("Could not find \"%s\"", localPath), 404)
	} else {
		http.ServeFile(w, r, localPath)
	}
}

func CreateLocalStore(localRoot string, host string) *LocalImageStore {

	port := 7080
	addr := fmt.Sprintf("%s:%d", host, port)

	store := &LocalImageStore{
		LocalRoot: localRoot,
		UrlRoot:   "http://" + addr + "/",
		logger:    kitlog.With(DefaultLogger, "module", "LocalImageStore"),
	}

	DefaultLogger.Log("msg",
		fmt.Sprintf("Creating local image store at \"%s\", exposed at \"%s\"\n", store.LocalRoot, store.UrlRoot))

	s := &http.Server{
		Addr:    addr,
		Handler: store,
	}

	go s.ListenAndServe()

	return store
}
<commit_msg>Corrected formatting for local image addresses.<commit_after>package lazycache

import (
	"fmt"
	kitlog "github.com/go-kit/kit/log"
	"io"
	"net/http"
	"os"
	"path"
)

type LocalImageStore struct {
	LocalRoot string
	UrlRoot   string
	logger    kitlog.Logger

	Stats struct {
		cacheRequests int
		cacheMisses   int
	}
}

func (store LocalImageStore) Has(key string) bool {
	filename := store.LocalRoot + key
	DefaultLogger.Log("level", "debug", "msg", fmt.Sprintf("Checking for \"%s\"", filename))
	_, err := os.Stat(filename)
	return err == nil
}

func (store *LocalImageStore) Url(key string) (string, bool) {

	store.Stats.cacheRequests++

	if store.Has(key) {
		return store.UrlRoot + key, true
	} else {
		store.Stats.cacheMisses++
		return "", false
	}
}

func RecursiveMkdir(dir string) {
	_, err := os.Stat(dir)
	if err != nil {
		RecursiveMkdir(path.Dir(dir))

		os.Mkdir(dir, 0755)
	}
}

func (store LocalImageStore) Store(key string, data io.Reader) {
	filename := store.LocalRoot + key
	RecursiveMkdir(path.Dir(filename))

	f, err := os.Create(filename)
	if err != nil {
		DefaultLogger.Log("msg", err.Error(), "type", "error")
	}

	io.Copy(f, data)
}

func (store LocalImageStore) Retrieve(key string) (io.Reader, error) {

	f, err := os.Open(store.LocalRoot + key)

	return f, err
}

func (store LocalImageStore) Statistics() interface{} {
	return struct {
		Type          string
		CacheRequests int `json: "cache_requests"`
		CacheMisses   int `json: "cache_misses"`
	}{
		Type:          "local_storage",
		CacheRequests: store.Stats.cacheRequests,
		CacheMisses:   store.Stats.cacheMisses,
	}
}

func (store LocalImageStore) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	localPath := path.Join(store.LocalRoot, r.URL.Path)

	if _, err := os.Stat(localPath); err != nil {
		http.Error(w, fmt.Sprintf("Could not find \"%s\"", localPath), 404)
	} else {
		http.ServeFile(w, r, localPath)
	}
}

func CreateLocalStore(localRoot string, addr string) *LocalImageStore {

	// port := 7080
	// addr := fmt.Sprintf("%s:%d", host, port)

	store := &LocalImageStore{
		LocalRoot: localRoot,
		UrlRoot:   addr,
		logger:    kitlog.With(DefaultLogger, "module", "LocalImageStore"),
	}

	DefaultLogger.Log("msg",
		fmt.Sprintf("Creating local image store at \"%s\", exposed at \"%s\"\n", store.LocalRoot, store.UrlRoot))

	s := &http.Server{
		Addr:    addr,
		Handler: store,
	}

	go s.ListenAndServe()

	return store
}
<|endoftext|>


<commit_before>//
// @RequestTest
//
// @author darryl.west <darryl.west@raincitysoftware.com>
// @created 2017-03-19 12:09:24
//

package test

import (
	"reflect"
	"testing"

	"spotcache"

	. "github.com/franela/goblin"
)

func TestRequest(t *testing.T) {
	g := Goblin(t)

	g.Describe("Request", func() {
		cfg := spotcache.NewConfigForEnvironment("test")
		spotcache.CreateLogger(cfg)
		var session spotcache.SessionType
		copy(session[:12], []byte(spotcache.CreateSessionId()))

		builder := spotcache.NewRequestBuilder(session)
		key := []byte(spotcache.CreateULID())
		value := CreateRandomData()
		metadata := []byte("expire:40;")

        keysz := uint16(len(key))
        metasz := uint16(len(metadata))
        valsz := uint32(len(value))

        g.It("should create a new instance of Request from NewCommand", func() {
            req := builder.NewRequest(spotcache.NOOP)

			g.Assert(reflect.TypeOf(req).String()).Equal("spotcache.Request")
			g.Assert(len(req.Id)).Equal(26)
			g.Assert(len(req.Session)).Equal(12)
            g.Assert(req.Op).Equal(spotcache.NOOP)
        })

		g.It("should create a put instance of Request", func() {
			req := builder.CreatePutRequest(key, value, metadata)

			g.Assert(reflect.TypeOf(req).String()).Equal("*spotcache.Request")
			g.Assert(req.Op).Equal(spotcache.PUT)
            g.Assert(req.MetaSize).Equal(metasz)
            g.Assert(req.KeySize).Equal(keysz)
            g.Assert(req.DataSize).Equal(valsz)

            g.Assert(req.Metadata).Equal(metadata)
            g.Assert(req.Key).Equal(key)
            g.Assert(req.Value).Equal(value)
		})

		g.It("should create a get instance of Request", func() {
            req := builder.CreateGetRequest(key, metadata)

			g.Assert(reflect.TypeOf(req).String()).Equal("*spotcache.Request")
			g.Assert(req.Op).Equal(spotcache.GET)
        })

		g.It("should create a has request")
		g.It("should create a delete request")
		g.It("should create an expire request")
		g.It("should create a ttl request")
		g.It("should create a subscribe request")
		g.It("should create an unsubscribe request")
		g.It("should create a publish request")
		g.It("should create a status request")
		g.It("should create a ping request")
		g.It("should create a shutdown request")

		g.It("should create an instance of RequestBuilder")
		g.It("should create a byte stream from a request object")
		g.It("should read and parse a byte stream into a request object")
	})
}
<commit_msg>add get implementation and tests<commit_after>//
// @RequestTest
//
// @author darryl.west <darryl.west@raincitysoftware.com>
// @created 2017-03-19 12:09:24
//

package test

import (
	"reflect"
	"testing"

	"spotcache"

	. "github.com/franela/goblin"
)

func TestRequest(t *testing.T) {
	g := Goblin(t)

	g.Describe("Request", func() {
		cfg := spotcache.NewConfigForEnvironment("test")
		spotcache.CreateLogger(cfg)
		var session spotcache.SessionType
		copy(session[:12], []byte(spotcache.CreateSessionId()))

		builder := spotcache.NewRequestBuilder(session)
		key := []byte(spotcache.CreateULID())
		value := CreateRandomData()
		metadata := []byte("expire:40;")

        keysz := uint16(len(key))
        metasz := uint16(len(metadata))
        valsz := uint32(len(value))

        g.It("should create a new instance of Request from NewCommand", func() {
            req := builder.NewRequest(spotcache.NOOP)

			g.Assert(reflect.TypeOf(req).String()).Equal("spotcache.Request")
			g.Assert(len(req.Id)).Equal(26)
			g.Assert(len(req.Session)).Equal(12)
            g.Assert(req.Op).Equal(spotcache.NOOP)
        })

		g.It("should create a put instance of Request", func() {
			req := builder.CreatePutRequest(key, value, metadata)

			g.Assert(reflect.TypeOf(req).String()).Equal("*spotcache.Request")
			g.Assert(req.Op).Equal(spotcache.PUT)
            g.Assert(req.MetaSize).Equal(metasz)
            g.Assert(req.KeySize).Equal(keysz)
            g.Assert(req.DataSize).Equal(valsz)

            g.Assert(req.Metadata).Equal(metadata)
            g.Assert(req.Key).Equal(key)
            g.Assert(req.Value).Equal(value)
		})

		g.It("should create a get instance of Request", func() {
            req := builder.CreateGetRequest(key, metadata)

			g.Assert(reflect.TypeOf(req).String()).Equal("*spotcache.Request")
			g.Assert(req.Op).Equal(spotcache.GET)
            g.Assert(req.MetaSize).Equal(metasz)
            g.Assert(req.KeySize).Equal(keysz)
            g.Assert(req.DataSize).Equal(uint32(0))

            g.Assert(req.Metadata).Equal(metadata)
            g.Assert(req.Key).Equal(key)
            g.Assert(req.Value).Equal(make([]byte, 0))
        })

		g.It("should create a has request")
		g.It("should create a delete request")
		g.It("should create an expire request")
		g.It("should create a ttl request")
		g.It("should create a subscribe request")
		g.It("should create an unsubscribe request")
		g.It("should create a publish request")
		g.It("should create a status request")
		g.It("should create a ping request")
		g.It("should create a shutdown request")

		g.It("should create an instance of RequestBuilder")
		g.It("should create a byte stream from a request object")
		g.It("should read and parse a byte stream into a request object")
	})
}
<|endoftext|>


<commit_before>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package http

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"

	"github.com/couchbaselabs/tuqtng/network"
	"github.com/couchbaselabs/tuqtng/query"
	"github.com/gorilla/mux"
)

type HttpResponse struct {
	w       http.ResponseWriter
	results query.ValueChannel
	err     bool
}

func (this *HttpResponse) SendError(err error) {
	this.err = true
	showError(this.w, fmt.Sprintf("%v", err), 500)
	close(this.results)
}

func (this *HttpResponse) SendResult(val query.Value) {
	this.results <- val
}

func (this *HttpResponse) NoMoreResults() {
	close(this.results)
}

type HttpEndpoint struct {
	queryChannel network.QueryChannel
}

func NewHttpEndpoint(address string) *HttpEndpoint {
	rv := &HttpEndpoint{}

	r := mux.NewRouter()

	r.HandleFunc("/", welcome).Methods("GET")
	r.Handle("/query", rv).Methods("GET", "POST")

	go func() {
		err := http.ListenAndServe(address, r)
		if err != nil {
			log.Fatal("ListenAndServe: ", err)
		}
	}()

	return rv
}

func (this *HttpEndpoint) SendQueriesTo(queryChannel network.QueryChannel) {
	this.queryChannel = queryChannel
}

func welcome(w http.ResponseWriter, r *http.Request) {
	mustEncode(w, map[string]interface{}{
		"tuqtng":  "where no query has relaxed before",
		"version": "tuqtng 0.0",
	})
}

func (this *HttpEndpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	queryString := r.FormValue("q")
	if queryString == "" && r.Method == "POST" {
		queryStringBytes, err := ioutil.ReadAll(r.Body)
		if err == nil {
			queryString = string(queryStringBytes)
		}
	}

	if queryString == "" {
		showError(w, "Missing required query string", 500)
		return
	} else {
		log.Printf("Query String: %v", queryString)
	}

	response := HttpResponse{w: w, results: make(query.ValueChannel)}
	query := network.Query{
		Request:  network.UNQLStringQueryRequest{QueryString: queryString},
		Response: &response,
	}

	this.queryChannel <- query

	first := true
	count := 0
	for val := range response.results {
		if first {
			// open up our response
			fmt.Fprint(w, "{\n")
			fmt.Fprint(w, "    \"resultset\": [\n")
			first = false
		} else {
			fmt.Fprint(w, ",\n")
		}
		body, err := json.MarshalIndent(val, "        ", "    ")
		if err != nil {
			log.Printf("Unable to format result to display %#v, %v", val, err)
		} else {
			fmt.Fprintf(w, "        %v", string(body))
		}
		count++
	}

	if count == 0 {
		fmt.Fprint(w, "{\n")
		fmt.Fprint(w, "    \"resultset\": [")
	}

	if !response.err {
		fmt.Fprint(w, "\n    ],\n")
		fmt.Fprintf(w, "    \"total_rows\": %d\n", count)
		fmt.Fprint(w, "}\n")
	}
}

func mustEncode(w io.Writer, i interface{}) {
	if headered, ok := w.(http.ResponseWriter); ok {
		headered.Header().Set("Cache-Control", "no-cache")
		headered.Header().Set("Content-type", "application/json")
	}

	e := json.NewEncoder(w)
	if err := e.Encode(i); err != nil {
		panic(err)
	}
}

func showError(w http.ResponseWriter, msg string, code int) {
	log.Printf("Reporting error %v/%v", code, msg)
	http.Error(w, msg, code)
}
<commit_msg>fix http output when error<commit_after>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package http

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"

	"github.com/couchbaselabs/tuqtng/network"
	"github.com/couchbaselabs/tuqtng/query"
	"github.com/gorilla/mux"
)

type HttpResponse struct {
	w       http.ResponseWriter
	results query.ValueChannel
	err     bool
}

func (this *HttpResponse) SendError(err error) {
	this.err = true
	showError(this.w, fmt.Sprintf("%v", err), 500)
	close(this.results)
}

func (this *HttpResponse) SendResult(val query.Value) {
	this.results <- val
}

func (this *HttpResponse) NoMoreResults() {
	close(this.results)
}

type HttpEndpoint struct {
	queryChannel network.QueryChannel
}

func NewHttpEndpoint(address string) *HttpEndpoint {
	rv := &HttpEndpoint{}

	r := mux.NewRouter()

	r.HandleFunc("/", welcome).Methods("GET")
	r.Handle("/query", rv).Methods("GET", "POST")

	go func() {
		err := http.ListenAndServe(address, r)
		if err != nil {
			log.Fatal("ListenAndServe: ", err)
		}
	}()

	return rv
}

func (this *HttpEndpoint) SendQueriesTo(queryChannel network.QueryChannel) {
	this.queryChannel = queryChannel
}

func welcome(w http.ResponseWriter, r *http.Request) {
	mustEncode(w, map[string]interface{}{
		"tuqtng":  "where no query has relaxed before",
		"version": "tuqtng 0.0",
	})
}

func (this *HttpEndpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	queryString := r.FormValue("q")
	if queryString == "" && r.Method == "POST" {
		queryStringBytes, err := ioutil.ReadAll(r.Body)
		if err == nil {
			queryString = string(queryStringBytes)
		}
	}

	if queryString == "" {
		showError(w, "Missing required query string", 500)
		return
	} else {
		log.Printf("Query String: %v", queryString)
	}

	response := HttpResponse{w: w, results: make(query.ValueChannel)}
	query := network.Query{
		Request:  network.UNQLStringQueryRequest{QueryString: queryString},
		Response: &response,
	}

	this.queryChannel <- query

	first := true
	count := 0
	for val := range response.results {
		if first {
			// open up our response
			fmt.Fprint(w, "{\n")
			fmt.Fprint(w, "    \"resultset\": [\n")
			first = false
		} else {
			fmt.Fprint(w, ",\n")
		}
		body, err := json.MarshalIndent(val, "        ", "    ")
		if err != nil {
			log.Printf("Unable to format result to display %#v, %v", val, err)
		} else {
			fmt.Fprintf(w, "        %v", string(body))
		}
		count++
	}

	if !response.err {
		if count == 0 {
			fmt.Fprint(w, "{\n")
			fmt.Fprint(w, "    \"resultset\": [")
		}
		fmt.Fprint(w, "\n    ],\n")
		fmt.Fprintf(w, "    \"total_rows\": %d\n", count)
		fmt.Fprint(w, "}\n")
	}
}

func mustEncode(w io.Writer, i interface{}) {
	if headered, ok := w.(http.ResponseWriter); ok {
		headered.Header().Set("Cache-Control", "no-cache")
		headered.Header().Set("Content-type", "application/json")
	}

	e := json.NewEncoder(w)
	if err := e.Encode(i); err != nil {
		panic(err)
	}
}

func showError(w http.ResponseWriter, msg string, code int) {
	log.Printf("Reporting error %v/%v", code, msg)
	http.Error(w, msg, code)
}
<|endoftext|>


<commit_before>package sentry

import (
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/Sirupsen/logrus/hooks/sentry"
)

// default timeout
var timeout = 200 * time.Millisecond

// default logging hook level
var hookLevel = []logrus.Level{
	logrus.PanicLevel,
	logrus.ErrorLevel,
}

func SetTimeout(i time.Duration) {
	timeout = i * time.Millisecond
}

func SetLevels(levels []logrus.Level) {
	hookLevel = levels
}

func AddLevel(level logrus.Level) {
	hookLevel = append(hookLevel, level)
}

func Set(dsn string) {
	hook, err := logrus_sentry.NewSentryHook(dsn, hookLevel)
	if err != nil {
		return
	}
	hook.Timeout = timeout
	logrus.AddHook(hook)
}
<commit_msg>Change import path for sentry<commit_after>package sentry

import (
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/evalphobia/logrus_sentry"
)

// default timeout
var timeout = 200 * time.Millisecond

// default logging hook level
var hookLevel = []logrus.Level{
	logrus.PanicLevel,
	logrus.ErrorLevel,
}

func SetTimeout(i time.Duration) {
	timeout = i * time.Millisecond
}

func SetLevels(levels []logrus.Level) {
	hookLevel = levels
}

func AddLevel(level logrus.Level) {
	hookLevel = append(hookLevel, level)
}

func Set(dsn string) {
	hook, err := logrus_sentry.NewSentryHook(dsn, hookLevel)
	if err != nil {
		return
	}
	hook.Timeout = timeout
	logrus.AddHook(hook)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/go.net/websocket"
	"github.com/coreos/go-etcd/etcd"
	"github.com/fsouza/go-dockerclient"
	"github.com/go-martini/martini"
)

var debugMode bool

func debug(v ...interface{}) {
	if debugMode {
		log.Println(v...)
	}
}

func assert(err error, context string) {
	if err != nil {
		log.Fatalf("%s: %v", context, err)
	}
}

func getopt(name, dfault string) string {
	value := os.Getenv(name)
	if value == "" {
		value = dfault
	}
	return value
}

type Colorizer map[string]int

// returns up to 14 color escape codes (then repeats) for each unique key
func (c Colorizer) Get(key string) string {
	i, exists := c[key]
	if !exists {
		c[key] = len(c)
		i = c[key]
	}
	bright := "1;"
	if i%14 > 6 {
		bright = ""
	}
	return "\x1b[" + bright + "3" + strconv.Itoa(7-(i%7)) + "m"
}

func syslogStreamer(target Target, types []string, logstream chan *Log) {
	typestr := "," + strings.Join(types, ",") + ","
	for logline := range logstream {
		if typestr != ",," && !strings.Contains(typestr, logline.Type) {
			continue
		}
		tag, pid := getLogName(logline.Name)
		conn, err := net.Dial("udp", target.Addr)
		assert(err, "syslog")
		// bump up the packet size for large log lines
		assert(conn.SetWriteBuffer(1048576), "syslog")
		// HACK: Go's syslog package hardcodes the log format, so let's send our own message
		_, err = fmt.Fprintf(conn,
			"%s %s[%s]: %s",
			time.Now().Format("2006-01-02T15:04:05MST"),
			tag,
			pid,
			logline.Data)
		assert(err, "syslog")
	}
}

// getLogName returns a custom tag and PID for containers that
// match Deis' specific application name format. Otherwise,
// it returns the original name and 1 as the PID.
func getLogName(name string) (string, string) {
	// example regex that should match: go_v2.web.1
	r := regexp.MustCompile(`(^[a-z0-9-]+)_(v[0-9]+)\.([a-z-_]+\.[0-9]+)$`)
	match := r.FindStringSubmatch(name)
	if match == nil {
		return name, "1"
	} else {
		return match[1], match[3]
	}
}

func websocketStreamer(w http.ResponseWriter, req *http.Request, logstream chan *Log, closer chan bool) {
	websocket.Handler(func(conn *websocket.Conn) {
		for logline := range logstream {
			if req.URL.Query().Get("type") != "" && logline.Type != req.URL.Query().Get("type") {
				continue
			}
			_, err := conn.Write(append(marshal(logline), '\n'))
			if err != nil {
				closer <- true
				return
			}
		}
	}).ServeHTTP(w, req)
}

func httpStreamer(w http.ResponseWriter, req *http.Request, logstream chan *Log, multi bool) {
	var colors Colorizer
	var usecolor, usejson bool
	nameWidth := 16
	if req.URL.Query().Get("colors") != "off" {
		colors = make(Colorizer)
		usecolor = true
	}
	if req.Header.Get("Accept") == "application/json" {
		w.Header().Add("Content-Type", "application/json")
		usejson = true
	} else {
		w.Header().Add("Content-Type", "text/plain")
	}
	for logline := range logstream {
		if req.URL.Query().Get("types") != "" && logline.Type != req.URL.Query().Get("types") {
			continue
		}
		if usejson {
			w.Write(append(marshal(logline), '\n'))
		} else {
			if multi {
				if len(logline.Name) > nameWidth {
					nameWidth = len(logline.Name)
				}
				if usecolor {
					w.Write([]byte(fmt.Sprintf(
						"%s%"+strconv.Itoa(nameWidth)+"s|%s\x1b[0m\n",
						colors.Get(logline.Name), logline.Name, logline.Data,
					)))
				} else {
					w.Write([]byte(fmt.Sprintf(
						"%"+strconv.Itoa(nameWidth)+"s|%s\n", logline.Name, logline.Data,
					)))
				}
			} else {
				w.Write(append([]byte(logline.Data), '\n'))
			}
		}
		w.(http.Flusher).Flush()
	}
}

func main() {
	debugMode = getopt("DEBUG", "") != ""
	port := getopt("PORT", "8000")
	endpoint := getopt("DOCKER_HOST", "unix:///var/run/docker.sock")
	routespath := getopt("ROUTESPATH", "/var/lib/logspout")

	client, err := docker.NewClient(endpoint)
	assert(err, "docker")
	attacher := NewAttachManager(client)
	router := NewRouteManager(attacher)

	// HACK: if we are connecting to etcd, get the logger's connection
	// details from there
	if etcdHost := os.Getenv("ETCD_HOST"); etcdHost != "" {
		connectionString := []string{"http://" + etcdHost + ":4001"}
		debug("etcd:", connectionString[0])
		etcd := etcd.NewClient(connectionString)
		etcd.SetDialTimeout(3 * time.Second)
		hostResp, err := etcd.Get("/deis/logs/host", false, false)
		assert(err, "url")
		portResp, err := etcd.Get("/deis/logs/port", false, false)
		assert(err, "url")
		host := fmt.Sprintf("%s:%s", hostResp.Node.Value, portResp.Node.Value)
		log.Println("routing all to " + host)
		router.Add(&Route{Target: Target{Type: "syslog", Addr: host}})
	}

	if len(os.Args) > 1 {
		u, err := url.Parse(os.Args[1])
		assert(err, "url")
		log.Println("routing all to " + os.Args[1])
		router.Add(&Route{Target: Target{Type: u.Scheme, Addr: u.Host}})
	}

	if _, err := os.Stat(routespath); err == nil {
		log.Println("loading and persisting routes in " + routespath)
		assert(router.Load(RouteFileStore(routespath)), "persistor")
	}

	m := martini.Classic()

	m.Get("/logs(?:/(?P<predicate>[a-zA-Z]+):(?P<value>.+))?", func(w http.ResponseWriter, req *http.Request, params martini.Params) {
		source := new(Source)
		switch {
		case params["predicate"] == "id" && params["value"] != "":
			source.ID = params["value"][:12]
		case params["predicate"] == "name" && params["value"] != "":
			source.Name = params["value"]
		case params["predicate"] == "filter" && params["value"] != "":
			source.Filter = params["value"]
		}

		if source.ID != "" && attacher.Get(source.ID) == nil {
			http.NotFound(w, req)
			return
		}

		logstream := make(chan *Log)
		defer close(logstream)

		var closer <-chan bool
		if req.Header.Get("Upgrade") == "websocket" {
			closerBi := make(chan bool)
			go websocketStreamer(w, req, logstream, closerBi)
			closer = closerBi
		} else {
			go httpStreamer(w, req, logstream, source.All() || source.Filter != "")
			closer = w.(http.CloseNotifier).CloseNotify()
		}

		attacher.Listen(source, logstream, closer)
	})

	m.Get("/routes", func(w http.ResponseWriter, req *http.Request) {
		w.Header().Add("Content-Type", "application/json")
		routes, _ := router.GetAll()
		w.Write(append(marshal(routes), '\n'))
	})

	m.Post("/routes", func(w http.ResponseWriter, req *http.Request) (int, string) {
		route := new(Route)
		if err := unmarshal(req.Body, route); err != nil {
			return http.StatusBadRequest, "Bad request: " + err.Error()
		}

		// TODO: validate?
		router.Add(route)

		w.Header().Add("Content-Type", "application/json")
		return http.StatusCreated, string(append(marshal(route), '\n'))
	})

	m.Get("/routes/:id", func(w http.ResponseWriter, req *http.Request, params martini.Params) {
		route, _ := router.Get(params["id"])
		if route == nil {
			http.NotFound(w, req)
			return
		}
		w.Write(append(marshal(route), '\n'))
	})

	m.Delete("/routes/:id", func(w http.ResponseWriter, req *http.Request, params martini.Params) {
		if ok := router.Remove(params["id"]); !ok {
			http.NotFound(w, req)
		}
	})

	log.Println("logspout serving http on :" + port)
	log.Fatal(http.ListenAndServe(":"+port, m))
}
<commit_msg>ref(logspout): use UDP conn<commit_after>package main

import (
	"fmt"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/go.net/websocket"
	"github.com/coreos/go-etcd/etcd"
	"github.com/fsouza/go-dockerclient"
	"github.com/go-martini/martini"
)

var debugMode bool

func debug(v ...interface{}) {
	if debugMode {
		log.Println(v...)
	}
}

func assert(err error, context string) {
	if err != nil {
		log.Fatalf("%s: %v", context, err)
	}
}

func getopt(name, dfault string) string {
	value := os.Getenv(name)
	if value == "" {
		value = dfault
	}
	return value
}

type Colorizer map[string]int

// returns up to 14 color escape codes (then repeats) for each unique key
func (c Colorizer) Get(key string) string {
	i, exists := c[key]
	if !exists {
		c[key] = len(c)
		i = c[key]
	}
	bright := "1;"
	if i%14 > 6 {
		bright = ""
	}
	return "\x1b[" + bright + "3" + strconv.Itoa(7-(i%7)) + "m"
}

func syslogStreamer(target Target, types []string, logstream chan *Log) {
	typestr := "," + strings.Join(types, ",") + ","
	for logline := range logstream {
		if typestr != ",," && !strings.Contains(typestr, logline.Type) {
			continue
		}
		tag, pid := getLogName(logline.Name)
		addr, err := net.ResolveUDPAddr("udp", target.Addr)
		assert(err, "syslog")
		conn, err := net.DialUDP("udp", nil, addr)
		assert(err, "syslog")
		// bump up the packet size for large log lines
		assert(conn.SetWriteBuffer(1048576), "syslog")
		// HACK: Go's syslog package hardcodes the log format, so let's send our own message
		_, err = fmt.Fprintf(conn,
			"%s %s[%s]: %s",
			time.Now().Format("2006-01-02T15:04:05MST"),
			tag,
			pid,
			logline.Data)
		assert(err, "syslog")
	}
}

// getLogName returns a custom tag and PID for containers that
// match Deis' specific application name format. Otherwise,
// it returns the original name and 1 as the PID.
func getLogName(name string) (string, string) {
	// example regex that should match: go_v2.web.1
	r := regexp.MustCompile(`(^[a-z0-9-]+)_(v[0-9]+)\.([a-z-_]+\.[0-9]+)$`)
	match := r.FindStringSubmatch(name)
	if match == nil {
		return name, "1"
	} else {
		return match[1], match[3]
	}
}

func websocketStreamer(w http.ResponseWriter, req *http.Request, logstream chan *Log, closer chan bool) {
	websocket.Handler(func(conn *websocket.Conn) {
		for logline := range logstream {
			if req.URL.Query().Get("type") != "" && logline.Type != req.URL.Query().Get("type") {
				continue
			}
			_, err := conn.Write(append(marshal(logline), '\n'))
			if err != nil {
				closer <- true
				return
			}
		}
	}).ServeHTTP(w, req)
}

func httpStreamer(w http.ResponseWriter, req *http.Request, logstream chan *Log, multi bool) {
	var colors Colorizer
	var usecolor, usejson bool
	nameWidth := 16
	if req.URL.Query().Get("colors") != "off" {
		colors = make(Colorizer)
		usecolor = true
	}
	if req.Header.Get("Accept") == "application/json" {
		w.Header().Add("Content-Type", "application/json")
		usejson = true
	} else {
		w.Header().Add("Content-Type", "text/plain")
	}
	for logline := range logstream {
		if req.URL.Query().Get("types") != "" && logline.Type != req.URL.Query().Get("types") {
			continue
		}
		if usejson {
			w.Write(append(marshal(logline), '\n'))
		} else {
			if multi {
				if len(logline.Name) > nameWidth {
					nameWidth = len(logline.Name)
				}
				if usecolor {
					w.Write([]byte(fmt.Sprintf(
						"%s%"+strconv.Itoa(nameWidth)+"s|%s\x1b[0m\n",
						colors.Get(logline.Name), logline.Name, logline.Data,
					)))
				} else {
					w.Write([]byte(fmt.Sprintf(
						"%"+strconv.Itoa(nameWidth)+"s|%s\n", logline.Name, logline.Data,
					)))
				}
			} else {
				w.Write(append([]byte(logline.Data), '\n'))
			}
		}
		w.(http.Flusher).Flush()
	}
}

func main() {
	debugMode = getopt("DEBUG", "") != ""
	port := getopt("PORT", "8000")
	endpoint := getopt("DOCKER_HOST", "unix:///var/run/docker.sock")
	routespath := getopt("ROUTESPATH", "/var/lib/logspout")

	client, err := docker.NewClient(endpoint)
	assert(err, "docker")
	attacher := NewAttachManager(client)
	router := NewRouteManager(attacher)

	// HACK: if we are connecting to etcd, get the logger's connection
	// details from there
	if etcdHost := os.Getenv("ETCD_HOST"); etcdHost != "" {
		connectionString := []string{"http://" + etcdHost + ":4001"}
		debug("etcd:", connectionString[0])
		etcd := etcd.NewClient(connectionString)
		etcd.SetDialTimeout(3 * time.Second)
		hostResp, err := etcd.Get("/deis/logs/host", false, false)
		assert(err, "url")
		portResp, err := etcd.Get("/deis/logs/port", false, false)
		assert(err, "url")
		host := fmt.Sprintf("%s:%s", hostResp.Node.Value, portResp.Node.Value)
		log.Println("routing all to " + host)
		router.Add(&Route{Target: Target{Type: "syslog", Addr: host}})
	}

	if len(os.Args) > 1 {
		u, err := url.Parse(os.Args[1])
		assert(err, "url")
		log.Println("routing all to " + os.Args[1])
		router.Add(&Route{Target: Target{Type: u.Scheme, Addr: u.Host}})
	}

	if _, err := os.Stat(routespath); err == nil {
		log.Println("loading and persisting routes in " + routespath)
		assert(router.Load(RouteFileStore(routespath)), "persistor")
	}

	m := martini.Classic()

	m.Get("/logs(?:/(?P<predicate>[a-zA-Z]+):(?P<value>.+))?", func(w http.ResponseWriter, req *http.Request, params martini.Params) {
		source := new(Source)
		switch {
		case params["predicate"] == "id" && params["value"] != "":
			source.ID = params["value"][:12]
		case params["predicate"] == "name" && params["value"] != "":
			source.Name = params["value"]
		case params["predicate"] == "filter" && params["value"] != "":
			source.Filter = params["value"]
		}

		if source.ID != "" && attacher.Get(source.ID) == nil {
			http.NotFound(w, req)
			return
		}

		logstream := make(chan *Log)
		defer close(logstream)

		var closer <-chan bool
		if req.Header.Get("Upgrade") == "websocket" {
			closerBi := make(chan bool)
			go websocketStreamer(w, req, logstream, closerBi)
			closer = closerBi
		} else {
			go httpStreamer(w, req, logstream, source.All() || source.Filter != "")
			closer = w.(http.CloseNotifier).CloseNotify()
		}

		attacher.Listen(source, logstream, closer)
	})

	m.Get("/routes", func(w http.ResponseWriter, req *http.Request) {
		w.Header().Add("Content-Type", "application/json")
		routes, _ := router.GetAll()
		w.Write(append(marshal(routes), '\n'))
	})

	m.Post("/routes", func(w http.ResponseWriter, req *http.Request) (int, string) {
		route := new(Route)
		if err := unmarshal(req.Body, route); err != nil {
			return http.StatusBadRequest, "Bad request: " + err.Error()
		}

		// TODO: validate?
		router.Add(route)

		w.Header().Add("Content-Type", "application/json")
		return http.StatusCreated, string(append(marshal(route), '\n'))
	})

	m.Get("/routes/:id", func(w http.ResponseWriter, req *http.Request, params martini.Params) {
		route, _ := router.Get(params["id"])
		if route == nil {
			http.NotFound(w, req)
			return
		}
		w.Write(append(marshal(route), '\n'))
	})

	m.Delete("/routes/:id", func(w http.ResponseWriter, req *http.Request, params martini.Params) {
		if ok := router.Remove(params["id"]); !ok {
			http.NotFound(w, req)
		}
	})

	log.Println("logspout serving http on :" + port)
	log.Fatal(http.ListenAndServe(":"+port, m))
}
<|endoftext|>