<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package httputil

import (
	"bufio"
	"errors"
	"io"
	"net"
	"net/http"
	"net/textproto"
	"sync"
)

var (
	ErrPersistEOF = &http.ProtocolError{ErrorString: "persistent connection closed"}
	ErrClosed     = &http.ProtocolError{ErrorString: "connection closed by user"}
	ErrPipeline   = &http.ProtocolError{ErrorString: "pipeline error"}
)

// This is an API usage error - the local side is closed.
// ErrPersistEOF (above) reports that the remote side is closed.
var errClosed = errors.New("i/o operation on closed connection")

// A ServerConn reads requests and sends responses over an underlying
// connection, until the HTTP keepalive logic commands an end. ServerConn
// also allows hijacking the underlying connection by calling Hijack
// to regain control over the connection. ServerConn supports pipe-lining,
// i.e. requests can be read out of sync (but in the same order) while the
// respective responses are sent.
//
// ServerConn is low-level and old. Applications should instead use Server
// in the net/http package.
type ServerConn struct {
	mu              sync.Mutex // read-write protects the following fields
	c               net.Conn
	r               *bufio.Reader
	re, we          error // read/write errors
	lastbody        io.ReadCloser
	nread, nwritten int
	pipereq         map[*http.Request]uint

	pipe textproto.Pipeline
}

// NewServerConn returns a new ServerConn reading and writing c. If r is not
// nil, it is the buffer to use when reading c.
//
// ServerConn is low-level and old. Applications should instead use Server
// in the net/http package.
func NewServerConn(c net.Conn, r *bufio.Reader) *ServerConn {
	if r == nil {
		r = bufio.NewReader(c)
	}
	return &ServerConn{c: c, r: r, pipereq: make(map[*http.Request]uint)}
}

// Hijack detaches the ServerConn and returns the underlying connection as well
// as the read-side bufio which may have some left over data. Hijack may be
// called before Read has signaled the end of the keep-alive logic. The user
// should not call Hijack while Read or Write is in progress.
func (sc *ServerConn) Hijack() (c net.Conn, r *bufio.Reader) {
	sc.mu.Lock()
	defer sc.mu.Unlock()
	c = sc.c
	r = sc.r
	sc.c = nil
	sc.r = nil
	return
}

// Close calls Hijack and then also closes the underlying connection
func (sc *ServerConn) Close() error {
	c, _ := sc.Hijack()
	if c != nil {
		return c.Close()
	}
	return nil
}

// Read returns the next request on the wire. An ErrPersistEOF is returned if
// it is gracefully determined that there are no more requests (e.g. after the
// first request on an HTTP/1.0 connection, or after a Connection:close on a
// HTTP/1.1 connection).
func (sc *ServerConn) Read() (req *http.Request, err error) {

	// Ensure ordered execution of Reads and Writes
	id := sc.pipe.Next()
	sc.pipe.StartRequest(id)
	defer func() {
		sc.pipe.EndRequest(id)
		if req == nil {
			sc.pipe.StartResponse(id)
			sc.pipe.EndResponse(id)
		} else {
			// Remember the pipeline id of this request
			sc.mu.Lock()
			sc.pipereq[req] = id
			sc.mu.Unlock()
		}
	}()

	sc.mu.Lock()
	if sc.we != nil { // no point receiving if write-side broken or closed
		defer sc.mu.Unlock()
		return nil, sc.we
	}
	if sc.re != nil {
		defer sc.mu.Unlock()
		return nil, sc.re
	}
	if sc.r == nil { // connection closed by user in the meantime
		defer sc.mu.Unlock()
		return nil, errClosed
	}
	r := sc.r
	lastbody := sc.lastbody
	sc.lastbody = nil
	sc.mu.Unlock()

	// Make sure body is fully consumed, even if user does not call body.Close
	if lastbody != nil {
		// body.Close is assumed to be idempotent and multiple calls to
		// it should return the error that its first invocation
		// returned.
		err = lastbody.Close()
		if err != nil {
			sc.mu.Lock()
			defer sc.mu.Unlock()
			sc.re = err
			return nil, err
		}
	}

	req, err = http.ReadRequest(r)
	sc.mu.Lock()
	defer sc.mu.Unlock()
	if err != nil {
		if err == io.ErrUnexpectedEOF {
			// A close from the opposing client is treated as a
			// graceful close, even if there was some unparse-able
			// data before the close.
			sc.re = ErrPersistEOF
			return nil, sc.re
		} else {
			sc.re = err
			return req, err
		}
	}
	sc.lastbody = req.Body
	sc.nread++
	if req.Close {
		sc.re = ErrPersistEOF
		return req, sc.re
	}
	return req, err
}

// Pending returns the number of unanswered requests
// that have been received on the connection.
func (sc *ServerConn) Pending() int {
	sc.mu.Lock()
	defer sc.mu.Unlock()
	return sc.nread - sc.nwritten
}

// Write writes resp in response to req. To close the connection gracefully, set the
// Response.Close field to true. Write should be considered operational until
// it returns an error, regardless of any errors returned on the Read side.
func (sc *ServerConn) Write(req *http.Request, resp *http.Response) error {

	// Retrieve the pipeline ID of this request/response pair
	sc.mu.Lock()
	id, ok := sc.pipereq[req]
	delete(sc.pipereq, req)
	if !ok {
		sc.mu.Unlock()
		return ErrPipeline
	}
	sc.mu.Unlock()

	// Ensure pipeline order
	sc.pipe.StartResponse(id)
	defer sc.pipe.EndResponse(id)

	sc.mu.Lock()
	if sc.we != nil {
		defer sc.mu.Unlock()
		return sc.we
	}
	if sc.c == nil { // connection closed by user in the meantime
		defer sc.mu.Unlock()
		return ErrClosed
	}
	c := sc.c
	if sc.nread <= sc.nwritten {
		defer sc.mu.Unlock()
		return errors.New("persist server pipe count")
	}
	if resp.Close {
		// After signaling a keep-alive close, any pipelined unread
		// requests will be lost. It is up to the user to drain them
		// before signaling.
		sc.re = ErrPersistEOF
	}
	sc.mu.Unlock()

	err := resp.Write(c)
	sc.mu.Lock()
	defer sc.mu.Unlock()
	if err != nil {
		sc.we = err
		return err
	}
	sc.nwritten++

	return nil
}

// A ClientConn sends request and receives headers over an underlying
// connection, while respecting the HTTP keepalive logic. ClientConn
// supports hijacking the connection calling Hijack to
// regain control of the underlying net.Conn and deal with it as desired.
//
// ClientConn is low-level and old. Applications should instead use
// Client or Transport in the net/http package.
type ClientConn struct {
	mu              sync.Mutex // read-write protects the following fields
	c               net.Conn
	r               *bufio.Reader
	re, we          error // read/write errors
	lastbody        io.ReadCloser
	nread, nwritten int
	pipereq         map[*http.Request]uint

	pipe     textproto.Pipeline
	writeReq func(*http.Request, io.Writer) error
}

// NewClientConn returns a new ClientConn reading and writing c. If r is not
// nil, it is the buffer to use when reading c.
//
// ClientConn is low-level and old. Applications should use Client or
// Transport in the net/http package.
func NewClientConn(c net.Conn, r *bufio.Reader) *ClientConn {
	if r == nil {
		r = bufio.NewReader(c)
	}
	return &ClientConn{
		c:        c,
		r:        r,
		pipereq:  make(map[*http.Request]uint),
		writeReq: (*http.Request).Write,
	}
}

// NewProxyClientConn works like NewClientConn but writes Requests
// using Request's WriteProxy method.
//
// New code should not use NewProxyClientConn. See Client or
// Transport in the net/http package instead.
func NewProxyClientConn(c net.Conn, r *bufio.Reader) *ClientConn {
	cc := NewClientConn(c, r)
	cc.writeReq = (*http.Request).WriteProxy
	return cc
}

// Hijack detaches the ClientConn and returns the underlying connection as well
// as the read-side bufio which may have some left over data. Hijack may be
// called before the user or Read have signaled the end of the keep-alive
// logic. The user should not call Hijack while Read or Write is in progress.
func (cc *ClientConn) Hijack() (c net.Conn, r *bufio.Reader) {
	cc.mu.Lock()
	defer cc.mu.Unlock()
	c = cc.c
	r = cc.r
	cc.c = nil
	cc.r = nil
	return
}

// Close calls Hijack and then also closes the underlying connection
func (cc *ClientConn) Close() error {
	c, _ := cc.Hijack()
	if c != nil {
		return c.Close()
	}
	return nil
}

// Write writes a request. An ErrPersistEOF error is returned if the connection
// has been closed in an HTTP keepalive sense. If req.Close equals true, the
// keepalive connection is logically closed after this request and the opposing
// server is informed. An ErrUnexpectedEOF indicates the remote closed the
// underlying TCP connection, which is usually considered as graceful close.
func (cc *ClientConn) Write(req *http.Request) (err error) {

	// Ensure ordered execution of Writes
	id := cc.pipe.Next()
	cc.pipe.StartRequest(id)
	defer func() {
		cc.pipe.EndRequest(id)
		if err != nil {
			cc.pipe.StartResponse(id)
			cc.pipe.EndResponse(id)
		} else {
			// Remember the pipeline id of this request
			cc.mu.Lock()
			cc.pipereq[req] = id
			cc.mu.Unlock()
		}
	}()

	cc.mu.Lock()
	if cc.re != nil { // no point sending if read-side closed or broken
		defer cc.mu.Unlock()
		return cc.re
	}
	if cc.we != nil {
		defer cc.mu.Unlock()
		return cc.we
	}
	if cc.c == nil { // connection closed by user in the meantime
		defer cc.mu.Unlock()
		return errClosed
	}
	c := cc.c
	if req.Close {
		// We write the EOF to the write-side error, because there
		// still might be some pipelined reads
		cc.we = ErrPersistEOF
	}
	cc.mu.Unlock()

	err = cc.writeReq(req, c)
	cc.mu.Lock()
	defer cc.mu.Unlock()
	if err != nil {
		cc.we = err
		return err
	}
	cc.nwritten++

	return nil
}

// Pending returns the number of unanswered requests
// that have been sent on the connection.
func (cc *ClientConn) Pending() int {
	cc.mu.Lock()
	defer cc.mu.Unlock()
	return cc.nwritten - cc.nread
}

// Read reads the next response from the wire. A valid response might be
// returned together with an ErrPersistEOF, which means that the remote
// requested that this be the last request serviced. Read can be called
// concurrently with Write, but not with another Read.
func (cc *ClientConn) Read(req *http.Request) (resp *http.Response, err error) {
	// Retrieve the pipeline ID of this request/response pair
	cc.mu.Lock()
	id, ok := cc.pipereq[req]
	delete(cc.pipereq, req)
	if !ok {
		cc.mu.Unlock()
		return nil, ErrPipeline
	}
	cc.mu.Unlock()

	// Ensure pipeline order
	cc.pipe.StartResponse(id)
	defer cc.pipe.EndResponse(id)

	cc.mu.Lock()
	if cc.re != nil {
		defer cc.mu.Unlock()
		return nil, cc.re
	}
	if cc.r == nil { // connection closed by user in the meantime
		defer cc.mu.Unlock()
		return nil, errClosed
	}
	r := cc.r
	lastbody := cc.lastbody
	cc.lastbody = nil
	cc.mu.Unlock()

	// Make sure body is fully consumed, even if user does not call body.Close
	if lastbody != nil {
		// body.Close is assumed to be idempotent and multiple calls to
		// it should return the error that its first invocation
		// returned.
		err = lastbody.Close()
		if err != nil {
			cc.mu.Lock()
			defer cc.mu.Unlock()
			cc.re = err
			return nil, err
		}
	}

	resp, err = http.ReadResponse(r, req)
	cc.mu.Lock()
	defer cc.mu.Unlock()
	if err != nil {
		cc.re = err
		return resp, err
	}
	cc.lastbody = resp.Body

	cc.nread++

	if resp.Close {
		cc.re = ErrPersistEOF // don't send any more requests
		return resp, cc.re
	}
	return resp, err
}

// Do is convenience method that writes a request and reads a response.
func (cc *ClientConn) Do(req *http.Request) (resp *http.Response, err error) {
	err = cc.Write(req)
	if err != nil {
		return
	}
	return cc.Read(req)
}
<commit_msg>net/http/httputil: deprecate Client+ServerConn harder, use Deprecated syntax<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package httputil

import (
	"bufio"
	"errors"
	"io"
	"net"
	"net/http"
	"net/textproto"
	"sync"
)

var (
	ErrPersistEOF = &http.ProtocolError{ErrorString: "persistent connection closed"}
	ErrClosed     = &http.ProtocolError{ErrorString: "connection closed by user"}
	ErrPipeline   = &http.ProtocolError{ErrorString: "pipeline error"}
)

// This is an API usage error - the local side is closed.
// ErrPersistEOF (above) reports that the remote side is closed.
var errClosed = errors.New("i/o operation on closed connection")

// ServerConn is an artifact of Go's early HTTP implementation.
// Is is low-level, old, and unused by Go's current HTTP stack.
// We should have deleted it before Go 1.
//
// Deprecated: use the Server in package net/http instead.
type ServerConn struct {
	mu              sync.Mutex // read-write protects the following fields
	c               net.Conn
	r               *bufio.Reader
	re, we          error // read/write errors
	lastbody        io.ReadCloser
	nread, nwritten int
	pipereq         map[*http.Request]uint

	pipe textproto.Pipeline
}

// NewServerConn is an artifact of Go's early HTTP implementation.
// Is is low-level, old, and unused by Go's current HTTP stack.
// We should have deleted it before Go 1.
//
// Deprecated: use the Server in package net/http instead.
func NewServerConn(c net.Conn, r *bufio.Reader) *ServerConn {
	if r == nil {
		r = bufio.NewReader(c)
	}
	return &ServerConn{c: c, r: r, pipereq: make(map[*http.Request]uint)}
}

// Hijack detaches the ServerConn and returns the underlying connection as well
// as the read-side bufio which may have some left over data. Hijack may be
// called before Read has signaled the end of the keep-alive logic. The user
// should not call Hijack while Read or Write is in progress.
func (sc *ServerConn) Hijack() (net.Conn, *bufio.Reader) {
	sc.mu.Lock()
	defer sc.mu.Unlock()
	c := sc.c
	r := sc.r
	sc.c = nil
	sc.r = nil
	return c, r
}

// Close calls Hijack and then also closes the underlying connection.
func (sc *ServerConn) Close() error {
	c, _ := sc.Hijack()
	if c != nil {
		return c.Close()
	}
	return nil
}

// Read returns the next request on the wire. An ErrPersistEOF is returned if
// it is gracefully determined that there are no more requests (e.g. after the
// first request on an HTTP/1.0 connection, or after a Connection:close on a
// HTTP/1.1 connection).
func (sc *ServerConn) Read() (*http.Request, error) {
	var req *http.Request
	var err error

	// Ensure ordered execution of Reads and Writes
	id := sc.pipe.Next()
	sc.pipe.StartRequest(id)
	defer func() {
		sc.pipe.EndRequest(id)
		if req == nil {
			sc.pipe.StartResponse(id)
			sc.pipe.EndResponse(id)
		} else {
			// Remember the pipeline id of this request
			sc.mu.Lock()
			sc.pipereq[req] = id
			sc.mu.Unlock()
		}
	}()

	sc.mu.Lock()
	if sc.we != nil { // no point receiving if write-side broken or closed
		defer sc.mu.Unlock()
		return nil, sc.we
	}
	if sc.re != nil {
		defer sc.mu.Unlock()
		return nil, sc.re
	}
	if sc.r == nil { // connection closed by user in the meantime
		defer sc.mu.Unlock()
		return nil, errClosed
	}
	r := sc.r
	lastbody := sc.lastbody
	sc.lastbody = nil
	sc.mu.Unlock()

	// Make sure body is fully consumed, even if user does not call body.Close
	if lastbody != nil {
		// body.Close is assumed to be idempotent and multiple calls to
		// it should return the error that its first invocation
		// returned.
		err = lastbody.Close()
		if err != nil {
			sc.mu.Lock()
			defer sc.mu.Unlock()
			sc.re = err
			return nil, err
		}
	}

	req, err = http.ReadRequest(r)
	sc.mu.Lock()
	defer sc.mu.Unlock()
	if err != nil {
		if err == io.ErrUnexpectedEOF {
			// A close from the opposing client is treated as a
			// graceful close, even if there was some unparse-able
			// data before the close.
			sc.re = ErrPersistEOF
			return nil, sc.re
		} else {
			sc.re = err
			return req, err
		}
	}
	sc.lastbody = req.Body
	sc.nread++
	if req.Close {
		sc.re = ErrPersistEOF
		return req, sc.re
	}
	return req, err
}

// Pending returns the number of unanswered requests
// that have been received on the connection.
func (sc *ServerConn) Pending() int {
	sc.mu.Lock()
	defer sc.mu.Unlock()
	return sc.nread - sc.nwritten
}

// Write writes resp in response to req. To close the connection gracefully, set the
// Response.Close field to true. Write should be considered operational until
// it returns an error, regardless of any errors returned on the Read side.
func (sc *ServerConn) Write(req *http.Request, resp *http.Response) error {

	// Retrieve the pipeline ID of this request/response pair
	sc.mu.Lock()
	id, ok := sc.pipereq[req]
	delete(sc.pipereq, req)
	if !ok {
		sc.mu.Unlock()
		return ErrPipeline
	}
	sc.mu.Unlock()

	// Ensure pipeline order
	sc.pipe.StartResponse(id)
	defer sc.pipe.EndResponse(id)

	sc.mu.Lock()
	if sc.we != nil {
		defer sc.mu.Unlock()
		return sc.we
	}
	if sc.c == nil { // connection closed by user in the meantime
		defer sc.mu.Unlock()
		return ErrClosed
	}
	c := sc.c
	if sc.nread <= sc.nwritten {
		defer sc.mu.Unlock()
		return errors.New("persist server pipe count")
	}
	if resp.Close {
		// After signaling a keep-alive close, any pipelined unread
		// requests will be lost. It is up to the user to drain them
		// before signaling.
		sc.re = ErrPersistEOF
	}
	sc.mu.Unlock()

	err := resp.Write(c)
	sc.mu.Lock()
	defer sc.mu.Unlock()
	if err != nil {
		sc.we = err
		return err
	}
	sc.nwritten++

	return nil
}

// ClientConn is an artifact of Go's early HTTP implementation.
// Is is low-level, old, and unused by Go's current HTTP stack.
// We should have deleted it before Go 1.
//
// Deprecated: use Client or Transport in package net/http instead.
type ClientConn struct {
	mu              sync.Mutex // read-write protects the following fields
	c               net.Conn
	r               *bufio.Reader
	re, we          error // read/write errors
	lastbody        io.ReadCloser
	nread, nwritten int
	pipereq         map[*http.Request]uint

	pipe     textproto.Pipeline
	writeReq func(*http.Request, io.Writer) error
}

// NewClientConn is an artifact of Go's early HTTP implementation.
// Is is low-level, old, and unused by Go's current HTTP stack.
// We should have deleted it before Go 1.
//
// Deprecated: use the Client or Transport in package net/http instead.
func NewClientConn(c net.Conn, r *bufio.Reader) *ClientConn {
	if r == nil {
		r = bufio.NewReader(c)
	}
	return &ClientConn{
		c:        c,
		r:        r,
		pipereq:  make(map[*http.Request]uint),
		writeReq: (*http.Request).Write,
	}
}

// NewProxyClientConn is an artifact of Go's early HTTP implementation.
// Is is low-level, old, and unused by Go's current HTTP stack.
// We should have deleted it before Go 1.
//
// Deprecated: use the Client or Transport in package net/http instead.
func NewProxyClientConn(c net.Conn, r *bufio.Reader) *ClientConn {
	cc := NewClientConn(c, r)
	cc.writeReq = (*http.Request).WriteProxy
	return cc
}

// Hijack detaches the ClientConn and returns the underlying connection as well
// as the read-side bufio which may have some left over data. Hijack may be
// called before the user or Read have signaled the end of the keep-alive
// logic. The user should not call Hijack while Read or Write is in progress.
func (cc *ClientConn) Hijack() (c net.Conn, r *bufio.Reader) {
	cc.mu.Lock()
	defer cc.mu.Unlock()
	c = cc.c
	r = cc.r
	cc.c = nil
	cc.r = nil
	return
}

// Close calls Hijack and then also closes the underlying connection.
func (cc *ClientConn) Close() error {
	c, _ := cc.Hijack()
	if c != nil {
		return c.Close()
	}
	return nil
}

// Write writes a request. An ErrPersistEOF error is returned if the connection
// has been closed in an HTTP keepalive sense. If req.Close equals true, the
// keepalive connection is logically closed after this request and the opposing
// server is informed. An ErrUnexpectedEOF indicates the remote closed the
// underlying TCP connection, which is usually considered as graceful close.
func (cc *ClientConn) Write(req *http.Request) error {
	var err error

	// Ensure ordered execution of Writes
	id := cc.pipe.Next()
	cc.pipe.StartRequest(id)
	defer func() {
		cc.pipe.EndRequest(id)
		if err != nil {
			cc.pipe.StartResponse(id)
			cc.pipe.EndResponse(id)
		} else {
			// Remember the pipeline id of this request
			cc.mu.Lock()
			cc.pipereq[req] = id
			cc.mu.Unlock()
		}
	}()

	cc.mu.Lock()
	if cc.re != nil { // no point sending if read-side closed or broken
		defer cc.mu.Unlock()
		return cc.re
	}
	if cc.we != nil {
		defer cc.mu.Unlock()
		return cc.we
	}
	if cc.c == nil { // connection closed by user in the meantime
		defer cc.mu.Unlock()
		return errClosed
	}
	c := cc.c
	if req.Close {
		// We write the EOF to the write-side error, because there
		// still might be some pipelined reads
		cc.we = ErrPersistEOF
	}
	cc.mu.Unlock()

	err = cc.writeReq(req, c)
	cc.mu.Lock()
	defer cc.mu.Unlock()
	if err != nil {
		cc.we = err
		return err
	}
	cc.nwritten++

	return nil
}

// Pending returns the number of unanswered requests
// that have been sent on the connection.
func (cc *ClientConn) Pending() int {
	cc.mu.Lock()
	defer cc.mu.Unlock()
	return cc.nwritten - cc.nread
}

// Read reads the next response from the wire. A valid response might be
// returned together with an ErrPersistEOF, which means that the remote
// requested that this be the last request serviced. Read can be called
// concurrently with Write, but not with another Read.
func (cc *ClientConn) Read(req *http.Request) (resp *http.Response, err error) {
	// Retrieve the pipeline ID of this request/response pair
	cc.mu.Lock()
	id, ok := cc.pipereq[req]
	delete(cc.pipereq, req)
	if !ok {
		cc.mu.Unlock()
		return nil, ErrPipeline
	}
	cc.mu.Unlock()

	// Ensure pipeline order
	cc.pipe.StartResponse(id)
	defer cc.pipe.EndResponse(id)

	cc.mu.Lock()
	if cc.re != nil {
		defer cc.mu.Unlock()
		return nil, cc.re
	}
	if cc.r == nil { // connection closed by user in the meantime
		defer cc.mu.Unlock()
		return nil, errClosed
	}
	r := cc.r
	lastbody := cc.lastbody
	cc.lastbody = nil
	cc.mu.Unlock()

	// Make sure body is fully consumed, even if user does not call body.Close
	if lastbody != nil {
		// body.Close is assumed to be idempotent and multiple calls to
		// it should return the error that its first invocation
		// returned.
		err = lastbody.Close()
		if err != nil {
			cc.mu.Lock()
			defer cc.mu.Unlock()
			cc.re = err
			return nil, err
		}
	}

	resp, err = http.ReadResponse(r, req)
	cc.mu.Lock()
	defer cc.mu.Unlock()
	if err != nil {
		cc.re = err
		return resp, err
	}
	cc.lastbody = resp.Body

	cc.nread++

	if resp.Close {
		cc.re = ErrPersistEOF // don't send any more requests
		return resp, cc.re
	}
	return resp, err
}

// Do is convenience method that writes a request and reads a response.
func (cc *ClientConn) Do(req *http.Request) (*http.Response, error) {
	err := cc.Write(req)
	if err != nil {
		return nil, err
	}
	return cc.Read(req)
}
<|endoftext|>


<commit_before>package keyring

import (
	"os"
	"os/user"

	keychain "github.com/99designs/aws-vault/Godeps/_workspace/src/github.com/99designs/go-osxkeychain"
)

var keychainFile string

type OSXKeychain struct {
	path string
}

func (k *OSXKeychain) Get(service, key string) ([]byte, error) {
	attributes := keychain.GenericPasswordAttributes{
		ServiceName: service,
		AccountName: key,
	}

	if k.path != "" {
		attributes.Keychain = []string{k.path}
	}

	if b, err := keychain.FindGenericPassword(&attributes); err == keychain.ErrItemNotFound {
		return b, ErrKeyNotFound
	} else {
		return b, err
	}
}

func (k *OSXKeychain) Set(service, key string, secret []byte) error {
	attributes := keychain.GenericPasswordAttributes{
		ServiceName: service,
		AccountName: key,
		Password:    secret,
	}

	if k.path != "" {
		if _, err := os.Stat(k.path); os.IsNotExist(err) {
			pass := os.Getenv("AWS_KEYCHAIN_PASSWORD")
			if pass != "" {
				keychain.CreateKeychain(k.path, pass)
			} else {
				keychain.CreateKeychainViaPrompt(k.path)
			}
		}
		attributes.Keychain = []string{k.path}
	}

	err := keychain.AddGenericPassword(&attributes)
	if err == keychain.ErrDuplicateItem {
		return keychain.RemoveAndAddGenericPassword(&attributes)
	}

	return err
}

func (k *OSXKeychain) Remove(service, key string) error {
	attributes := keychain.GenericPasswordAttributes{
		ServiceName: service,
		AccountName: key,
	}

	if k.path != "" {
		attributes.Keychain = []string{k.path}
	}

	if err := keychain.FindAndRemoveGenericPassword(&attributes); err == keychain.ErrItemNotFound {
		return ErrKeyNotFound
	} else {
		return err
	}
}

func (k *OSXKeychain) List(service string) ([]string, error) {
	keychains := []string{}

	if k.path != "" {
		keychains = []string{k.path}
	}

	return keychain.GetAllAccountNames(service, keychains...)
}

func init() {
	file := os.Getenv("AWS_KEYCHAIN_FILE")
	if file == "" {
		usr, err := user.Current()
		if err != nil {
			panic(err)
		}
		file = usr.HomeDir + "/Library/Keychains/aws-vault.keychain"
	}

	keyrings = append(keyrings, &OSXKeychain{file})
}
<commit_msg>Rewrote pretty much everything<commit_after>package keyring

import (
	"os"
	"os/user"

	keychain "github.com/99designs/go-osxkeychain"
)

var keychainFile string

type OSXKeychain struct {
	path string
}

func (k *OSXKeychain) Get(service, key string) ([]byte, error) {
	attributes := keychain.GenericPasswordAttributes{
		ServiceName: service,
		AccountName: key,
	}

	if k.path != "" {
		attributes.Keychain = []string{k.path}
	}

	if b, err := keychain.FindGenericPassword(&attributes); err == keychain.ErrItemNotFound {
		return b, ErrKeyNotFound
	} else {
		return b, err
	}
}

func (k *OSXKeychain) Set(service, key string, secret []byte) error {
	attributes := keychain.GenericPasswordAttributes{
		ServiceName: service,
		AccountName: key,
		Password:    secret,
	}

	if k.path != "" {
		if _, err := os.Stat(k.path); os.IsNotExist(err) {
			pass := os.Getenv("AWS_KEYCHAIN_PASSWORD")
			if pass != "" {
				keychain.CreateKeychain(k.path, pass)
			} else {
				keychain.CreateKeychainViaPrompt(k.path)
			}
		}
		attributes.Keychain = []string{k.path}
	}

	err := keychain.AddGenericPassword(&attributes)
	if err == keychain.ErrDuplicateItem {
		return keychain.RemoveAndAddGenericPassword(&attributes)
	}

	return err
}

func (k *OSXKeychain) Remove(service, key string) error {
	attributes := keychain.GenericPasswordAttributes{
		ServiceName: service,
		AccountName: key,
	}

	if k.path != "" {
		attributes.Keychain = []string{k.path}
	}

	if err := keychain.FindAndRemoveGenericPassword(&attributes); err == keychain.ErrItemNotFound {
		return ErrKeyNotFound
	} else {
		return err
	}
}

func (k *OSXKeychain) List(service string) ([]string, error) {
	keychains := []string{}

	if k.path != "" {
		keychains = []string{k.path}
	}

	return keychain.GetAllAccountNames(service, keychains...)
}

func init() {
	file := os.Getenv("AWS_KEYCHAIN_FILE")
	if file == "" {
		usr, err := user.Current()
		if err != nil {
			panic(err)
		}
		file = usr.HomeDir + "/Library/Keychains/aws-vault.keychain"
	}

	keyrings = append(keyrings, &OSXKeychain{file})
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/filepathfilter"
	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/git/githistory"
	"github.com/git-lfs/git-lfs/git/odb"
	"github.com/git-lfs/git-lfs/lfs"
	"github.com/git-lfs/git-lfs/tasklog"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/spf13/cobra"
)

func migrateExportCommand(cmd *cobra.Command, args []string) {
	l := tasklog.NewLogger(os.Stderr)
	defer l.Close()

	db, err := getObjectDatabase()
	if err != nil {
		ExitWithError(err)
	}
	defer db.Close()

	rewriter := getHistoryRewriter(cmd, db, l)

	filter := rewriter.Filter()
	if len(filter.Include()) <= 0 {
		ExitWithError(errors.Errorf("fatal: one or more files must be specified with --include"))
	}

	tracked := trackedFromExportFilter(filter)
	gitfilter := lfs.NewGitFilter(cfg)

	opts := &githistory.RewriteOptions{
		Verbose:           migrateVerbose,
		ObjectMapFilePath: objectMapFilePath,
		BlobFn: func(path string, b *odb.Blob) (*odb.Blob, error) {
			if filepath.Base(path) == ".gitattributes" {
				return b, nil
			}

			ptr, err := lfs.DecodePointer(b.Contents)
			if errors.IsNotAPointerError(err) {
				return b, nil
			}
			if err != nil {
				return nil, err
			}

			downloadPath, err := gitfilter.ObjectPath(ptr.Oid)
			if err != nil {
				return nil, err
			}

			return odb.NewBlobFromFile(downloadPath)
		},

		TreeCallbackFn: func(path string, t *odb.Tree) (*odb.Tree, error) {
			if path != "/" {
				// Ignore non-root trees.
				return t, nil
			}

			ours := tracked
			theirs, err := trackedFromAttrs(db, t)
			if err != nil {
				return nil, err
			}

			// Create a blob of the attributes that are optionally
			// present in the "t" tree's .gitattributes blob, and
			// union in the patterns that we've tracked.
			//
			// Perform this Union() operation each time we visit a
			// root tree such that if the underlying .gitattributes
			// is present and has a diff between commits in the
			// range of commits to migrate, those changes are
			// preserved.
			blob, err := trackedToBlob(db, theirs.Clone().Union(ours))
			if err != nil {
				return nil, err
			}

			// Finally, return a copy of the tree "t" that has the
			// new .gitattributes file included/replaced.
			return t.Merge(&odb.TreeEntry{
				Name:     ".gitattributes",
				Filemode: 0100644,
				Oid:      blob,
			}), nil
		},

		UpdateRefs: true,
	}

	requireInRepo()

	opts, err = rewriteOptions(args, opts, l)
	if err != nil {
		ExitWithError(err)
	}

	// If we have a valid remote, pre-download all objects using the Transfer Queue
	if remoteURL := getAPIClient().Endpoints.RemoteEndpoint("download", cfg.Remote()).Url; remoteURL != "" {
		q := newDownloadQueue(getTransferManifestOperationRemote("Download", cfg.Remote()), cfg.Remote())
		gs := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
			if err != nil {
				return
			}

			downloadPath, err := gitfilter.ObjectPath(p.Oid)
			if err != nil {
				return
			}

			if _, err := os.Stat(downloadPath); os.IsNotExist(err) {
				q.Add(p.Name, downloadPath, p.Oid, p.Size)
			}
		})
		gs.ScanRefs(opts.Include, opts.Exclude, nil)

		q.Wait()

		for _, err := range q.Errors() {
			if err != nil {
				ExitWithError(err)
			}
		}
	}

	// Perform the rewrite
	if _, err := rewriter.Rewrite(opts); err != nil {
		ExitWithError(err)
	}

	// Only perform `git-checkout(1) -f` if the repository is non-bare.
	if bare, _ := git.IsBare(); !bare {
		t := l.Waiter("migrate: checkout")
		err := git.Checkout("", nil, true)
		t.Complete()

		if err != nil {
			ExitWithError(err)
		}
	}
}

// trackedFromExportFilter returns an ordered set of strings where each entry
// is a line we intend to place in the .gitattributes file. It adds/removes the
// filter/diff/merge=lfs attributes based on patterns included/excluded in the
// given filter. Since `migrate export` removes files from Git LFS, it will
// remove attributes for included files, and add attributes for excluded files
func trackedFromExportFilter(filter *filepathfilter.Filter) *tools.OrderedSet {
	tracked := tools.NewOrderedSet()

	for _, include := range filter.Include() {
		tracked.Add(fmt.Sprintf("%s text !filter !merge !diff", escapeAttrPattern(include)))
	}

	for _, exclude := range filter.Exclude() {
		tracked.Add(fmt.Sprintf("%s filter=lfs diff=lfs merge=lfs -text", escapeAttrPattern(exclude)))
	}

	return tracked
}
<commit_msg>commands: only download objects matching filter<commit_after>package commands

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/filepathfilter"
	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/git/githistory"
	"github.com/git-lfs/git-lfs/git/odb"
	"github.com/git-lfs/git-lfs/lfs"
	"github.com/git-lfs/git-lfs/tasklog"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/spf13/cobra"
)

func migrateExportCommand(cmd *cobra.Command, args []string) {
	l := tasklog.NewLogger(os.Stderr)
	defer l.Close()

	db, err := getObjectDatabase()
	if err != nil {
		ExitWithError(err)
	}
	defer db.Close()

	rewriter := getHistoryRewriter(cmd, db, l)

	filter := rewriter.Filter()
	if len(filter.Include()) <= 0 {
		ExitWithError(errors.Errorf("fatal: one or more files must be specified with --include"))
	}

	tracked := trackedFromExportFilter(filter)
	gitfilter := lfs.NewGitFilter(cfg)

	opts := &githistory.RewriteOptions{
		Verbose:           migrateVerbose,
		ObjectMapFilePath: objectMapFilePath,
		BlobFn: func(path string, b *odb.Blob) (*odb.Blob, error) {
			if filepath.Base(path) == ".gitattributes" {
				return b, nil
			}

			ptr, err := lfs.DecodePointer(b.Contents)
			if errors.IsNotAPointerError(err) {
				return b, nil
			}
			if err != nil {
				return nil, err
			}

			downloadPath, err := gitfilter.ObjectPath(ptr.Oid)
			if err != nil {
				return nil, err
			}

			return odb.NewBlobFromFile(downloadPath)
		},

		TreeCallbackFn: func(path string, t *odb.Tree) (*odb.Tree, error) {
			if path != "/" {
				// Ignore non-root trees.
				return t, nil
			}

			ours := tracked
			theirs, err := trackedFromAttrs(db, t)
			if err != nil {
				return nil, err
			}

			// Create a blob of the attributes that are optionally
			// present in the "t" tree's .gitattributes blob, and
			// union in the patterns that we've tracked.
			//
			// Perform this Union() operation each time we visit a
			// root tree such that if the underlying .gitattributes
			// is present and has a diff between commits in the
			// range of commits to migrate, those changes are
			// preserved.
			blob, err := trackedToBlob(db, theirs.Clone().Union(ours))
			if err != nil {
				return nil, err
			}

			// Finally, return a copy of the tree "t" that has the
			// new .gitattributes file included/replaced.
			return t.Merge(&odb.TreeEntry{
				Name:     ".gitattributes",
				Filemode: 0100644,
				Oid:      blob,
			}), nil
		},

		UpdateRefs: true,
	}

	requireInRepo()

	opts, err = rewriteOptions(args, opts, l)
	if err != nil {
		ExitWithError(err)
	}

	// If we have a valid remote, pre-download all objects using the Transfer Queue
	if remoteURL := getAPIClient().Endpoints.RemoteEndpoint("download", cfg.Remote()).Url; remoteURL != "" {
		q := newDownloadQueue(getTransferManifestOperationRemote("Download", cfg.Remote()), cfg.Remote())
		gs := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
			if err != nil {
				return
			}

			if !filter.Allows(p.Name) {
				return
			}

			downloadPath, err := gitfilter.ObjectPath(p.Oid)
			if err != nil {
				return
			}

			if _, err := os.Stat(downloadPath); os.IsNotExist(err) {
				q.Add(p.Name, downloadPath, p.Oid, p.Size)
			}
		})
		gs.ScanRefs(opts.Include, opts.Exclude, nil)

		q.Wait()

		for _, err := range q.Errors() {
			if err != nil {
				ExitWithError(err)
			}
		}
	}

	// Perform the rewrite
	if _, err := rewriter.Rewrite(opts); err != nil {
		ExitWithError(err)
	}

	// Only perform `git-checkout(1) -f` if the repository is non-bare.
	if bare, _ := git.IsBare(); !bare {
		t := l.Waiter("migrate: checkout")
		err := git.Checkout("", nil, true)
		t.Complete()

		if err != nil {
			ExitWithError(err)
		}
	}
}

// trackedFromExportFilter returns an ordered set of strings where each entry
// is a line we intend to place in the .gitattributes file. It adds/removes the
// filter/diff/merge=lfs attributes based on patterns included/excluded in the
// given filter. Since `migrate export` removes files from Git LFS, it will
// remove attributes for included files, and add attributes for excluded files
func trackedFromExportFilter(filter *filepathfilter.Filter) *tools.OrderedSet {
	tracked := tools.NewOrderedSet()

	for _, include := range filter.Include() {
		tracked.Add(fmt.Sprintf("%s text !filter !merge !diff", escapeAttrPattern(include)))
	}

	for _, exclude := range filter.Exclude() {
		tracked.Add(fmt.Sprintf("%s filter=lfs diff=lfs merge=lfs -text", escapeAttrPattern(exclude)))
	}

	return tracked
}
<|endoftext|>


<commit_before>/*

Encrypts and decripts vigenere encoded strings

*/
package main

import (
	"bufio"
	"encoding/hex"
	"fmt"
	"os"
	"strings"
)

func main() {
	fmt.Println("I'm going to crack one time pad")
	byteText := readFile("cipheredTextTimePad.txt")
	fmt.Println(byteText)
	var i byte
	for i = 1; i < 255; i++ {
		for column := 0; column < len(byteText[1]); column++ {
			if checkTransform(byteText, column, i) == true {
				fmt.Printf("\nByte %d makes sense for column %d ", i, column)
			} else {
				//fmt.Printf("Bad %d %d", i, column)
			}
		}
	}
}

func checkTransform(byteText [][]byte, column int, index byte) bool {
	byteColumn := make([]byte, len(byteText))
	for i := 0; i < len(byteText); i++ {
		byteColumn[i] = byteText[i][column] ^ index
	}

	result := hasGoodChars(byteColumn)
	if result == true {
		fmt.Print(string(byteColumn))
	}
	return result
}

func hasGoodChars(text []byte) bool {
	for i := 0; i < len(text); i++ {
		if text[i] < 32 || text[i] > 122 {
			return false
		}
	}

	//Check counters
	return true
}

func readFile(fileName string) [][]byte {
	byteText := make([][]byte, 100)
	var byteLine []byte
	f, err := os.Open(fileName)
	check(err)
	defer f.Close()
	reader := bufio.NewReader(f)
	// Get all the lines converted to byte arrat
	line, err := reader.ReadString('\n')
	i := 0
	for err == nil {
		byteLine, _ = hex.DecodeString(strings.Trim(line, "\n "))
		fmt.Print(byteLine)
		byteText[i] = byteLine
		line, err = reader.ReadString('\n') // 0x0A separator = newline
		i++
	}
	byteLine, _ = hex.DecodeString(line)
	byteText[i] = byteLine

	byteText = byteText[:i+1]

	return byteText
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}
<commit_msg>Done with onetimepad<commit_after>/*

Encrypts and decripts vigenere encoded strings

*/
package main

import (
	"bufio"
	"encoding/hex"
	"fmt"
	"os"
	"regexp"
	"strings"
)

func main() {
	fmt.Println("I'm going to crack one time pad")
	byteText := readFile("cipheredTextTimePad.txt")
	fmt.Println(byteText)
	var i byte
	solution := []byte{242, 26, 4, 155, 208, 115, 35, 200, 57, 152,
		206, 9, 14, 188, 134, 218, 201, 224, 57, 137,
		42, 95, 114, 103, 131, 165, 97, 253, 37, 238,
		48}

	checkSolution(byteText, solution)
	byteText = readFile("cipheredTextTimePad.txt")
	// Look for spaces
	//findSpaceColumns(byteText)

	// Look for remaining
	for i = 1; i < 255; i++ {
		for column := 0; column < len(byteText[1]); column++ {
			if checkTransform(byteText, column, i) == true {
				fmt.Printf(" byte %d makes sense for column %d \n", i, column)
			} else {
				//fmt.Printf("Bad %d %d", i, column)
			}
		}
	}
}

func checkSolution(text [][]byte, solution []byte) {

	for col := 0; col < len(solution); col++ {
		for row := 0; row < 7; row++ {
			text[row][col] = text[row][col] ^ solution[col]
		}
	}
	for row := 0; row < 7; row++ {
		fmt.Println(string(text[row]))
	}
}

func checkTransform(byteText [][]byte, column int, index byte) bool {
	byteColumn := make([]byte, len(byteText))
	for i := 0; i < len(byteText); i++ {
		byteColumn[i] = byteText[i][column] ^ index
	}

	result := hasGoodChars(byteColumn)
	if result == true {
		fmt.Print(string(byteColumn))
	}
	return result
}

func hasGoodChars(text []byte) bool {
	result := true
	//alphaChars := 0
	re := regexp.MustCompile(`[[:lower:]]|[[:blank:]]|,|\.|;|\!|\?`)
	for i := 0; i < len(text); i++ {
		if text[i] < 32 || text[i] > 122 {
			result = false
		}
	}

	if result == true {
		//fmt.Print("\n Found strings")
		//fmt.Print(re.FindAllStringIndex(string(text), -1))
		//fmt.Printf(" - %s\n", string(text))
		if len(re.FindAllStringIndex(string(text), -1)) < 7 {
			result = false
		}

	}
	return result
}

func readFile(fileName string) [][]byte {
	byteText := make([][]byte, 100)
	var byteLine []byte
	f, err := os.Open(fileName)
	check(err)
	defer f.Close()
	reader := bufio.NewReader(f)
	// Get all the lines converted to byte arrat
	line, err := reader.ReadString('\n')
	i := 0
	for err == nil {
		byteLine, _ = hex.DecodeString(strings.Trim(line, "\n "))
		fmt.Print(byteLine)
		byteText[i] = byteLine
		line, err = reader.ReadString('\n') // 0x0A separator = newline
		i++
	}
	byteLine, _ = hex.DecodeString(line)
	byteText[i] = byteLine

	byteText = byteText[:i+1]

	return byteText
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}
<|endoftext|>


<commit_before>package mpaccesslog

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/Songmu/axslogparser"
	"github.com/Songmu/postailer"
	mp "github.com/mackerelio/go-mackerel-plugin"
	"github.com/mackerelio/golib/pluginutil"
	"github.com/montanaflynn/stats"
)

// AccesslogPlugin mackerel plugin
type AccesslogPlugin struct {
	prefix  string
	file    string
	posFile string
}

// MetricKeyPrefix interface for PluginWithPrefix
func (p *AccesslogPlugin) MetricKeyPrefix() string {
	if p.prefix == "" {
		p.prefix = "accesslog"
	}
	return p.prefix
}

// GraphDefinition interface for mackerelplugin
func (p *AccesslogPlugin) GraphDefinition() map[string]mp.Graphs {
	labelPrefix := strings.Title(p.prefix)
	return map[string]mp.Graphs{
		"access_num": {
			Label: labelPrefix + " Access Num",
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "total_count", Label: "Total Count"},
				{Name: "5xx_count", Label: "HTTP 5xx Count", Stacked: true},
				{Name: "4xx_count", Label: "HTTP 4xx Count", Stacked: true},
				{Name: "3xx_count", Label: "HTTP 3xx Count", Stacked: true},
				{Name: "2xx_count", Label: "HTTP 2xx Count", Stacked: true},
			},
		},
		"access_rate": {
			Label: labelPrefix + " Access Rate",
			Unit:  "percentage",
			Metrics: []mp.Metrics{
				{Name: "5xx_percentage", Label: "HTTP 5xx Percentage", Stacked: true},
				{Name: "4xx_percentage", Label: "HTTP 4xx Percentage", Stacked: true},
				{Name: "3xx_percentage", Label: "HTTP 3xx Percentage", Stacked: true},
				{Name: "2xx_percentage", Label: "HTTP 2xx Percentage", Stacked: true},
			},
		},
		"latency": {
			Label: labelPrefix + " Latency",
			Unit:  "float",
			Metrics: []mp.Metrics{
				{Name: "99_percentile", Label: "99 Percentile"},
				{Name: "95_percentile", Label: "95 Percentile"},
				{Name: "90_percentile", Label: "90 Percentile"},
				{Name: "average", Label: "Average"},
			},
		},
	}
}

func (p *AccesslogPlugin) getPos() string {
	base := p.file + ".pos.json"
	if p.posFile != "" {
		if filepath.IsAbs(p.posFile) {
			return p.posFile
		}
		base = p.posFile
	}
	return filepath.Join(pluginutil.PluginWorkDir(), "mackerel-plugin-accesslog.d", base)
}

// FetchMetrics interface for mackerelplugin
func (p *AccesslogPlugin) FetchMetrics() (map[string]float64, error) {
	posfile := p.getPos()
	fi, err := os.Stat(posfile)
	// don't output count metrics when the pos file doesn't exist or is too old
	takeCount := err == nil && fi.ModTime().After(time.Now().Add(-2*time.Minute))
	pt, err := postailer.Open(p.file, posfile)
	if err != nil {
		return nil, err
	}
	defer pt.Close()

	countMetrics := []string{"total_count", "2xx_count", "3xx_count", "4xx_count", "5xx_count"}
	ret := make(map[string]float64)
	for _, k := range countMetrics {
		ret[k] = 0
	}
	var reqtimes []float64
	var psr axslogparser.Parser
	s := bufio.NewScanner(pt)
	for s.Scan() {
		var (
			l   *axslogparser.Log
			err error
		)
		line := s.Text()
		if psr == nil {
			psr, l, err = axslogparser.GuessParser(line)
		} else {
			l, err = psr.Parse(line)
		}
		if err != nil {
			log.Println(err)
			continue
		}
		ret[string(fmt.Sprintf("%d", l.Status)[0])+"xx_count"]++
		ret["total_count"]++

		if l.ReqTime != nil {
			reqtimes = append(reqtimes, *l.ReqTime)
		} else if l.TakenSec != nil {
			reqtimes = append(reqtimes, *l.TakenSec)
		}
	}
	if s.Err() != nil {
		log.Println(s.Err())
	}
	if ret["total_count"] > 0 {
		for _, v := range []string{"2xx", "3xx", "4xx", "5xx"} {
			ret[v+"_percentage"] = ret[v+"_count"] * 100 / ret["total_count"]
		}
	}
	if len(reqtimes) > 0 {
		ret["average"], _ = stats.Mean(reqtimes)
		for _, v := range []int{90, 95, 99} {
			ret[fmt.Sprintf("%d", v)+"_percentile"], _ = stats.Percentile(reqtimes, float64(v))
		}
	}
	if !takeCount {
		for _, k := range countMetrics {
			delete(ret, k)
		}
	}
	return ret, nil
}

// Do the plugin
func Do() {
	var (
		optPrefix  = flag.String("metric-key-prefix", "", "Metric key prefix")
		optPosFile = flag.String("posfile", "", "(not necessary to specify it in the usual use case) posfile")
	)
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [OPTION] /path/to/access.log\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()
	if flag.NArg() < 1 {
		flag.Usage()
		os.Exit(1)
	}
	mp.NewMackerelPlugin(&AccesslogPlugin{
		prefix:  *optPrefix,
		file:    flag.Args()[0],
		posFile: *optPosFile,
	}).Run()
}
<commit_msg>add -no-posfile feature<commit_after>package mpaccesslog

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/Songmu/axslogparser"
	"github.com/Songmu/postailer"
	mp "github.com/mackerelio/go-mackerel-plugin"
	"github.com/mackerelio/golib/pluginutil"
	"github.com/montanaflynn/stats"
)

// AccesslogPlugin mackerel plugin
type AccesslogPlugin struct {
	prefix    string
	file      string
	posFile   string
	noPosFile bool
}

// MetricKeyPrefix interface for PluginWithPrefix
func (p *AccesslogPlugin) MetricKeyPrefix() string {
	if p.prefix == "" {
		p.prefix = "accesslog"
	}
	return p.prefix
}

// GraphDefinition interface for mackerelplugin
func (p *AccesslogPlugin) GraphDefinition() map[string]mp.Graphs {
	labelPrefix := strings.Title(p.prefix)
	return map[string]mp.Graphs{
		"access_num": {
			Label: labelPrefix + " Access Num",
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "total_count", Label: "Total Count"},
				{Name: "5xx_count", Label: "HTTP 5xx Count", Stacked: true},
				{Name: "4xx_count", Label: "HTTP 4xx Count", Stacked: true},
				{Name: "3xx_count", Label: "HTTP 3xx Count", Stacked: true},
				{Name: "2xx_count", Label: "HTTP 2xx Count", Stacked: true},
			},
		},
		"access_rate": {
			Label: labelPrefix + " Access Rate",
			Unit:  "percentage",
			Metrics: []mp.Metrics{
				{Name: "5xx_percentage", Label: "HTTP 5xx Percentage", Stacked: true},
				{Name: "4xx_percentage", Label: "HTTP 4xx Percentage", Stacked: true},
				{Name: "3xx_percentage", Label: "HTTP 3xx Percentage", Stacked: true},
				{Name: "2xx_percentage", Label: "HTTP 2xx Percentage", Stacked: true},
			},
		},
		"latency": {
			Label: labelPrefix + " Latency",
			Unit:  "float",
			Metrics: []mp.Metrics{
				{Name: "99_percentile", Label: "99 Percentile"},
				{Name: "95_percentile", Label: "95 Percentile"},
				{Name: "90_percentile", Label: "90 Percentile"},
				{Name: "average", Label: "Average"},
			},
		},
	}
}

func (p *AccesslogPlugin) getPos() string {
	base := p.file + ".pos.json"
	if p.posFile != "" {
		if filepath.IsAbs(p.posFile) {
			return p.posFile
		}
		base = p.posFile
	}
	return filepath.Join(pluginutil.PluginWorkDir(), "mackerel-plugin-accesslog.d", base)
}

func (p *AccesslogPlugin) getReadCloser() (io.ReadCloser, bool, error) {
	if p.noPosFile {
		rc, err := os.Open(p.file)
		return rc, true, err
	}
	posfile := p.getPos()
	fi, err := os.Stat(posfile)
	// don't output count metrics when the pos file doesn't exist or is too old
	takeCount := err == nil && fi.ModTime().After(time.Now().Add(-2*time.Minute))
	rc, err := postailer.Open(p.file, posfile)
	return rc, takeCount, err
}

// FetchMetrics interface for mackerelplugin
func (p *AccesslogPlugin) FetchMetrics() (map[string]float64, error) {
	rc, takeCount, err := p.getReadCloser()
	if err != nil {
		return nil, err
	}
	defer rc.Close()

	countMetrics := []string{"total_count", "2xx_count", "3xx_count", "4xx_count", "5xx_count"}
	ret := make(map[string]float64)
	for _, k := range countMetrics {
		ret[k] = 0
	}
	var reqtimes []float64
	var psr axslogparser.Parser
	s := bufio.NewScanner(rc)
	for s.Scan() {
		var (
			l   *axslogparser.Log
			err error
		)
		line := s.Text()
		if psr == nil {
			psr, l, err = axslogparser.GuessParser(line)
		} else {
			l, err = psr.Parse(line)
		}
		if err != nil {
			log.Println(err)
			continue
		}
		ret[string(fmt.Sprintf("%d", l.Status)[0])+"xx_count"]++
		ret["total_count"]++

		if l.ReqTime != nil {
			reqtimes = append(reqtimes, *l.ReqTime)
		} else if l.TakenSec != nil {
			reqtimes = append(reqtimes, *l.TakenSec)
		}
	}
	if s.Err() != nil {
		log.Println(s.Err())
	}
	if ret["total_count"] > 0 {
		for _, v := range []string{"2xx", "3xx", "4xx", "5xx"} {
			ret[v+"_percentage"] = ret[v+"_count"] * 100 / ret["total_count"]
		}
	}
	if len(reqtimes) > 0 {
		ret["average"], _ = stats.Mean(reqtimes)
		for _, v := range []int{90, 95, 99} {
			ret[fmt.Sprintf("%d", v)+"_percentile"], _ = stats.Percentile(reqtimes, float64(v))
		}
	}
	if !takeCount {
		for _, k := range countMetrics {
			delete(ret, k)
		}
	}
	return ret, nil
}

// Do the plugin
func Do() {
	var (
		optPrefix    = flag.String("metric-key-prefix", "", "Metric key prefix")
		optPosFile   = flag.String("posfile", "", "(not necessary to specify it in the usual use case) posfile")
		optNoPosFile = flag.Bool("no-posfile", false, "no position file")
	)
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [OPTION] /path/to/access.log\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()
	if flag.NArg() < 1 {
		flag.Usage()
		os.Exit(1)
	}
	mp.NewMackerelPlugin(&AccesslogPlugin{
		prefix:    *optPrefix,
		file:      flag.Args()[0],
		posFile:   *optPosFile,
		noPosFile: *optNoPosFile,
	}).Run()
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

// http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This file was initially generated by gen_to_start.go (add link), as a start
// of the Traffic Ops golang data model

package main

import (
	"database/sql"
	"fmt"
	"os"
	"regexp"
	"strings"
)

func checkErr(err error, txn *sql.Tx) {
	if err != nil {
		fmt.Println("Error 33:", err)
		fmt.Println("Attempting Roll back!")
		txn.Rollback()
		os.Exit(1)
	}
}

func doExec(stmt string, txn *sql.Tx) {
	fmt.Println("  " + stmt)
	_, err := txn.Exec(stmt)
	if err != nil {
		fmt.Println("Error:", err)
		fmt.Println("Attempting Roll back!")
		txn.Rollback()
		os.Exit(1)
	}
}

// Up is executed when this migration is applied
func Up_20170205101432(txn *sql.Tx) {

	algorithmStr := map[int64]string{
		0: "consistent_hash",
		1: "false",
		2: "strict",
		3: "true",
		4: "latched",
	}

	fmt.Println("  Starting migration 20170205101432.go...")
	doExec("CREATE TYPE profile_type AS ENUM ("+
		"'ATS_PROFILE', 'TR_PROFILE', 'TM_PROFILE', 'TS_PROFILE', 'TP_PROFILE', 'INFLUXDB_PROFILE',"+
		"'RIAK_PROFILE', 'SPLUNK_PROFILE', 'DS_PROFILE', 'ORG_PROFILE', 'KAFKA_PROFILE', 'LOGSTASH_PROFILE',"+
		"'ES_PROFILE', 'UNK_PROFILE')", txn)

	doExec("CREATE OR REPLACE VIEW \"profile_type_values\" AS SELECT unnest(enum_range(NULL::profile_type )) AS value ORDER BY value", txn)

	doExec("ALTER TABLE profile ADD COLUMN type profile_type", txn)
	doExec("UPDATE profile SET type='UNK_PROFILE'", txn) // So we don't get any NULL, these should be checked.
	doExec("UPDATE profile SET type='TR_PROFILE' WHERE name like 'CCR_%' OR name like 'TR_%'", txn)
	doExec("UPDATE profile SET type='TM_PROFILE' WHERE name like 'RASCAL_%' OR name like 'TM_%'", txn)
	doExec("UPDATE profile SET type='TS_PROFILE' WHERE name like 'TRAFFIC_STATS%'", txn)
	doExec("UPDATE profile SET type='TP_PROFILE' WHERE name like 'TRAFFIC_PORTAL%'", txn)
	doExec("UPDATE profile SET type='INFLUXDB_PROFILE' WHERE name like 'INFLUXDB%'", txn)
	doExec("UPDATE profile SET type='RIAK_PROFILE' WHERE name like 'RIAK%'", txn)
	doExec("UPDATE profile SET type='SPLUNK_PROFILE' WHERE name like 'SPLUNK%'", txn)
	doExec("UPDATE profile SET type='ORG_PROFILE' WHERE name like '%ORG%' or name like 'MSO%' or name like '%ORIGIN%'", txn)
	doExec("UPDATE profile SET type='KAFKA_PROFILE' WHERE name like 'KAFKA%'", txn)
	doExec("UPDATE profile SET type='LOGSTASH_PROFILE' WHERE name like 'LOGSTASH_%'", txn)
	doExec("UPDATE profile SET type='ES_PROFILE' WHERE name like 'ELASTICSEARCH%'", txn)
	doExec("UPDATE profile SET type='ATS_PROFILE' WHERE name like 'EDGE%' or name like 'MID%'", txn)

	doExec("ALTER TABLE profile ALTER type SET NOT NULL", txn)

	doExec("ALTER TABLE cdn ADD COLUMN domain_name text", txn)

	doExec("UPDATE cdn SET domain_name=domainlist.value "+
		"FROM (SELECT distinct cdn_id,value FROM server,parameter WHERE type=(SELECT id FROM type WHERE name='EDGE') "+
		"AND parameter.id in (select parameter from profile_parameter WHERE profile_parameter.profile=server.profile) "+
		"AND parameter.name='domain_name' "+
		"AND config_file='CRConfig.json') AS domainlist "+
		"WHERE id = domainlist.cdn_id", txn)

	doExec("UPDATE cdn SET domain_name='-' WHERE name='ALL'", txn)

	doExec("ALTER TABLE cdn ALTER COLUMN domain_name SET NOT NULL", txn)

	doExec("ALTER TABLE profile ADD COLUMN cdn bigint", txn)

	doExec("ALTER TABLE profile "+
		"ADD CONSTRAINT fk_cdn1 FOREIGN KEY (cdn) "+
		"REFERENCES cdn (id) MATCH SIMPLE "+
		"ON UPDATE RESTRICT ON DELETE RESTRICT", txn)

	doExec("CREATE INDEX idx_181818_fk_cdn1 "+
		"ON profile "+
		"USING btree "+
		"(cdn)", txn)

	doExec("UPDATE profile set cdn=domainlist.cdn_id "+
		"FROM (SELECT distinct profile.id AS profile_id, value AS profile_domain_name, cdn.id cdn_id "+
		"FROM profile, parameter, cdn, profile_parameter "+
		"WHERE parameter.name='domain_name' "+
		"AND parameter.config_file='CRConfig.json' "+
		"AND parameter.value = cdn.domain_name "+
		"AND parameter.id in (select parameter from profile_parameter where profile=profile.id)) as domainlist "+
		"WHERE id = domainlist.profile_id", txn)

	doExec("ALTER TABLE deliveryservice ALTER profile DROP NOT NULL", txn)

	doExec("UPDATE deliveryservice SET profile=NULL", txn)

	doExec("INSERT INTO parameter (name, config_file, value) VALUES ('mso.parent_retry', 'parent.config', 'simple_retry');", txn)
	doExec("INSERT INTO parameter (name, config_file, value) VALUES ('mso.parent_retry', 'parent.config', 'unavailable_server_retry');", txn)
	doExec("INSERT INTO parameter (name, config_file, value) VALUES ('mso.parent_retry', 'parent.config', 'both');", txn)

	type Profile struct {
		Id                 int64
		Name               string
		Desc               string
		Type               string
		Cdn                int64
		MidHeaderRewrite   string
		MultiSiteOriginAlg int64
		XMLId              string
	}
	// move data
	pmap := make(map[string]Profile)
	rows, err := txn.Query("SELECT id,xml_id,mid_header_rewrite,multi_site_origin,multi_site_origin_algorithm,cdn_id FROM deliveryservice WHERE multi_site_origin=true")
	if err != nil {
		fmt.Println("Error:", err)
		fmt.Println("Attempting Roll back!")
		txn.Rollback()
		os.Exit(1)
	}
	existingParam := make(map[string]int64)
	for rows.Next() {
		var (
			id                          int64
			xml_id                      string
			mid_header_rewrite          sql.NullString
			multi_site_origin           sql.NullBool
			multi_site_origin_algorithm sql.NullInt64
			cdn_id                      int64
		)
		err := rows.Scan(&id, &xml_id, &mid_header_rewrite, &multi_site_origin, &multi_site_origin_algorithm, &cdn_id)
		checkErr(err, txn)

		pName := "DS_" + xml_id
		pDesc := "Deliveryservice profile for " + xml_id
		pType := "DS_PROFILE"
		mhrString := mid_header_rewrite.String
		pmap[pName] = Profile{
			Id:                 -1,
			Name:               pName,
			Desc:               pDesc,
			Type:               pType,
			Cdn:                cdn_id,
			MidHeaderRewrite:   mhrString,
			MultiSiteOriginAlg: multi_site_origin_algorithm.Int64,
			XMLId:              xml_id,
		}
	}
	err = rows.Err()
	checkErr(err, txn)
	rows.Close()

	for _, prof := range pmap {
		fmt.Println("--\nINSERT INTO PROFILE (name, description, type, cdn) VALUES($1, $2, $3, $4) RETURNING id", prof.Name, prof.Desc, prof.Type, prof.Cdn)
		newRow := txn.QueryRow("INSERT INTO PROFILE (name, description, type, cdn) VALUES($1, $2, $3, $4) RETURNING id", prof.Name, prof.Desc, prof.Type, prof.Cdn)
		var newProfileId int64
		err := newRow.Scan(&newProfileId)
		checkErr(err, txn)

		remainingString := ""
		parent_retry := ""
		var regexpOne = regexp.MustCompile(`^\s*set-config\s+proxy.config.http.parent_origin.(\S+)\s+(.*)$`)
		if prof.MidHeaderRewrite != "" {
			remapParts := strings.Split(prof.MidHeaderRewrite, "__RETURN__")
			for _, line := range remapParts {
				fmt.Println(line)
				match := regexpOne.FindSubmatch([]byte(line))
				if len(match) != 0 {
					var newId int64
					var ok bool
					newId, ok = existingParam[string(match[1])+string(match[2])]
					//	fmt.Printf("%s -> %v %v\n", string(match[1])+string(match[2]), newId, ok)
					if !ok {
						if string(match[1]) == "simple_retry_enabled" || string(match[1]) == "dead_server_retry_enabled" {
							if parent_retry == "" {
								parent_retry = string(match[1])
							} else {
								parent_retry = "both"
							}
						} else if string(match[1]) == "dead_server_retry_response_codes" {
							// unavailable_server_retry_responses is the only one that survives?
							//	if string(match[1]) == "dead_server_retry_response_codes" {
							pName := strings.Replace("mso."+string(match[1]), "dead", "unavailable", 1) // "mso.unavailable_server_retry_response_codes"
							pName = strings.Replace(pName, "response_codes", "responses", 1) // "mso.unavailable_server_retry_responses"
							fmt.Println("INSERT INTO PARAMETER (name, config_file, value) VALUES ($1, $2, $3) RETURNING id",
								pName, "parent.config", string(match[2]))
							newRow := txn.QueryRow("INSERT INTO PARAMETER (name, config_file, value) VALUES ($1, $2, $3) RETURNING id",
								pName, "parent.config", string(match[2]))
							err := newRow.Scan(&newId)
							checkErr(err, txn)
							existingParam[string(match[1])+string(match[2])] = newId
						}
					} else {
						newId = existingParam[string(match[1])+string(match[2])]
					}
					if newId != 0 {
						fmt.Println("INSERT INTO PROFILE_PARAMETER (parameter, profile) VALUES ($1, $2)", newId, newProfileId)
						_, err = txn.Exec("INSERT INTO PROFILE_PARAMETER (parameter, profile) VALUES ($1, $2)", newId, newProfileId)
						checkErr(err, txn)
					}
				} else {
					if !strings.HasSuffix(line, "__RETURN__") {
						remainingString = remainingString + line + "__RETURN__"
					}
				}
			}
		}
		/* Not doing the removal of the header_rewrite stuff  at this time, so we can support both 5.x and 6.x - 6.x will have a warning if these are
		   not removed from the DS, but should run just fine.

		fmt.Printf("MHRW was: %s, \nMHRW now: %s\n", prof.MidHeaderRewrite, remainingString)

		_, err = txn.Exec("UPDATE deliveryservice set mid_header_rewrite=$1 where xml_id=$2", remainingString, prof.XMLId)
		checkErr(err, txn)

		*/

		var newId int64
		var ok bool
		newId, ok = existingParam["mso.algorithm"+"parent.config"+algorithmStr[prof.MultiSiteOriginAlg]]
		if !ok {
			fmt.Println("INSERT INTO PARAMETER (name, config_file, value) VALUES ($1, $2, $3) RETURNING id",
				"mso.algorithm", "parent.config", algorithmStr[prof.MultiSiteOriginAlg])
			newRow = txn.QueryRow("INSERT INTO PARAMETER (name, config_file, value) VALUES ($1, $2, $3) RETURNING id",
				"mso.algorithm", "parent.config", algorithmStr[prof.MultiSiteOriginAlg])
			err = newRow.Scan(&newId)
			checkErr(err, txn)
			existingParam["mso.algorithm"+"parent.config"+algorithmStr[prof.MultiSiteOriginAlg]] = newId
		} else {
			newId = existingParam["mso.algorithm"+"parent.config"+algorithmStr[prof.MultiSiteOriginAlg]]
		}

		fmt.Println("INSERT INTO PROFILE_PARAMETER (parameter, profile) VALUES ($1, $2)", newId, newProfileId)
		_, err = txn.Exec("INSERT INTO PROFILE_PARAMETER (parameter, profile) VALUES ($1, $2)", newId, newProfileId)
		checkErr(err, txn)

		fmt.Println("INSERT INTO profile_parameter (profile, parameter) SELECT $1, id FROM parameter "+
			"WHERE config_file='parent.config' AND name='mso.parent_retry' AND value=$2", newProfileId, parent_retry)
		_, err = txn.Exec("INSERT INTO profile_parameter (profile, parameter) SELECT $1, id FROM parameter "+
			"WHERE config_file='parent.config' AND name='mso.parent_retry' AND value=$2", newProfileId, parent_retry)
		checkErr(err, txn)

		_, err = txn.Exec("UPDATE deliveryservice SET profile=$1 WHERE xml_id=$2", newProfileId, prof.XMLId)
		checkErr(err, txn)
	}
}

// Down is executed when this migration is rolled back
func Down_20170205101432(txn *sql.Tx) {
	//
}
<commit_msg>Removed ".go" typo<commit_after>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

// http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This file was initially generated by gen_to_start.go (add link), as a start
// of the Traffic Ops golang data model

package main

import (
	"database/sql"
	"fmt"
	"os"
	"regexp"
	"strings"
)

func checkErr(err error, txn *sql.Tx) {
	if err != nil {
		fmt.Println("Error 33:", err)
		fmt.Println("Attempting Roll back!")
		txn.Rollback()
		os.Exit(1)
	}
}

func doExec(stmt string, txn *sql.Tx) {
	fmt.Println("  " + stmt)
	_, err := txn.Exec(stmt)
	if err != nil {
		fmt.Println("Error:", err)
		fmt.Println("Attempting Roll back!")
		txn.Rollback()
		os.Exit(1)
	}
}

// Up is executed when this migration is applied
func Up_20170205101432(txn *sql.Tx) {

	algorithmStr := map[int64]string{
		0: "consistent_hash",
		1: "false",
		2: "strict",
		3: "true",
		4: "latched",
	}

	fmt.Println("  Starting migration 20170205101432...")
	doExec("CREATE TYPE profile_type AS ENUM ("+
		"'ATS_PROFILE', 'TR_PROFILE', 'TM_PROFILE', 'TS_PROFILE', 'TP_PROFILE', 'INFLUXDB_PROFILE',"+
		"'RIAK_PROFILE', 'SPLUNK_PROFILE', 'DS_PROFILE', 'ORG_PROFILE', 'KAFKA_PROFILE', 'LOGSTASH_PROFILE',"+
		"'ES_PROFILE', 'UNK_PROFILE')", txn)

	doExec("CREATE OR REPLACE VIEW \"profile_type_values\" AS SELECT unnest(enum_range(NULL::profile_type )) AS value ORDER BY value", txn)

	doExec("ALTER TABLE profile ADD COLUMN type profile_type", txn)
	doExec("UPDATE profile SET type='UNK_PROFILE'", txn) // So we don't get any NULL, these should be checked.
	doExec("UPDATE profile SET type='TR_PROFILE' WHERE name like 'CCR_%' OR name like 'TR_%'", txn)
	doExec("UPDATE profile SET type='TM_PROFILE' WHERE name like 'RASCAL_%' OR name like 'TM_%'", txn)
	doExec("UPDATE profile SET type='TS_PROFILE' WHERE name like 'TRAFFIC_STATS%'", txn)
	doExec("UPDATE profile SET type='TP_PROFILE' WHERE name like 'TRAFFIC_PORTAL%'", txn)
	doExec("UPDATE profile SET type='INFLUXDB_PROFILE' WHERE name like 'INFLUXDB%'", txn)
	doExec("UPDATE profile SET type='RIAK_PROFILE' WHERE name like 'RIAK%'", txn)
	doExec("UPDATE profile SET type='SPLUNK_PROFILE' WHERE name like 'SPLUNK%'", txn)
	doExec("UPDATE profile SET type='ORG_PROFILE' WHERE name like '%ORG%' or name like 'MSO%' or name like '%ORIGIN%'", txn)
	doExec("UPDATE profile SET type='KAFKA_PROFILE' WHERE name like 'KAFKA%'", txn)
	doExec("UPDATE profile SET type='LOGSTASH_PROFILE' WHERE name like 'LOGSTASH_%'", txn)
	doExec("UPDATE profile SET type='ES_PROFILE' WHERE name like 'ELASTICSEARCH%'", txn)
	doExec("UPDATE profile SET type='ATS_PROFILE' WHERE name like 'EDGE%' or name like 'MID%'", txn)

	doExec("ALTER TABLE profile ALTER type SET NOT NULL", txn)

	doExec("ALTER TABLE cdn ADD COLUMN domain_name text", txn)

	doExec("UPDATE cdn SET domain_name=domainlist.value "+
		"FROM (SELECT distinct cdn_id,value FROM server,parameter WHERE type=(SELECT id FROM type WHERE name='EDGE') "+
		"AND parameter.id in (select parameter from profile_parameter WHERE profile_parameter.profile=server.profile) "+
		"AND parameter.name='domain_name' "+
		"AND config_file='CRConfig.json') AS domainlist "+
		"WHERE id = domainlist.cdn_id", txn)

	doExec("UPDATE cdn SET domain_name='-' WHERE name='ALL'", txn)

	doExec("ALTER TABLE cdn ALTER COLUMN domain_name SET NOT NULL", txn)

	doExec("ALTER TABLE profile ADD COLUMN cdn bigint", txn)

	doExec("ALTER TABLE profile "+
		"ADD CONSTRAINT fk_cdn1 FOREIGN KEY (cdn) "+
		"REFERENCES cdn (id) MATCH SIMPLE "+
		"ON UPDATE RESTRICT ON DELETE RESTRICT", txn)

	doExec("CREATE INDEX idx_181818_fk_cdn1 "+
		"ON profile "+
		"USING btree "+
		"(cdn)", txn)

	doExec("UPDATE profile set cdn=domainlist.cdn_id "+
		"FROM (SELECT distinct profile.id AS profile_id, value AS profile_domain_name, cdn.id cdn_id "+
		"FROM profile, parameter, cdn, profile_parameter "+
		"WHERE parameter.name='domain_name' "+
		"AND parameter.config_file='CRConfig.json' "+
		"AND parameter.value = cdn.domain_name "+
		"AND parameter.id in (select parameter from profile_parameter where profile=profile.id)) as domainlist "+
		"WHERE id = domainlist.profile_id", txn)

	doExec("ALTER TABLE deliveryservice ALTER profile DROP NOT NULL", txn)

	doExec("UPDATE deliveryservice SET profile=NULL", txn)

	doExec("INSERT INTO parameter (name, config_file, value) VALUES ('mso.parent_retry', 'parent.config', 'simple_retry');", txn)
	doExec("INSERT INTO parameter (name, config_file, value) VALUES ('mso.parent_retry', 'parent.config', 'unavailable_server_retry');", txn)
	doExec("INSERT INTO parameter (name, config_file, value) VALUES ('mso.parent_retry', 'parent.config', 'both');", txn)

	type Profile struct {
		Id                 int64
		Name               string
		Desc               string
		Type               string
		Cdn                int64
		MidHeaderRewrite   string
		MultiSiteOriginAlg int64
		XMLId              string
	}
	// move data
	pmap := make(map[string]Profile)
	rows, err := txn.Query("SELECT id,xml_id,mid_header_rewrite,multi_site_origin,multi_site_origin_algorithm,cdn_id FROM deliveryservice WHERE multi_site_origin=true")
	if err != nil {
		fmt.Println("Error:", err)
		fmt.Println("Attempting Roll back!")
		txn.Rollback()
		os.Exit(1)
	}
	existingParam := make(map[string]int64)
	for rows.Next() {
		var (
			id                          int64
			xml_id                      string
			mid_header_rewrite          sql.NullString
			multi_site_origin           sql.NullBool
			multi_site_origin_algorithm sql.NullInt64
			cdn_id                      int64
		)
		err := rows.Scan(&id, &xml_id, &mid_header_rewrite, &multi_site_origin, &multi_site_origin_algorithm, &cdn_id)
		checkErr(err, txn)

		pName := "DS_" + xml_id
		pDesc := "Deliveryservice profile for " + xml_id
		pType := "DS_PROFILE"
		mhrString := mid_header_rewrite.String
		pmap[pName] = Profile{
			Id:                 -1,
			Name:               pName,
			Desc:               pDesc,
			Type:               pType,
			Cdn:                cdn_id,
			MidHeaderRewrite:   mhrString,
			MultiSiteOriginAlg: multi_site_origin_algorithm.Int64,
			XMLId:              xml_id,
		}
	}
	err = rows.Err()
	checkErr(err, txn)
	rows.Close()

	for _, prof := range pmap {
		fmt.Println("--\nINSERT INTO PROFILE (name, description, type, cdn) VALUES($1, $2, $3, $4) RETURNING id", prof.Name, prof.Desc, prof.Type, prof.Cdn)
		newRow := txn.QueryRow("INSERT INTO PROFILE (name, description, type, cdn) VALUES($1, $2, $3, $4) RETURNING id", prof.Name, prof.Desc, prof.Type, prof.Cdn)
		var newProfileId int64
		err := newRow.Scan(&newProfileId)
		checkErr(err, txn)

		remainingString := ""
		parent_retry := ""
		var regexpOne = regexp.MustCompile(`^\s*set-config\s+proxy.config.http.parent_origin.(\S+)\s+(.*)$`)
		if prof.MidHeaderRewrite != "" {
			remapParts := strings.Split(prof.MidHeaderRewrite, "__RETURN__")
			for _, line := range remapParts {
				fmt.Println(line)
				match := regexpOne.FindSubmatch([]byte(line))
				if len(match) != 0 {
					var newId int64
					var ok bool
					newId, ok = existingParam[string(match[1])+string(match[2])]
					//	fmt.Printf("%s -> %v %v\n", string(match[1])+string(match[2]), newId, ok)
					if !ok {
						if string(match[1]) == "simple_retry_enabled" || string(match[1]) == "dead_server_retry_enabled" {
							if parent_retry == "" {
								parent_retry = string(match[1])
							} else {
								parent_retry = "both"
							}
						} else if string(match[1]) == "dead_server_retry_response_codes" {
							// unavailable_server_retry_responses is the only one that survives?
							//	if string(match[1]) == "dead_server_retry_response_codes" {
							pName := strings.Replace("mso."+string(match[1]), "dead", "unavailable", 1) // "mso.unavailable_server_retry_response_codes"
							pName = strings.Replace(pName, "response_codes", "responses", 1) // "mso.unavailable_server_retry_responses"
							fmt.Println("INSERT INTO PARAMETER (name, config_file, value) VALUES ($1, $2, $3) RETURNING id",
								pName, "parent.config", string(match[2]))
							newRow := txn.QueryRow("INSERT INTO PARAMETER (name, config_file, value) VALUES ($1, $2, $3) RETURNING id",
								pName, "parent.config", string(match[2]))
							err := newRow.Scan(&newId)
							checkErr(err, txn)
							existingParam[string(match[1])+string(match[2])] = newId
						}
					} else {
						newId = existingParam[string(match[1])+string(match[2])]
					}
					if newId != 0 {
						fmt.Println("INSERT INTO PROFILE_PARAMETER (parameter, profile) VALUES ($1, $2)", newId, newProfileId)
						_, err = txn.Exec("INSERT INTO PROFILE_PARAMETER (parameter, profile) VALUES ($1, $2)", newId, newProfileId)
						checkErr(err, txn)
					}
				} else {
					if !strings.HasSuffix(line, "__RETURN__") {
						remainingString = remainingString + line + "__RETURN__"
					}
				}
			}
		}
		/* Not doing the removal of the header_rewrite stuff  at this time, so we can support both 5.x and 6.x - 6.x will have a warning if these are
		   not removed from the DS, but should run just fine.

		fmt.Printf("MHRW was: %s, \nMHRW now: %s\n", prof.MidHeaderRewrite, remainingString)

		_, err = txn.Exec("UPDATE deliveryservice set mid_header_rewrite=$1 where xml_id=$2", remainingString, prof.XMLId)
		checkErr(err, txn)

		*/

		var newId int64
		var ok bool
		newId, ok = existingParam["mso.algorithm"+"parent.config"+algorithmStr[prof.MultiSiteOriginAlg]]
		if !ok {
			fmt.Println("INSERT INTO PARAMETER (name, config_file, value) VALUES ($1, $2, $3) RETURNING id",
				"mso.algorithm", "parent.config", algorithmStr[prof.MultiSiteOriginAlg])
			newRow = txn.QueryRow("INSERT INTO PARAMETER (name, config_file, value) VALUES ($1, $2, $3) RETURNING id",
				"mso.algorithm", "parent.config", algorithmStr[prof.MultiSiteOriginAlg])
			err = newRow.Scan(&newId)
			checkErr(err, txn)
			existingParam["mso.algorithm"+"parent.config"+algorithmStr[prof.MultiSiteOriginAlg]] = newId
		} else {
			newId = existingParam["mso.algorithm"+"parent.config"+algorithmStr[prof.MultiSiteOriginAlg]]
		}

		fmt.Println("INSERT INTO PROFILE_PARAMETER (parameter, profile) VALUES ($1, $2)", newId, newProfileId)
		_, err = txn.Exec("INSERT INTO PROFILE_PARAMETER (parameter, profile) VALUES ($1, $2)", newId, newProfileId)
		checkErr(err, txn)

		fmt.Println("INSERT INTO profile_parameter (profile, parameter) SELECT $1, id FROM parameter "+
			"WHERE config_file='parent.config' AND name='mso.parent_retry' AND value=$2", newProfileId, parent_retry)
		_, err = txn.Exec("INSERT INTO profile_parameter (profile, parameter) SELECT $1, id FROM parameter "+
			"WHERE config_file='parent.config' AND name='mso.parent_retry' AND value=$2", newProfileId, parent_retry)
		checkErr(err, txn)

		_, err = txn.Exec("UPDATE deliveryservice SET profile=$1 WHERE xml_id=$2", newProfileId, prof.XMLId)
		checkErr(err, txn)
	}
}

// Down is executed when this migration is rolled back
func Down_20170205101432(txn *sql.Tx) {
	//
}
<|endoftext|>


<commit_before>package approuter

import (
	"context"

	"github.com/rancher/rancher/pkg/ingresswrapper"
	"github.com/rancher/rancher/pkg/types/config"
)

// This controller is responsible for watching all the ingress resources in the cluster and creates the following DNS entries
// in <rancher_root_domain>:
// <$ingress_name>.<$namespace>.<$cluster_id>.<rancher_root_domain> => [ingress IPs]
// When an ingress resource has more than 10 IPs, only 10 IPs will be returned by DNS.
// In an RKE cluster, when a node becomes unhealthy and the corresponding nginx ingress resource becomes unavailable,
// the dynamic DNS controller updates the DNS mapping to remove that node IP from the list.
// Every once in a while (default 24h), the dynamic DNS controller will call renew to update the expiration time

func Register(ctx context.Context, cluster *config.UserContext) {
	secrets := cluster.Management.Core.Secrets("")
	secretLister := cluster.Management.Core.Secrets("").Controller().Lister()
	workload := cluster.UserOnlyContext()
	c := &Controller{
		ingressInterface: ingresswrapper.NewCompatInterface(workload.Networking, workload.Extensions, workload.K8sClient),
		ingressLister:    ingresswrapper.NewCompatLister(workload.Networking, workload.Extensions, workload.K8sClient),
		dnsClient:        NewClient(secrets, secretLister, workload.ClusterName),
	}
	if c.ingressInterface.ServerSupportsIngressV1 {
		workload.Networking.Ingresses("").AddHandler(ctx, "approuterController", ingresswrapper.CompatSyncV1(c.sync))
	} else {
		workload.Extensions.Ingresses("").AddHandler(ctx, "approuterController", ingresswrapper.CompatSyncV1Beta1(c.sync))
	}
	go c.renew(ctx)
}
<commit_msg>Defer start of approuter user controller<commit_after>package approuter

import (
	"context"

	v3 "github.com/rancher/rancher/pkg/apis/management.cattle.io/v3"
	"github.com/rancher/rancher/pkg/ingresswrapper"
	"github.com/rancher/rancher/pkg/settings"
	"github.com/rancher/rancher/pkg/types/config"
	"k8s.io/apimachinery/pkg/runtime"
)

var (
	invalidRDNSServerBaseURL = "https://api.lb.rancher.cloud/v1"
)

// This controller is responsible for watching all the ingress resources in the cluster and creates the following DNS entries
// in <rancher_root_domain>:
// <$ingress_name>.<$namespace>.<$cluster_id>.<rancher_root_domain> => [ingress IPs]
// When an ingress resource has more than 10 IPs, only 10 IPs will be returned by DNS.
// In an RKE cluster, when a node becomes unhealthy and the corresponding nginx ingress resource becomes unavailable,
// the dynamic DNS controller updates the DNS mapping to remove that node IP from the list.
// Every once in a while (default 24h), the dynamic DNS controller will call renew to update the expiration time

func Register(ctx context.Context, cluster *config.UserContext) {
	starter := cluster.DeferredStart(ctx, func(ctx context.Context) error {
		registerDeferred(ctx, cluster)
		return nil
	})
	settingsController := cluster.Management.Management.Settings("")
	settingsController.AddHandler(ctx, "approuter-deferred",
		func(key string, setting *v3.Setting) (runtime.Object, error) {
			if settings.RDNSServerBaseURL.Get() != invalidRDNSServerBaseURL &&
				settings.RDNSServerBaseURL.Get() != "" {
				return setting, starter()
			}
			return setting, nil
		})
}

func registerDeferred(ctx context.Context, cluster *config.UserContext) {
	secrets := cluster.Management.Core.Secrets("")
	secretLister := cluster.Management.Core.Secrets("").Controller().Lister()
	workload := cluster.UserOnlyContext()
	c := &Controller{
		ingressInterface: ingresswrapper.NewCompatInterface(workload.Networking, workload.Extensions, workload.K8sClient),
		ingressLister:    ingresswrapper.NewCompatLister(workload.Networking, workload.Extensions, workload.K8sClient),
		dnsClient:        NewClient(secrets, secretLister, workload.ClusterName),
	}
	if c.ingressInterface.ServerSupportsIngressV1 {
		workload.Networking.Ingresses("").AddHandler(ctx, "approuterController", ingresswrapper.CompatSyncV1(c.sync))
	} else {
		workload.Extensions.Ingresses("").AddHandler(ctx, "approuterController", ingresswrapper.CompatSyncV1Beta1(c.sync))
	}
	go c.renew(ctx)
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"math/rand"
	"net/http"
	"reflect"
	"time"

	"golang.org/x/net/context"

	"github.com/pressly/chi"
	"github.com/pressly/chi/_examples/render/data"
	"github.com/pressly/chi/_examples/render/presenter/v1"
	"github.com/pressly/chi/_examples/render/presenter/v2"
	"github.com/pressly/chi/_examples/render/presenter/v3"
	"github.com/pressly/chi/middleware"
	"github.com/pressly/chi/render"
)

func main() {
	r := chi.NewRouter()

	r.Use(middleware.RequestID)
	r.Use(middleware.Logger)
	r.Use(middleware.Recoverer)
	r.Use(render.ParseContentType)
	r.Use(render.UsePresenter(v3.Presenter)) // API version 3 (latest) by default.

	r.Get("/", getArticle)                                      // API version 3 (latest).
	r.Get("/v3", getArticle)                                    // API version 3.
	r.Get("/v2", render.UsePresenter(v2.Presenter), getArticle) // API version 2.
	r.Get("/v1", render.UsePresenter(v1.Presenter), getArticle) // API version 1.

	r.Get("/error", randomErrorHandler)

	http.ListenAndServe(":3333", r)
}

func getArticle(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	article := &data.Article{
		ID:    1,
		Title: "Article #1",
		Data:  []string{"one", "two", "three", "four"},
		CustomDataForAuthUsers: "secret data for auth'd users only",
	}

	// Simulate some context values:
	// 1. ?auth=true simluates authenticated session/user.
	// 2. ?error=true simulates random error.
	if r.URL.Query().Get("auth") != "" {
		ctx = context.WithValue(ctx, "auth", true)
	}
	if r.URL.Query().Get("error") != "" {
		render.Respond(ctx, w, errors.New("error"))
		return
	}

	render.Respond(ctx, w, article)
}

func randomErrorHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	errors := []error{data.ErrUnauthorized, data.ErrForbidden, data.ErrNotFound}

	rand.Seed(time.Now().Unix())
	render.Respond(ctx, w, errors[rand.Intn(len(errors))])
}

func init() {
	// custom responder that sets response status code based on Error value/type.
	render.Respond = func(ctx context.Context, w http.ResponseWriter, v interface{}) {
		val := reflect.ValueOf(v)
		if err, ok := val.Interface().(error); ok {
			switch err {
			case data.ErrUnauthorized:
				ctx = render.Status(ctx, 401)
			case data.ErrForbidden:
				ctx = render.Status(ctx, 403)
			case data.ErrNotFound:
				ctx = render.Status(ctx, 404)
			default:
				ctx = render.Status(ctx, 500)
			}
			render.DefaultRespond(ctx, w, map[string]string{"error": err.Error()})
			return
		}

		render.DefaultRespond(ctx, w, v)
	}
}
<commit_msg>Render + Presenter example for chi v2<commit_after>package main

import (
	"errors"
	"math/rand"
	"net/http"
	"reflect"
	"time"

	"golang.org/x/net/context"

	"github.com/pressly/chi"
	"github.com/pressly/chi/_examples/render/data"
	"github.com/pressly/chi/_examples/render/presenter/v1"
	"github.com/pressly/chi/_examples/render/presenter/v2"
	"github.com/pressly/chi/_examples/render/presenter/v3"
	"github.com/pressly/chi/middleware"
	"github.com/pressly/chi/render"
)

func main() {
	r := chi.NewRouter()

	r.Use(middleware.RequestID)
	r.Use(middleware.Logger)
	r.Use(middleware.Recoverer)
	r.Use(render.ParseContentType)
	r.Use(render.UsePresenter(v3.Presenter)) // API version 3 (latest) by default.

	// API version 3.
	r.Get("/", redirectV3)
	r.Route("/v3", func(r chi.Router) {
		r.Use(render.UsePresenter(v3.Presenter))
		r.Get("/", getArticle)
	})

	// API version 2.
	r.Route("/v2", func(r chi.Router) {
		r.Use(render.UsePresenter(v2.Presenter))
		r.Get("/", getArticle)
	})

	// API version 1.
	r.Route("/v1", func(r chi.Router) {
		r.Use(render.UsePresenter(v1.Presenter))
		r.Get("/", getArticle)
	})

	r.Get("/error", randomErrorHandler)

	http.ListenAndServe(":3333", r)
}

func getArticle(w http.ResponseWriter, r *http.Request) {
	article := &data.Article{
		ID:    1,
		Title: "Article #1",
		Data:  []string{"one", "two", "three", "four"},
		CustomDataForAuthUsers: "secret data for auth'd users only",
	}

	// Simulate some context values:
	// 1. ?auth=true simluates authenticated session/user.
	// 2. ?error=true simulates random error.
	if r.URL.Query().Get("auth") != "" {
		r = r.WithContext(context.WithValue(r.Context(), "auth", true))
	}
	if r.URL.Query().Get("error") != "" {
		render.Respond(w, r, errors.New("error"))
		return
	}

	render.Respond(w, r, article)
}

func redirectV3(w http.ResponseWriter, r *http.Request) {
	http.Redirect(w, r, "/v3", 302)
}

func randomErrorHandler(w http.ResponseWriter, r *http.Request) {
	errors := []error{data.ErrUnauthorized, data.ErrForbidden, data.ErrNotFound}

	rand.Seed(time.Now().Unix())
	render.Respond(w, r, errors[rand.Intn(len(errors))])
}

func init() {
	// custom responder that sets response status code based on Error value/type.
	render.Respond = func(w http.ResponseWriter, r *http.Request, v interface{}) {
		val := reflect.ValueOf(v)
		if err, ok := val.Interface().(error); ok {
			switch err {
			case data.ErrUnauthorized:
				r = render.Status(r, 401)
			case data.ErrForbidden:
				r = render.Status(r, 403)
			case data.ErrNotFound:
				r = render.Status(r, 404)
			default:
				r = render.Status(r, 500)
			}
			render.DefaultRespond(w, r, map[string]string{"error": err.Error()})
			return
		}

		render.DefaultRespond(w, r, v)
	}
}
<|endoftext|>


<commit_before>// +build linux
package devmapper

import (
	"fmt"
	"os"
	"os/exec"
	"reflect"
	"strings"
	"testing"
)

const (
	dataDev     = "/dev/loop0"
	metadataDev = "/dev/loop1"
	poolName    = "test_pool"
	devRoot     = "/tmp/devmapper"
)

func setUp() error {
	cmd := exec.Command("mkdir", "-p", devRoot)
	err := cmd.Run()
	if err != nil {
		return err
	}
	return nil
}

func tearDown() error {
	cmd := exec.Command("dmsetup", "remove", poolName)
	err := cmd.Run()
	if err != nil {
		return err
	}
	cmd = exec.Command("rm", "-rf", devRoot)
	err = cmd.Run()
	if err != nil {
		return err
	}
	return nil
}

func TestMain(m *testing.M) {
	err := setUp()
	if err != nil {
		fmt.Println("Failed to setup due to ", err)
		os.Exit(-1)
	}

	errCode := m.Run()

	err = tearDown()
	if err != nil {
		fmt.Println("Failed to tear down due to ", err)
		os.Exit(-1)
	}

	os.Exit(errCode)
}

func TestInit(t *testing.T) {
	config := make(map[string]string)

	_, err := Init(devRoot, config)
	if err == nil || err.Error() != "data device or metadata device unspecified" {
		t.Fatal("Expected error doesn't happened")
	}
	config[DM_DATA_DEV] = dataDev
	config[DM_METADATA_DEV] = metadataDev
	config[DM_THINPOOL_BLOCK_SIZE] = "100"
	_, err = Init(devRoot, config)
	if err == nil || !strings.HasPrefix(err.Error(), "Block size must") {
		t.Fatal("Expected error doesn't happened")
	}

	config[DM_THINPOOL_NAME] = "test_pool"
	delete(config, DM_THINPOOL_BLOCK_SIZE)

	driver, err := Init(devRoot, config)
	if err != nil {
		t.Fatal("Fail to init volmgr", err)
	}

	newDriver, err := Init(devRoot, config)
	if err != nil {
		t.Fatal("Fail to reload device mapper driver", err)
	}

	drv1, ok := driver.(*Driver)
	if !ok {
		t.Fatal("Fail to convert type")
	}
	drv2, ok := newDriver.(*Driver)
	if !ok {
		t.Fatal("Fail to convert type")
	}

	if !reflect.DeepEqual(*drv1, *drv2) {
		t.Fatal("Fail to verify the information from driver config")
	}

	if drv1.configFile != devRoot+"/devmapper.cfg" {
		t.Fatal("Failed driver.configFile")
	}

	if drv1.DataDevice != dataDev || drv1.MetadataDevice != metadataDev {
		t.Fatal("Failed device.DataDevice or dev.MetadataDevice check")
	}
}
<commit_msg>Change devmapper_test.go to use testify framework<commit_after>// +build linux
package devmapper

import (
	"fmt"
	"github.com/stretchr/testify/assert"
	"os"
	"os/exec"
	"reflect"
	"strings"
	"testing"
)

const (
	dataDev     = "/dev/loop0"
	metadataDev = "/dev/loop1"
	poolName    = "test_pool"
	devRoot     = "/tmp/devmapper"
)

func setUp() error {
	cmd := exec.Command("mkdir", "-p", devRoot)
	err := cmd.Run()
	if err != nil {
		return err
	}
	return nil
}

func tearDown() error {
	cmd := exec.Command("dmsetup", "remove", poolName)
	err := cmd.Run()
	if err != nil {
		return err
	}
	cmd = exec.Command("rm", "-rf", devRoot)
	err = cmd.Run()
	if err != nil {
		return err
	}
	return nil
}

func TestMain(m *testing.M) {
	err := setUp()
	if err != nil {
		fmt.Println("Failed to setup due to ", err)
		os.Exit(-1)
	}

	errCode := m.Run()

	err = tearDown()
	if err != nil {
		fmt.Println("Failed to tear down due to ", err)
		os.Exit(-1)
	}

	os.Exit(errCode)
}

func TestInit(t *testing.T) {
	config := make(map[string]string)

	_, err := Init(devRoot, config)
	assert.NotNil(t, err)
	assert.Equal(t, err.Error(), "data device or metadata device unspecified")

	config[DM_DATA_DEV] = dataDev
	config[DM_METADATA_DEV] = metadataDev
	config[DM_THINPOOL_BLOCK_SIZE] = "100"
	_, err = Init(devRoot, config)
	assert.NotNil(t, err)
	assert.True(t, strings.HasPrefix(err.Error(), "Block size must"))

	config[DM_THINPOOL_NAME] = "test_pool"
	delete(config, DM_THINPOOL_BLOCK_SIZE)

	driver, err := Init(devRoot, config)
	assert.Nil(t, err)

	newDriver, err := Init(devRoot, config)
	assert.Nil(t, err)

	drv1, ok := driver.(*Driver)
	assert.True(t, ok)
	drv2, ok := newDriver.(*Driver)
	assert.True(t, ok)

	if !reflect.DeepEqual(*drv1, *drv2) {
		t.Fatal("Fail to verify the information from driver config")
	}

	assert.Equal(t, drv1.configFile, devRoot+"/devicemapper.cfg")

	assert.Equal(t, drv1.DataDevice, dataDev)
	assert.Equal(t, drv1.MetadataDevice, metadataDev)
}

func TestVolumeCreate(t *testing.T) {
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package unicode_normalize

import (
	"fmt"

	"code.google.com/p/go.text/unicode/norm"
	"github.com/blevesearch/bleve/analysis"
	"github.com/blevesearch/bleve/registry"
)

const Name = "normalize_unicode"

const NFC = "nfc"
const NFD = "nfd"
const NFKC = "nfkc"
const NFKD = "nfkd"

var forms = map[string]norm.Form{
	NFC:  norm.NFC,
	NFD:  norm.NFD,
	NFKC: norm.NFKC,
	NFKD: norm.NFKC,
}

type UnicodeNormalizeFilter struct {
	form norm.Form
}

func NewUnicodeNormalizeFilter(formName string) (*UnicodeNormalizeFilter, error) {
	form, ok := forms[formName]
	if !ok {
		return nil, fmt.Errorf("no form named %s", formName)
	}
	return &UnicodeNormalizeFilter{
		form: form,
	}, nil
}

func MustNewUnicodeNormalizeFilter(formName string) *UnicodeNormalizeFilter {
	filter, err := NewUnicodeNormalizeFilter(formName)
	if err != nil {
		panic(err)
	}
	return filter
}

func (s *UnicodeNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
	for _, token := range input {
		token.Term = s.form.Bytes(token.Term)
	}
	return input
}

func UnicodeNormalizeFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
	formVal, ok := config["form"].(string)
	if !ok {
		return nil, fmt.Errorf("must specify form")
	}
	form := formVal
	return NewUnicodeNormalizeFilter(form)
}

func init() {
	registry.RegisterTokenFilter(Name, UnicodeNormalizeFilterConstructor)
}
<commit_msg>Update "code.google.com" import paths<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package unicode_normalize

import (
	"fmt"

	"golang.org/x/text/unicode/norm"
	"github.com/blevesearch/bleve/analysis"
	"github.com/blevesearch/bleve/registry"
)

const Name = "normalize_unicode"

const NFC = "nfc"
const NFD = "nfd"
const NFKC = "nfkc"
const NFKD = "nfkd"

var forms = map[string]norm.Form{
	NFC:  norm.NFC,
	NFD:  norm.NFD,
	NFKC: norm.NFKC,
	NFKD: norm.NFKC,
}

type UnicodeNormalizeFilter struct {
	form norm.Form
}

func NewUnicodeNormalizeFilter(formName string) (*UnicodeNormalizeFilter, error) {
	form, ok := forms[formName]
	if !ok {
		return nil, fmt.Errorf("no form named %s", formName)
	}
	return &UnicodeNormalizeFilter{
		form: form,
	}, nil
}

func MustNewUnicodeNormalizeFilter(formName string) *UnicodeNormalizeFilter {
	filter, err := NewUnicodeNormalizeFilter(formName)
	if err != nil {
		panic(err)
	}
	return filter
}

func (s *UnicodeNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
	for _, token := range input {
		token.Term = s.form.Bytes(token.Term)
	}
	return input
}

func UnicodeNormalizeFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
	formVal, ok := config["form"].(string)
	if !ok {
		return nil, fmt.Errorf("must specify form")
	}
	form := formVal
	return NewUnicodeNormalizeFilter(form)
}

func init() {
	registry.RegisterTokenFilter(Name, UnicodeNormalizeFilterConstructor)
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package command

import (
	"errors"
	"fmt"
	"sync/atomic"
	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/clientv3/mirror"
	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
	"github.com/coreos/etcd/mvcc/mvccpb"
	"github.com/spf13/cobra"
	"golang.org/x/net/context"
)

var (
	mminsecureTr bool
	mmcert       string
	mmkey        string
	mmcacert     string
	mmprefix     string
)

// NewMakeMirrorCommand returns the cobra command for "makeMirror".
func NewMakeMirrorCommand() *cobra.Command {
	c := &cobra.Command{
		Use:   "make-mirror [options] <destination>",
		Short: "make-mirror makes a mirror at the destination etcd cluster",
		Run:   makeMirrorCommandFunc,
	}

	c.Flags().StringVar(&mmprefix, "prefix", "", "the key-value prefix to mirror")
	// TODO: add dest-prefix to mirror a prefix to a different prefix in the destination cluster?
	c.Flags().StringVar(&mmcert, "dest-cert", "", "identify secure client using this TLS certificate file for the destination cluster")
	c.Flags().StringVar(&mmkey, "dest-key", "", "identify secure client using this TLS key file")
	c.Flags().StringVar(&mmcacert, "dest-cacert", "", "verify certificates of TLS enabled secure servers using this CA bundle")
	// TODO: secure by default when etcd enables secure gRPC by default.
	c.Flags().BoolVar(&mminsecureTr, "dest-insecure-transport", true, "disable transport security for client connections")

	return c
}

func makeMirrorCommandFunc(cmd *cobra.Command, args []string) {
	if len(args) != 1 {
		ExitWithError(ExitBadArgs, errors.New("make-mirror takes one destination argument."))
	}

	dialTimeout := dialTimeoutFromCmd(cmd)
	sec := &secureCfg{
		cert:              mmcert,
		key:               mmkey,
		cacert:            mmcacert,
		insecureTransport: mminsecureTr,
	}

	dc := mustClient([]string{args[0]}, dialTimeout, sec, nil)
	c := mustClientFromCmd(cmd)

	err := makeMirror(context.TODO(), c, dc)
	ExitWithError(ExitError, err)
}

func makeMirror(ctx context.Context, c *clientv3.Client, dc *clientv3.Client) error {
	total := int64(0)

	go func() {
		for {
			time.Sleep(30 * time.Second)
			fmt.Println(atomic.LoadInt64(&total))
		}
	}()

	// TODO: remove the prefix of the destination cluster?
	s := mirror.NewSyncer(c, mmprefix, 0)

	rc, errc := s.SyncBase(ctx)

	for r := range rc {
		for _, kv := range r.Kvs {
			_, err := dc.Put(ctx, string(kv.Key), string(kv.Value))
			if err != nil {
				return err
			}
			atomic.AddInt64(&total, 1)
		}
	}

	err := <-errc
	if err != nil {
		return err
	}

	wc := s.SyncUpdates(ctx)

	for wr := range wc {
		if wr.CompactRevision != 0 {
			return rpctypes.ErrCompacted
		}

		var rev int64
		ops := []clientv3.Op{}

		for _, ev := range wr.Events {
			nrev := ev.Kv.ModRevision
			if rev != 0 && nrev > rev {
				_, err := dc.Txn(ctx).Then(ops...).Commit()
				if err != nil {
					return err
				}
				ops = []clientv3.Op{}
			}
			switch ev.Type {
			case mvccpb.PUT:
				ops = append(ops, clientv3.OpPut(string(ev.Kv.Key), string(ev.Kv.Value)))
				atomic.AddInt64(&total, 1)
			case mvccpb.DELETE, mvccpb.EXPIRE:
				ops = append(ops, clientv3.OpDelete(string(ev.Kv.Key)))
				atomic.AddInt64(&total, 1)
			default:
				panic("unexpected event type")
			}
		}

		if len(ops) != 0 {
			_, err := dc.Txn(ctx).Then(ops...).Commit()
			if err != nil {
				return err
			}
		}
	}

	return nil
}
<commit_msg>etcdctl/ctlv3: remove mvccpb.EXPIRE in mirror cmd<commit_after>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package command

import (
	"errors"
	"fmt"
	"sync/atomic"
	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/clientv3/mirror"
	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
	"github.com/coreos/etcd/mvcc/mvccpb"
	"github.com/spf13/cobra"
	"golang.org/x/net/context"
)

var (
	mminsecureTr bool
	mmcert       string
	mmkey        string
	mmcacert     string
	mmprefix     string
)

// NewMakeMirrorCommand returns the cobra command for "makeMirror".
func NewMakeMirrorCommand() *cobra.Command {
	c := &cobra.Command{
		Use:   "make-mirror [options] <destination>",
		Short: "make-mirror makes a mirror at the destination etcd cluster",
		Run:   makeMirrorCommandFunc,
	}

	c.Flags().StringVar(&mmprefix, "prefix", "", "the key-value prefix to mirror")
	// TODO: add dest-prefix to mirror a prefix to a different prefix in the destination cluster?
	c.Flags().StringVar(&mmcert, "dest-cert", "", "identify secure client using this TLS certificate file for the destination cluster")
	c.Flags().StringVar(&mmkey, "dest-key", "", "identify secure client using this TLS key file")
	c.Flags().StringVar(&mmcacert, "dest-cacert", "", "verify certificates of TLS enabled secure servers using this CA bundle")
	// TODO: secure by default when etcd enables secure gRPC by default.
	c.Flags().BoolVar(&mminsecureTr, "dest-insecure-transport", true, "disable transport security for client connections")

	return c
}

func makeMirrorCommandFunc(cmd *cobra.Command, args []string) {
	if len(args) != 1 {
		ExitWithError(ExitBadArgs, errors.New("make-mirror takes one destination argument."))
	}

	dialTimeout := dialTimeoutFromCmd(cmd)
	sec := &secureCfg{
		cert:              mmcert,
		key:               mmkey,
		cacert:            mmcacert,
		insecureTransport: mminsecureTr,
	}

	dc := mustClient([]string{args[0]}, dialTimeout, sec, nil)
	c := mustClientFromCmd(cmd)

	err := makeMirror(context.TODO(), c, dc)
	ExitWithError(ExitError, err)
}

func makeMirror(ctx context.Context, c *clientv3.Client, dc *clientv3.Client) error {
	total := int64(0)

	go func() {
		for {
			time.Sleep(30 * time.Second)
			fmt.Println(atomic.LoadInt64(&total))
		}
	}()

	// TODO: remove the prefix of the destination cluster?
	s := mirror.NewSyncer(c, mmprefix, 0)

	rc, errc := s.SyncBase(ctx)

	for r := range rc {
		for _, kv := range r.Kvs {
			_, err := dc.Put(ctx, string(kv.Key), string(kv.Value))
			if err != nil {
				return err
			}
			atomic.AddInt64(&total, 1)
		}
	}

	err := <-errc
	if err != nil {
		return err
	}

	wc := s.SyncUpdates(ctx)

	for wr := range wc {
		if wr.CompactRevision != 0 {
			return rpctypes.ErrCompacted
		}

		var rev int64
		ops := []clientv3.Op{}

		for _, ev := range wr.Events {
			nrev := ev.Kv.ModRevision
			if rev != 0 && nrev > rev {
				_, err := dc.Txn(ctx).Then(ops...).Commit()
				if err != nil {
					return err
				}
				ops = []clientv3.Op{}
			}
			switch ev.Type {
			case mvccpb.PUT:
				ops = append(ops, clientv3.OpPut(string(ev.Kv.Key), string(ev.Kv.Value)))
				atomic.AddInt64(&total, 1)
			case mvccpb.DELETE:
				ops = append(ops, clientv3.OpDelete(string(ev.Kv.Key)))
				atomic.AddInt64(&total, 1)
			default:
				panic("unexpected event type")
			}
		}

		if len(ops) != 0 {
			_, err := dc.Txn(ctx).Then(ops...).Commit()
			if err != nil {
				return err
			}
		}
	}

	return nil
}
<|endoftext|>