<commit_before>package ttransport

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"runtime/debug"
	"strconv"
	"sync"
	"testing"
	"time"

	crand "crypto/rand"
	mrand "math/rand"

	"github.com/libp2p/go-libp2p-core/mux"
	"github.com/libp2p/go-libp2p-core/peer"
	"github.com/libp2p/go-libp2p-core/transport"
	"github.com/libp2p/go-libp2p-testing/race"

	ma "github.com/multiformats/go-multiaddr"
)

// VerboseDebugging can be set to true to enable verbose debug logging in the
// stream stress tests.
var VerboseDebugging = false

var randomness []byte

var StressTestTimeout = 1 * time.Minute

func init() {
	// read 1MB of randomness
	randomness = make([]byte, 1<<20)
	if _, err := crand.Read(randomness); err != nil {
		panic(err)
	}

	if timeout := os.Getenv("TEST_STRESS_TIMEOUT_MS"); timeout != "" {
		if v, err := strconv.ParseInt(timeout, 10, 32); err == nil {
			StressTestTimeout = time.Duration(v) * time.Millisecond
		}
	}
}

type Options struct {
	connNum   int
	streamNum int
	msgNum    int
	msgMin    int
	msgMax    int
}

func fullClose(t *testing.T, s mux.MuxedStream) {
	if err := s.Close(); err != nil {
		t.Error(err)
		s.Reset()
		return
	}
	b, err := ioutil.ReadAll(s)
	if err != nil {
		t.Error(err)
	}
	if len(b) != 0 {
		t.Error("expected to be done reading")
	}
}

func randBuf(size int) []byte {
	n := len(randomness) - size
	if size < 1 {
		panic(fmt.Errorf("requested too large buffer (%d). max is %d", size, len(randomness)))
	}

	start := mrand.Intn(n)
	return randomness[start : start+size]
}

func checkErr(t *testing.T, err error) {
	t.Helper()
	if err != nil {
		debug.PrintStack()
		// TODO: not safe to call in parallel
		t.Fatal(err)
	}
}

func debugLog(t *testing.T, s string, args ...interface{}) {
	if VerboseDebugging {
		t.Logf(s, args...)
	}
}

func echoStream(t *testing.T, s mux.MuxedStream) {
	defer s.Close()
	// echo everything
	var err error
	if VerboseDebugging {
		t.Logf("accepted stream")
		_, err = io.Copy(&logWriter{t, s}, s)
		t.Log("closing stream")
	} else {
		_, err = io.Copy(s, s) // echo everything
	}
	if err != nil {
		t.Error(err)
	}
}

type logWriter struct {
	t *testing.T
	W io.Writer
}

func (lw *logWriter) Write(buf []byte) (int, error) {
	lw.t.Logf("logwriter: writing %d bytes", len(buf))
	return lw.W.Write(buf)
}

func goServe(t *testing.T, l transport.Listener) (done func()) {
	closed := make(chan struct{}, 1)

	go func() {
		for {
			c, err := l.Accept()
			if err != nil {
				select {
				case <-closed:
					return // closed naturally.
				default:
					checkErr(t, err)
				}
			}

			debugLog(t, "accepted connection")
			go func() {
				for {
					str, err := c.AcceptStream()
					if err != nil {
						break
					}
					go echoStream(t, str)
				}
			}()
		}
	}()

	return func() {
		closed <- struct{}{}
	}
}

func SubtestStress(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID, opt Options) {
	msgsize := 1 << 11
	errs := make(chan error, 0) // dont block anything.

	rateLimitN := 5000 // max of 5k funcs, because -race has 8k max.
	rateLimitChan := make(chan struct{}, rateLimitN)
	for i := 0; i < rateLimitN; i++ {
		rateLimitChan <- struct{}{}
	}

	rateLimit := func(f func()) {
		<-rateLimitChan
		f()
		rateLimitChan <- struct{}{}
	}

	writeStream := func(s mux.MuxedStream, bufs chan<- []byte) {
		debugLog(t, "writeStream %p, %d msgNum", s, opt.msgNum)

		for i := 0; i < opt.msgNum; i++ {
			buf := randBuf(msgsize)
			bufs <- buf
			debugLog(t, "%p writing %d bytes (message %d/%d #%x)", s, len(buf), i, opt.msgNum, buf[:3])
			if _, err := s.Write(buf); err != nil {
				errs <- fmt.Errorf("s.Write(buf): %s", err)
				continue
			}
		}
	}

	readStream := func(s mux.MuxedStream, bufs <-chan []byte) {
		debugLog(t, "readStream %p, %d msgNum", s, opt.msgNum)

		buf2 := make([]byte, msgsize)
		i := 0
		for buf1 := range bufs {
			i++
			debugLog(t, "%p reading %d bytes (message %d/%d #%x)", s, len(buf1), i-1, opt.msgNum, buf1[:3])

			if _, err := io.ReadFull(s, buf2); err != nil {
				errs <- fmt.Errorf("io.ReadFull(s, buf2): %s", err)
				debugLog(t, "%p failed to read %d bytes (message %d/%d #%x)", s, len(buf1), i-1, opt.msgNum, buf1[:3])
				continue
			}
			if !bytes.Equal(buf1, buf2) {
				errs <- fmt.Errorf("buffers not equal (%x != %x)", buf1[:3], buf2[:3])
			}
		}
	}

	openStreamAndRW := func(c mux.MuxedConn) {
		debugLog(t, "openStreamAndRW %p, %d opt.msgNum", c, opt.msgNum)

		s, err := c.OpenStream()
		if err != nil {
			errs <- fmt.Errorf("Failed to create NewStream: %s", err)
			return
		}

		bufs := make(chan []byte, opt.msgNum)
		go func() {
			writeStream(s, bufs)
			close(bufs)
		}()

		readStream(s, bufs)
		fullClose(t, s)
	}

	openConnAndRW := func() {
		debugLog(t, "openConnAndRW")

		l, err := ta.Listen(maddr)
		checkErr(t, err)

		done := goServe(t, l)
		defer done()

		c, err := tb.Dial(context.Background(), l.Multiaddr(), peerA)
		checkErr(t, err)

		// serve the outgoing conn, because some muxers assume
		// that we _always_ call serve. (this is an error?)
		go func() {
			debugLog(t, "serving connection")
			for {
				str, err := c.AcceptStream()
				if err != nil {
					break
				}
				go echoStream(t, str)
			}
		}()

		var wg sync.WaitGroup
		for i := 0; i < opt.streamNum; i++ {
			wg.Add(1)
			go rateLimit(func() {
				defer wg.Done()
				openStreamAndRW(c)
			})
		}
		wg.Wait()
		c.Close()
	}

	openConnsAndRW := func() {
		debugLog(t, "openConnsAndRW, %d conns", opt.connNum)

		var wg sync.WaitGroup
		for i := 0; i < opt.connNum; i++ {
			wg.Add(1)
			go rateLimit(func() {
				defer wg.Done()
				openConnAndRW()
			})
		}
		wg.Wait()
	}

	go func() {
		openConnsAndRW()
		close(errs) // done
	}()

	for err := range errs {
		t.Error(err)
	}

}

func SubtestStreamOpenStress(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	l, err := ta.Listen(maddr)
	checkErr(t, err)
	defer l.Close()

	count := 10000
	workers := 5

	if race.WithRace() {
		// the race detector can only deal with 8128 simultaneous goroutines, so let's make sure we don't go overboard.
		count = 1000
	}

	var (
		connA, connB transport.CapableConn
	)

	accepted := make(chan error, 1)
	go func() {
		var err error
		connA, err = l.Accept()
		accepted <- err
	}()
	connB, err = tb.Dial(context.Background(), l.Multiaddr(), peerA)
	checkErr(t, err)
	checkErr(t, <-accepted)

	defer func() {
		if connA != nil {
			connA.Close()
		}
		if connB != nil {
			connB.Close()
		}
	}()

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		for j := 0; j < workers; j++ {
			wg.Add(1)
			go func() {
				defer wg.Done()
				for i := 0; i < count; i++ {
					s, err := connA.OpenStream()
					if err != nil {
						t.Error(err)
						return
					}
					wg.Add(1)
					go func() {
						defer wg.Done()
						fullClose(t, s)
					}()
				}
			}()
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < count*workers; i++ {
			str, err := connB.AcceptStream()
			if err != nil {
				break
			}
			wg.Add(1)
			go func() {
				defer wg.Done()
				fullClose(t, str)
			}()
		}
	}()

	timeout := time.After(StressTestTimeout)
	done := make(chan struct{})

	go func() {
		wg.Wait()
		close(done)
	}()

	select {
	case <-timeout:
		t.Fatal("timed out receiving streams")
	case <-done:
	}
}

func SubtestStreamReset(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	l, err := ta.Listen(maddr)
	checkErr(t, err)

	done := make(chan struct{}, 2)
	go func() {
		muxa, err := l.Accept()
		checkErr(t, err)

		s, err := muxa.OpenStream()
		if err != nil {
			panic(err)
		}

		// Some transports won't open the stream until we write. That's
		// fine.
		s.Write([]byte("foo"))

		time.Sleep(time.Millisecond * 50)

		_, err = s.Write([]byte("bar"))
		if err == nil {
			t.Error("should have failed to write")
		}

		s.Close()
		done <- struct{}{}
	}()

	muxb, err := tb.Dial(context.Background(), l.Multiaddr(), peerA)
	checkErr(t, err)

	go func() {
		str, err := muxb.AcceptStream()
		checkErr(t, err)
		str.Reset()
		done <- struct{}{}
	}()

	<-done
	<-done
}

func SubtestStress1Conn1Stream1Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		connNum:   1,
		streamNum: 1,
		msgNum:    1,
		msgMax:    100,
		msgMin:    100,
	})
}

func SubtestStress1Conn1Stream100Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		connNum:   1,
		streamNum: 1,
		msgNum:    100,
		msgMax:    100,
		msgMin:    100,
	})
}

func SubtestStress1Conn100Stream100Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		connNum:   1,
		streamNum: 100,
		msgNum:    100,
		msgMax:    100,
		msgMin:    100,
	})
}

func SubtestStress50Conn10Stream50Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		connNum:   50,
		streamNum: 10,
		msgNum:    50,
		msgMax:    100,
		msgMin:    100,
	})
}

func SubtestStress1Conn1000Stream10Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		connNum:   1,
		streamNum: 1000,
		msgNum:    10,
		msgMax:    100,
		msgMin:    100,
	})
}

func SubtestStress1Conn100Stream100Msg10MB(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		connNum:   1,
		streamNum: 100,
		msgNum:    100,
		msgMax:    10000,
		msgMin:    1000,
	})
}
<commit_msg>Allow custom SubtestStress. (#7)<commit_after>package ttransport

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"runtime/debug"
	"strconv"
	"sync"
	"testing"
	"time"

	crand "crypto/rand"
	mrand "math/rand"

	"github.com/libp2p/go-libp2p-core/mux"
	"github.com/libp2p/go-libp2p-core/peer"
	"github.com/libp2p/go-libp2p-core/transport"
	"github.com/libp2p/go-libp2p-testing/race"

	ma "github.com/multiformats/go-multiaddr"
)

// VerboseDebugging can be set to true to enable verbose debug logging in the
// stream stress tests.
var VerboseDebugging = false

var randomness []byte

var StressTestTimeout = 1 * time.Minute

func init() {
	// read 1MB of randomness
	randomness = make([]byte, 1<<20)
	if _, err := crand.Read(randomness); err != nil {
		panic(err)
	}

	if timeout := os.Getenv("TEST_STRESS_TIMEOUT_MS"); timeout != "" {
		if v, err := strconv.ParseInt(timeout, 10, 32); err == nil {
			StressTestTimeout = time.Duration(v) * time.Millisecond
		}
	}
}

type Options struct {
	ConnNum   int
	StreamNum int
	MsgNum    int
	MsgMin    int
	MsgMax    int
}

func fullClose(t *testing.T, s mux.MuxedStream) {
	if err := s.Close(); err != nil {
		t.Error(err)
		s.Reset()
		return
	}
	b, err := ioutil.ReadAll(s)
	if err != nil {
		t.Error(err)
	}
	if len(b) != 0 {
		t.Error("expected to be done reading")
	}
}

func randBuf(size int) []byte {
	n := len(randomness) - size
	if size < 1 {
		panic(fmt.Errorf("requested too large buffer (%d). max is %d", size, len(randomness)))
	}

	start := mrand.Intn(n)
	return randomness[start : start+size]
}

func checkErr(t *testing.T, err error) {
	t.Helper()
	if err != nil {
		debug.PrintStack()
		// TODO: not safe to call in parallel
		t.Fatal(err)
	}
}

func debugLog(t *testing.T, s string, args ...interface{}) {
	if VerboseDebugging {
		t.Logf(s, args...)
	}
}

func echoStream(t *testing.T, s mux.MuxedStream) {
	defer s.Close()
	// echo everything
	var err error
	if VerboseDebugging {
		t.Logf("accepted stream")
		_, err = io.Copy(&logWriter{t, s}, s)
		t.Log("closing stream")
	} else {
		_, err = io.Copy(s, s) // echo everything
	}
	if err != nil {
		t.Error(err)
	}
}

type logWriter struct {
	t *testing.T
	W io.Writer
}

func (lw *logWriter) Write(buf []byte) (int, error) {
	lw.t.Logf("logwriter: writing %d bytes", len(buf))
	return lw.W.Write(buf)
}

func goServe(t *testing.T, l transport.Listener) (done func()) {
	closed := make(chan struct{}, 1)

	go func() {
		for {
			c, err := l.Accept()
			if err != nil {
				select {
				case <-closed:
					return // closed naturally.
				default:
					checkErr(t, err)
				}
			}

			debugLog(t, "accepted connection")
			go func() {
				for {
					str, err := c.AcceptStream()
					if err != nil {
						break
					}
					go echoStream(t, str)
				}
			}()
		}
	}()

	return func() {
		closed <- struct{}{}
	}
}

func SubtestStress(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID, opt Options) {
	msgsize := 1 << 11
	errs := make(chan error, 0) // dont block anything.

	rateLimitN := 5000 // max of 5k funcs, because -race has 8k max.
	rateLimitChan := make(chan struct{}, rateLimitN)
	for i := 0; i < rateLimitN; i++ {
		rateLimitChan <- struct{}{}
	}

	rateLimit := func(f func()) {
		<-rateLimitChan
		f()
		rateLimitChan <- struct{}{}
	}

	writeStream := func(s mux.MuxedStream, bufs chan<- []byte) {
		debugLog(t, "writeStream %p, %d MsgNum", s, opt.MsgNum)

		for i := 0; i < opt.MsgNum; i++ {
			buf := randBuf(msgsize)
			bufs <- buf
			debugLog(t, "%p writing %d bytes (message %d/%d #%x)", s, len(buf), i, opt.MsgNum, buf[:3])
			if _, err := s.Write(buf); err != nil {
				errs <- fmt.Errorf("s.Write(buf): %s", err)
				continue
			}
		}
	}

	readStream := func(s mux.MuxedStream, bufs <-chan []byte) {
		debugLog(t, "readStream %p, %d MsgNum", s, opt.MsgNum)

		buf2 := make([]byte, msgsize)
		i := 0
		for buf1 := range bufs {
			i++
			debugLog(t, "%p reading %d bytes (message %d/%d #%x)", s, len(buf1), i-1, opt.MsgNum, buf1[:3])

			if _, err := io.ReadFull(s, buf2); err != nil {
				errs <- fmt.Errorf("io.ReadFull(s, buf2): %s", err)
				debugLog(t, "%p failed to read %d bytes (message %d/%d #%x)", s, len(buf1), i-1, opt.MsgNum, buf1[:3])
				continue
			}
			if !bytes.Equal(buf1, buf2) {
				errs <- fmt.Errorf("buffers not equal (%x != %x)", buf1[:3], buf2[:3])
			}
		}
	}

	openStreamAndRW := func(c mux.MuxedConn) {
		debugLog(t, "openStreamAndRW %p, %d opt.MsgNum", c, opt.MsgNum)

		s, err := c.OpenStream()
		if err != nil {
			errs <- fmt.Errorf("Failed to create NewStream: %s", err)
			return
		}

		bufs := make(chan []byte, opt.MsgNum)
		go func() {
			writeStream(s, bufs)
			close(bufs)
		}()

		readStream(s, bufs)
		fullClose(t, s)
	}

	openConnAndRW := func() {
		debugLog(t, "openConnAndRW")

		l, err := ta.Listen(maddr)
		checkErr(t, err)

		done := goServe(t, l)
		defer done()

		c, err := tb.Dial(context.Background(), l.Multiaddr(), peerA)
		checkErr(t, err)

		// serve the outgoing conn, because some muxers assume
		// that we _always_ call serve. (this is an error?)
		go func() {
			debugLog(t, "serving connection")
			for {
				str, err := c.AcceptStream()
				if err != nil {
					break
				}
				go echoStream(t, str)
			}
		}()

		var wg sync.WaitGroup
		for i := 0; i < opt.StreamNum; i++ {
			wg.Add(1)
			go rateLimit(func() {
				defer wg.Done()
				openStreamAndRW(c)
			})
		}
		wg.Wait()
		c.Close()
	}

	openConnsAndRW := func() {
		debugLog(t, "openConnsAndRW, %d conns", opt.ConnNum)

		var wg sync.WaitGroup
		for i := 0; i < opt.ConnNum; i++ {
			wg.Add(1)
			go rateLimit(func() {
				defer wg.Done()
				openConnAndRW()
			})
		}
		wg.Wait()
	}

	go func() {
		openConnsAndRW()
		close(errs) // done
	}()

	for err := range errs {
		t.Error(err)
	}

}

func SubtestStreamOpenStress(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	l, err := ta.Listen(maddr)
	checkErr(t, err)
	defer l.Close()

	count := 10000
	workers := 5

	if race.WithRace() {
		// the race detector can only deal with 8128 simultaneous goroutines, so let's make sure we don't go overboard.
		count = 1000
	}

	var (
		connA, connB transport.CapableConn
	)

	accepted := make(chan error, 1)
	go func() {
		var err error
		connA, err = l.Accept()
		accepted <- err
	}()
	connB, err = tb.Dial(context.Background(), l.Multiaddr(), peerA)
	checkErr(t, err)
	checkErr(t, <-accepted)

	defer func() {
		if connA != nil {
			connA.Close()
		}
		if connB != nil {
			connB.Close()
		}
	}()

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		for j := 0; j < workers; j++ {
			wg.Add(1)
			go func() {
				defer wg.Done()
				for i := 0; i < count; i++ {
					s, err := connA.OpenStream()
					if err != nil {
						t.Error(err)
						return
					}
					wg.Add(1)
					go func() {
						defer wg.Done()
						fullClose(t, s)
					}()
				}
			}()
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < count*workers; i++ {
			str, err := connB.AcceptStream()
			if err != nil {
				break
			}
			wg.Add(1)
			go func() {
				defer wg.Done()
				fullClose(t, str)
			}()
		}
	}()

	timeout := time.After(StressTestTimeout)
	done := make(chan struct{})

	go func() {
		wg.Wait()
		close(done)
	}()

	select {
	case <-timeout:
		t.Fatal("timed out receiving streams")
	case <-done:
	}
}

func SubtestStreamReset(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	l, err := ta.Listen(maddr)
	checkErr(t, err)

	done := make(chan struct{}, 2)
	go func() {
		muxa, err := l.Accept()
		checkErr(t, err)

		s, err := muxa.OpenStream()
		if err != nil {
			panic(err)
		}

		// Some transports won't open the stream until we write. That's
		// fine.
		s.Write([]byte("foo"))

		time.Sleep(time.Millisecond * 50)

		_, err = s.Write([]byte("bar"))
		if err == nil {
			t.Error("should have failed to write")
		}

		s.Close()
		done <- struct{}{}
	}()

	muxb, err := tb.Dial(context.Background(), l.Multiaddr(), peerA)
	checkErr(t, err)

	go func() {
		str, err := muxb.AcceptStream()
		checkErr(t, err)
		str.Reset()
		done <- struct{}{}
	}()

	<-done
	<-done
}

func SubtestStress1Conn1Stream1Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		ConnNum:   1,
		StreamNum: 1,
		MsgNum:    1,
		MsgMax:    100,
		MsgMin:    100,
	})
}

func SubtestStress1Conn1Stream100Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		ConnNum:   1,
		StreamNum: 1,
		MsgNum:    100,
		MsgMax:    100,
		MsgMin:    100,
	})
}

func SubtestStress1Conn100Stream100Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		ConnNum:   1,
		StreamNum: 100,
		MsgNum:    100,
		MsgMax:    100,
		MsgMin:    100,
	})
}

func SubtestStress50Conn10Stream50Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		ConnNum:   50,
		StreamNum: 10,
		MsgNum:    50,
		MsgMax:    100,
		MsgMin:    100,
	})
}

func SubtestStress1Conn1000Stream10Msg(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		ConnNum:   1,
		StreamNum: 1000,
		MsgNum:    10,
		MsgMax:    100,
		MsgMin:    100,
	})
}

func SubtestStress1Conn100Stream100Msg10MB(t *testing.T, ta, tb transport.Transport, maddr ma.Multiaddr, peerA peer.ID) {
	SubtestStress(t, ta, tb, maddr, peerA, Options{
		ConnNum:   1,
		StreamNum: 100,
		MsgNum:    100,
		MsgMax:    10000,
		MsgMin:    1000,
	})
}
<|endoftext|>


<commit_before>package apns

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"errors"
	"math/rand"
	"strconv"
	"time"
)

// Push commands always start with command value 2.
const pushCommandValue = 2

// Your total notification payload cannot exceed 2 KB.
const MaxPayloadSizeBytes = 2048

// Every push notification gets a pseudo-unique identifier;
// this establishes the upper boundary for it. Apple will return
// this identifier if there is an issue sending your notification.
const IdentifierUbound = 9999

// Constants related to the payload fields and their lengths.
const (
	deviceTokenItemid            = 1
	payloadItemid                = 2
	notificationIdentifierItemid = 3
	expirationDateItemid         = 4
	priorityItemid               = 5
	deviceTokenLength            = 32
	notificationIdentifierLength = 4
	expirationDateLength         = 4
	priorityLength               = 1
)

// Payload contains the notification data for your request.
//
// Alert is an interface here because it supports either a string
// or a dictionary, represented within by an AlertDictionary struct.
type Payload struct {
	Alert            interface{} `json:"alert,omitempty"`
	Badge            int         `json:"badge,omitempty"`
	Sound            string      `json:"sound,omitempty"`
	ContentAvailable int         `json:"content-available,omitempty"`
	Category         string      `json:"category,omitempty"`
}

// NewPayload creates and returns a Payload structure.
func NewPayload() *Payload {
	return new(Payload)
}

// AlertDictionary is a more complex notification payload.
//
// From the APN docs: "Use the ... alert dictionary in general only if you absolutely need to."
// The AlertDictionary is suitable for specific localization needs.
type AlertDictionary struct {
	Body         string   `json:"body,omitempty"`
	ActionLocKey string   `json:"action-loc-key,omitempty"`
	LocKey       string   `json:"loc-key,omitempty"`
	LocArgs      []string `json:"loc-args,omitempty"`
	LaunchImage  string   `json:"launch-image,omitempty"`
}

// NewAlertDictionary creates and returns an AlertDictionary structure.
func NewAlertDictionary() *AlertDictionary {
	return new(AlertDictionary)
}

// PushNotification is the wrapper for the Payload.
// The length fields are computed in ToBytes() and aren't represented here.
type PushNotification struct {
	Identifier  uint32 `json:"identifier"`
	Expiry      uint32 `json:"expiry"`
	DeviceToken string `json:"device_token"`
	Payload     map[string]interface{} `json:"payload"`
	Priority    uint8 `json:"priority"`
}

// NewPushNotification creates and returns a PushNotification structure.
// It also initializes the pseudo-random identifier.
func NewPushNotification() (pn *PushNotification) {
	pn = new(PushNotification)
	pn.Payload = make(map[string]interface{})
	pn.Identifier = rand.New(rand.NewSource(time.Now().UnixNano())).Uint32()
	pn.Priority = 10
	return
}

// AddPayload sets the "aps" Payload section of the request. It also
// has a hack described within to deal with specific zero values.
func (pn *PushNotification) AddPayload(p *Payload) {
	// This deserves some explanation.
	//
	// Setting an exported field of type int to 0
	// triggers the omitempty behavior if you've set it.
	// Since the badge is optional, we should omit it if
	// it's not set. However, we want to include it if the
	// value is 0, so there's a hack in push_notification.go
	// that exploits the fact that Apple treats -1 for a
	// badge value as though it were 0 (i.e. it clears the
	// badge but doesn't stop the notification from going
	// through successfully.)
	//
	// Still a hack though :)
	if p.Badge == 0 {
		p.Badge = -1
	}
	pn.Set("aps", p)
}

// Get returns the value of a Payload key, if it exists.
func (pn *PushNotification) Get(key string) interface{} {
	return pn.Payload[key]
}

// Set defines the value of a Payload key.
func (pn *PushNotification) Set(key string, value interface{}) {
	pn.Payload[key] = value
}

// PayloadJSON returns the current Payload in JSON format.
func (pn *PushNotification) PayloadJSON() ([]byte, error) {
	return json.Marshal(pn.Payload)
}

// PayloadString returns the current Payload in string format.
func (pn *PushNotification) PayloadString() (string, error) {
	j, err := pn.PayloadJSON()
	return string(j), err
}

// ToBytes returns a byte array of the complete PushNotification
// struct. This array is what should be transmitted to the APN Service.
func (pn *PushNotification) ToBytes() ([]byte, error) {
	token, err := hex.DecodeString(pn.DeviceToken)
	if err != nil {
		return nil, err
	}
//	if len(token) != deviceTokenLength {
//		return nil, errors.New("device token has incorrect length")
//	}
	Payload, err := pn.PayloadJSON()
	if err != nil {
		return nil, err
	}
	if len(Payload) > MaxPayloadSizeBytes {
		return nil, errors.New("Payload is larger than the " + strconv.Itoa(MaxPayloadSizeBytes) + " byte limit")
	}

	frameBuffer := new(bytes.Buffer)

//	binary.Write(frameBuffer, binary.BigEndian, uint8(deviceTokenItemid))
//	binary.Write(frameBuffer, binary.BigEndian, uint16(deviceTokenLength))
	binary.Write(frameBuffer, binary.BigEndian, token)
//	binary.Write(frameBuffer, binary.BigEndian, uint8(payloadItemid))
//	binary.Write(frameBuffer, binary.BigEndian, uint16(len(Payload)))
	binary.Write(frameBuffer, binary.BigEndian, Payload)
//	binary.Write(frameBuffer, binary.BigEndian, uint8(notificationIdentifierItemid))
//	binary.Write(frameBuffer, binary.BigEndian, uint16(notificationIdentifierLength))
	binary.Write(frameBuffer, binary.BigEndian, pn.Identifier)
//	binary.Write(frameBuffer, binary.BigEndian, uint8(expirationDateItemid))
//	binary.Write(frameBuffer, binary.BigEndian, uint16(expirationDateLength))
	binary.Write(frameBuffer, binary.BigEndian, pn.Expiry)
//	binary.Write(frameBuffer, binary.BigEndian, uint8(priorityItemid))
//	binary.Write(frameBuffer, binary.BigEndian, uint16(priorityLength))
	binary.Write(frameBuffer, binary.BigEndian, pn.Priority)

	buffer := bytes.NewBuffer([]byte{})
	binary.Write(buffer, binary.BigEndian, uint8(pushCommandValue))
	binary.Write(buffer, binary.BigEndian, uint32(frameBuffer.Len()))
	binary.Write(buffer, binary.BigEndian, frameBuffer.Bytes())
	return buffer.Bytes(), nil
}
<commit_msg>Fixing bapush notification conversion<commit_after>package apns

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"errors"
	"math/rand"
	"strconv"
	"time"
)

// Push commands always start with command value 2.
const pushCommandValue = 2

// Your total notification payload cannot exceed 2 KB.
const MaxPayloadSizeBytes = 2048

// Every push notification gets a pseudo-unique identifier;
// this establishes the upper boundary for it. Apple will return
// this identifier if there is an issue sending your notification.
const IdentifierUbound = 9999

// Constants related to the payload fields and their lengths.
const (
	deviceTokenItemid            = 1
	payloadItemid                = 2
	notificationIdentifierItemid = 3
	expirationDateItemid         = 4
	priorityItemid               = 5
	deviceTokenItemLength            = 32
	notificationIdentifierLength = 4
	expirationDateLength         = 4
	priorityLength               = 1
)

// Payload contains the notification data for your request.
//
// Alert is an interface here because it supports either a string
// or a dictionary, represented within by an AlertDictionary struct.
type Payload struct {
	Alert            interface{} `json:"alert,omitempty"`
	Badge            int         `json:"badge,omitempty"`
	Sound            string      `json:"sound,omitempty"`
	ContentAvailable int         `json:"content-available,omitempty"`
	Category         string      `json:"category,omitempty"`
}

// NewPayload creates and returns a Payload structure.
func NewPayload() *Payload {
	return new(Payload)
}

// AlertDictionary is a more complex notification payload.
//
// From the APN docs: "Use the ... alert dictionary in general only if you absolutely need to."
// The AlertDictionary is suitable for specific localization needs.
type AlertDictionary struct {
	Body         string   `json:"body,omitempty"`
	ActionLocKey string   `json:"action-loc-key,omitempty"`
	LocKey       string   `json:"loc-key,omitempty"`
	LocArgs      []string `json:"loc-args,omitempty"`
	LaunchImage  string   `json:"launch-image,omitempty"`
}

// NewAlertDictionary creates and returns an AlertDictionary structure.
func NewAlertDictionary() *AlertDictionary {
	return new(AlertDictionary)
}

// PushNotification is the wrapper for the Payload.
// The length fields are computed in ToBytes() and aren't represented here.
type PushNotification struct {
	Identifier  uint32 `json:"identifier"`
	Expiry      uint32 `json:"expiry"`
	DeviceToken string `json:"device_token"`
	Payload     map[string]interface{} `json:"payload"`
	Priority    uint8 `json:"priority"`
}

// NewPushNotification creates and returns a PushNotification structure.
// It also initializes the pseudo-random identifier.
func NewPushNotification() (pn *PushNotification) {
	pn = new(PushNotification)
	pn.Payload = make(map[string]interface{})
	pn.Identifier = rand.New(rand.NewSource(time.Now().UnixNano())).Uint32()
	pn.Priority = 10
	return
}

// AddPayload sets the "aps" Payload section of the request. It also
// has a hack described within to deal with specific zero values.
func (pn *PushNotification) AddPayload(p *Payload) {
	// This deserves some explanation.
	//
	// Setting an exported field of type int to 0
	// triggers the omitempty behavior if you've set it.
	// Since the badge is optional, we should omit it if
	// it's not set. However, we want to include it if the
	// value is 0, so there's a hack in push_notification.go
	// that exploits the fact that Apple treats -1 for a
	// badge value as though it were 0 (i.e. it clears the
	// badge but doesn't stop the notification from going
	// through successfully.)
	//
	// Still a hack though :)
	if p.Badge == 0 {
		p.Badge = -1
	}
	pn.Set("aps", p)
}

// Get returns the value of a Payload key, if it exists.
func (pn *PushNotification) Get(key string) interface{} {
	return pn.Payload[key]
}

// Set defines the value of a Payload key.
func (pn *PushNotification) Set(key string, value interface{}) {
	pn.Payload[key] = value
}

// PayloadJSON returns the current Payload in JSON format.
func (pn *PushNotification) PayloadJSON() ([]byte, error) {
	return json.Marshal(pn.Payload)
}

// PayloadString returns the current Payload in string format.
func (pn *PushNotification) PayloadString() (string, error) {
	j, err := pn.PayloadJSON()
	return string(j), err
}

// ToBytes returns a byte array of the complete PushNotification
// struct. This array is what should be transmitted to the APN Service.
func (pn *PushNotification) ToBytes() ([]byte, error) {
	token, err := hex.DecodeString(pn.DeviceToken)
	if err != nil {
		return nil, err
	}
	Payload, err := pn.PayloadJSON()
	if err != nil {
		return nil, err
	}
	if len(Payload) > MaxPayloadSizeBytes {
		return nil, errors.New("Payload is larger than the " + strconv.Itoa(MaxPayloadSizeBytes) + " byte limit")
	}

	frameBuffer := new(bytes.Buffer)

	binary.Write(frameBuffer, binary.BigEndian, uint8(deviceTokenItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(deviceTokenItemLength))
	binary.Write(frameBuffer, binary.BigEndian, token)
	binary.Write(frameBuffer, binary.BigEndian, uint8(payloadItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(len(Payload)))
	binary.Write(frameBuffer, binary.BigEndian, Payload)
	binary.Write(frameBuffer, binary.BigEndian, uint8(notificationIdentifierItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(notificationIdentifierLength))
	binary.Write(frameBuffer, binary.BigEndian, pn.Identifier)
	binary.Write(frameBuffer, binary.BigEndian, uint8(expirationDateItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(expirationDateLength))

	binary.Write(frameBuffer, binary.BigEndian, pn.Expiry)

	binary.Write(frameBuffer, binary.BigEndian, uint8(priorityItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(priorityLength))
	binary.Write(frameBuffer, binary.BigEndian, pn.Priority)

	buffer := bytes.NewBuffer([]byte{})
	binary.Write(buffer, binary.BigEndian, pushCommandValue)
	binary.Write(buffer, binary.BigEndian, frameBuffer.Len())
	binary.Write(buffer, binary.BigEndian, frameBuffer.Bytes())

	return buffer.Bytes(), nil
}
<|endoftext|>


<commit_before>package obj

import (
	"bytes"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudfront/sign"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/aws/aws-sdk-go/service/storagegateway"
	"github.com/cenkalti/backoff"
	"go.pedge.io/lion"
)

type amazonClient struct {
	bucket                 string
	cloudfrontDistribution string
	cloudfrontURLSigner    *sign.URLSigner
	s3                     *s3.S3
	uploader               *s3manager.Uploader
}

func newAmazonClient(bucket string, cloudfrontDistribution string, id string, secret string, token string, region string) (*amazonClient, error) {
	session := session.New(&aws.Config{
		Credentials: credentials.NewStaticCredentials(id, secret, token),
		Region:      aws.String(region),
	})
	var signer *sign.URLSigner
	cloudfrontDistribution = strings.TrimSpace(cloudfrontDistribution)
	if cloudfrontDistribution != "" {
		rawCloudfrontPrivateKey, err := ioutil.ReadFile("/amazon-secret/cloudfrontPrivateKey")
		if err != nil {
			return nil, err
		}

		cloudfrontKeyPairId, err := ioutil.ReadFile("/amazon-secret/cloudfrontKeyPairId")
		if err != nil {
			return nil, err
		}
		block, _ := pem.Decode(bytes.TrimSpace(rawCloudfrontPrivateKey))
		if block == nil || block.Type != "RSA PRIVATE KEY" {
			return nil, fmt.Errorf("block undefined or wrong type: type is (%v) should be (RSA PRIVATE KEY)", block.Type)
		}
		cloudfrontPrivateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
		if err != nil {
			return nil, err
		}
		signer = sign.NewURLSigner(string(cloudfrontKeyPairId), cloudfrontPrivateKey)
		lion.Infof("Using cloudfront security credentials - keypair ID (%v) - to sign cloudfront URLs", string(cloudfrontKeyPairId))
	}
	return &amazonClient{
		bucket:                 bucket,
		cloudfrontDistribution: cloudfrontDistribution,
		cloudfrontURLSigner:    signer,
		s3:                     s3.New(session),
		uploader:               s3manager.NewUploader(session),
	}, nil
}

func (c *amazonClient) Writer(name string) (io.WriteCloser, error) {
	return newBackoffWriteCloser(c, newWriter(c, name)), nil
}

func (c *amazonClient) Walk(name string, fn func(name string) error) error {
	var fnErr error
	if err := c.s3.ListObjectsPages(
		&s3.ListObjectsInput{
			Bucket: aws.String(c.bucket),
			Prefix: aws.String(name),
		},
		func(listObjectsOutput *s3.ListObjectsOutput, lastPage bool) bool {
			for _, object := range listObjectsOutput.Contents {
				if err := fn(*object.Key); err != nil {
					fnErr = err
					return false
				}
			}
			return true
		},
	); err != nil {
		return err
	}
	return fnErr
}

func (c *amazonClient) Reader(name string, offset uint64, size uint64) (io.ReadCloser, error) {
	byteRange := byteRange(offset, size)
	if byteRange != "" {
		byteRange = fmt.Sprintf("bytes=%s", byteRange)
	}
	var reader io.ReadCloser
	if c.cloudfrontDistribution != "" {
		var resp *http.Response
		var connErr error
		url := fmt.Sprintf("http://%v.cloudfront.net/%v", c.cloudfrontDistribution, name)

		if c.cloudfrontURLSigner != nil {
			signedURL, err := c.cloudfrontURLSigner.Sign(url, time.Now().Add(1*time.Hour))
			if err != nil {
				return nil, err
			}
			url = strings.TrimSpace(signedURL)
		}
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			return nil, err
		}
		req.Header.Add("Range", byteRange)

		backoff.RetryNotify(func() error {
			resp, connErr = http.DefaultClient.Do(req)
			if connErr != nil && isNetRetryable(connErr) {
				return connErr
			}
			return nil
		}, backoff.NewExponentialBackOff(), func(err error, d time.Duration) {
			lion.Infof("Error connecting to (%v); retrying in %s: %#v", url, d, err)
		})
		if connErr != nil {
			return nil, connErr
		}
		if resp.StatusCode >= 300 {
			// Cloudfront returns 200s, and 206s as success codes
			return nil, fmt.Errorf("cloudfront returned HTTP error code %v for url %v", resp.Status, url)
		}
		reader = resp.Body
	} else {
		getObjectOutput, err := c.s3.GetObject(&s3.GetObjectInput{
			Bucket: aws.String(c.bucket),
			Key:    aws.String(name),
			Range:  aws.String(byteRange),
		})
		if err != nil {
			return nil, err
		}
		reader = getObjectOutput.Body
	}
	return newBackoffReadCloser(c, reader), nil
}

func (c *amazonClient) Delete(name string) error {
	_, err := c.s3.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err
}

func (c *amazonClient) Exists(name string) bool {
	_, err := c.s3.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err == nil
}

func (c *amazonClient) isRetryable(err error) (retVal bool) {
	if strings.Contains(err.Error(), "unexpected EOF") {
		return true
	}

	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	for _, c := range []string{
		storagegateway.ErrorCodeServiceUnavailable,
		storagegateway.ErrorCodeInternalError,
		storagegateway.ErrorCodeGatewayInternalError,
	} {
		if c == awsErr.Code() {
			return true
		}
	}
	return false
}

func (c *amazonClient) IsIgnorable(err error) bool {
	return false
}

func (c *amazonClient) IsNotExist(err error) bool {
	if c.cloudfrontDistribution != "" {
		// cloudfront returns forbidden error for nonexisting data
		if strings.Contains(err.Error(), "error code 403") {
			return true
		}
	}
	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	if awsErr.Code() == "NoSuchKey" {
		return true
	}
	return false
}

type amazonWriter struct {
	errChan chan error
	pipe    *io.PipeWriter
}

func newWriter(client *amazonClient, name string) *amazonWriter {
	reader, writer := io.Pipe()
	w := &amazonWriter{
		errChan: make(chan error),
		pipe:    writer,
	}
	go func() {
		_, err := client.uploader.Upload(&s3manager.UploadInput{
			Body:            reader,
			Bucket:          aws.String(client.bucket),
			Key:             aws.String(name),
			ContentEncoding: aws.String("application/octet-stream"),
		})
		w.errChan <- err
	}()
	return w
}

func (w *amazonWriter) Write(p []byte) (int, error) {
	return w.pipe.Write(p)
}

func (w *amazonWriter) Close() error {
	if err := w.pipe.Close(); err != nil {
		return err
	}
	return <-w.errChan
}
<commit_msg>Make linter happy<commit_after>package obj

import (
	"bytes"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudfront/sign"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/aws/aws-sdk-go/service/storagegateway"
	"github.com/cenkalti/backoff"
	"go.pedge.io/lion"
)

type amazonClient struct {
	bucket                 string
	cloudfrontDistribution string
	cloudfrontURLSigner    *sign.URLSigner
	s3                     *s3.S3
	uploader               *s3manager.Uploader
}

func newAmazonClient(bucket string, cloudfrontDistribution string, id string, secret string, token string, region string) (*amazonClient, error) {
	session := session.New(&aws.Config{
		Credentials: credentials.NewStaticCredentials(id, secret, token),
		Region:      aws.String(region),
	})
	var signer *sign.URLSigner
	cloudfrontDistribution = strings.TrimSpace(cloudfrontDistribution)
	if cloudfrontDistribution != "" {
		rawCloudfrontPrivateKey, err := ioutil.ReadFile("/amazon-secret/cloudfrontPrivateKey")
		if err != nil {
			return nil, err
		}

		cloudfrontKeyPairID, err := ioutil.ReadFile("/amazon-secret/cloudfrontKeyPairId")
		if err != nil {
			return nil, err
		}
		block, _ := pem.Decode(bytes.TrimSpace(rawCloudfrontPrivateKey))
		if block == nil || block.Type != "RSA PRIVATE KEY" {
			return nil, fmt.Errorf("block undefined or wrong type: type is (%v) should be (RSA PRIVATE KEY)", block.Type)
		}
		cloudfrontPrivateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
		if err != nil {
			return nil, err
		}
		signer = sign.NewURLSigner(string(cloudfrontKeyPairID), cloudfrontPrivateKey)
		lion.Infof("Using cloudfront security credentials - keypair ID (%v) - to sign cloudfront URLs", string(cloudfrontKeyPairID))
	}
	return &amazonClient{
		bucket:                 bucket,
		cloudfrontDistribution: cloudfrontDistribution,
		cloudfrontURLSigner:    signer,
		s3:                     s3.New(session),
		uploader:               s3manager.NewUploader(session),
	}, nil
}

func (c *amazonClient) Writer(name string) (io.WriteCloser, error) {
	return newBackoffWriteCloser(c, newWriter(c, name)), nil
}

func (c *amazonClient) Walk(name string, fn func(name string) error) error {
	var fnErr error
	if err := c.s3.ListObjectsPages(
		&s3.ListObjectsInput{
			Bucket: aws.String(c.bucket),
			Prefix: aws.String(name),
		},
		func(listObjectsOutput *s3.ListObjectsOutput, lastPage bool) bool {
			for _, object := range listObjectsOutput.Contents {
				if err := fn(*object.Key); err != nil {
					fnErr = err
					return false
				}
			}
			return true
		},
	); err != nil {
		return err
	}
	return fnErr
}

func (c *amazonClient) Reader(name string, offset uint64, size uint64) (io.ReadCloser, error) {
	byteRange := byteRange(offset, size)
	if byteRange != "" {
		byteRange = fmt.Sprintf("bytes=%s", byteRange)
	}
	var reader io.ReadCloser
	if c.cloudfrontDistribution != "" {
		var resp *http.Response
		var connErr error
		url := fmt.Sprintf("http://%v.cloudfront.net/%v", c.cloudfrontDistribution, name)

		if c.cloudfrontURLSigner != nil {
			signedURL, err := c.cloudfrontURLSigner.Sign(url, time.Now().Add(1*time.Hour))
			if err != nil {
				return nil, err
			}
			url = strings.TrimSpace(signedURL)
		}
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			return nil, err
		}
		req.Header.Add("Range", byteRange)

		backoff.RetryNotify(func() error {
			resp, connErr = http.DefaultClient.Do(req)
			if connErr != nil && isNetRetryable(connErr) {
				return connErr
			}
			return nil
		}, backoff.NewExponentialBackOff(), func(err error, d time.Duration) {
			lion.Infof("Error connecting to (%v); retrying in %s: %#v", url, d, err)
		})
		if connErr != nil {
			return nil, connErr
		}
		if resp.StatusCode >= 300 {
			// Cloudfront returns 200s, and 206s as success codes
			return nil, fmt.Errorf("cloudfront returned HTTP error code %v for url %v", resp.Status, url)
		}
		reader = resp.Body
	} else {
		getObjectOutput, err := c.s3.GetObject(&s3.GetObjectInput{
			Bucket: aws.String(c.bucket),
			Key:    aws.String(name),
			Range:  aws.String(byteRange),
		})
		if err != nil {
			return nil, err
		}
		reader = getObjectOutput.Body
	}
	return newBackoffReadCloser(c, reader), nil
}

func (c *amazonClient) Delete(name string) error {
	_, err := c.s3.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err
}

func (c *amazonClient) Exists(name string) bool {
	_, err := c.s3.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err == nil
}

func (c *amazonClient) isRetryable(err error) (retVal bool) {
	if strings.Contains(err.Error(), "unexpected EOF") {
		return true
	}

	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	for _, c := range []string{
		storagegateway.ErrorCodeServiceUnavailable,
		storagegateway.ErrorCodeInternalError,
		storagegateway.ErrorCodeGatewayInternalError,
	} {
		if c == awsErr.Code() {
			return true
		}
	}
	return false
}

func (c *amazonClient) IsIgnorable(err error) bool {
	return false
}

func (c *amazonClient) IsNotExist(err error) bool {
	if c.cloudfrontDistribution != "" {
		// cloudfront returns forbidden error for nonexisting data
		if strings.Contains(err.Error(), "error code 403") {
			return true
		}
	}
	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	if awsErr.Code() == "NoSuchKey" {
		return true
	}
	return false
}

type amazonWriter struct {
	errChan chan error
	pipe    *io.PipeWriter
}

func newWriter(client *amazonClient, name string) *amazonWriter {
	reader, writer := io.Pipe()
	w := &amazonWriter{
		errChan: make(chan error),
		pipe:    writer,
	}
	go func() {
		_, err := client.uploader.Upload(&s3manager.UploadInput{
			Body:            reader,
			Bucket:          aws.String(client.bucket),
			Key:             aws.String(name),
			ContentEncoding: aws.String("application/octet-stream"),
		})
		w.errChan <- err
	}()
	return w
}

func (w *amazonWriter) Write(p []byte) (int, error) {
	return w.pipe.Write(p)
}

func (w *amazonWriter) Close() error {
	if err := w.pipe.Close(); err != nil {
		return err
	}
	return <-w.errChan
}
<|endoftext|>


<commit_before>package worker

import (
	"bufio"
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"syscall"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"

	"github.com/gogo/protobuf/jsonpb"
	"github.com/gogo/protobuf/proto"
	"github.com/gogo/protobuf/types"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/limit"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/hashtree"
	filesync "github.com/pachyderm/pachyderm/src/server/pkg/sync"
)

const (
	// The maximum number of concurrent download/upload operations
	concurrency = 100
	maxLogItems = 10
)

// Input is a generic input object that can either be a pipeline input or
// a job input.  It only defines the attributes that the worker cares about.
type Input struct {
	Name string
	Lazy bool
}

// APIServer implements the worker API
type APIServer struct {
	sync.Mutex
	pachClient *client.APIClient

	// Information needed to process input data and upload output
	transform *pps.Transform
	inputs    []*Input

	// Information attached to log lines
	logMsgTemplate pps.LogMessage

	// The k8s pod name of this worker
	workerName string
}

type taggedLogger struct {
	template  pps.LogMessage
	stderrLog log.Logger
	marshaler *jsonpb.Marshaler
}

func (a *APIServer) getTaggedLogger(req *ProcessRequest) *taggedLogger {
	result := &taggedLogger{
		template:  a.logMsgTemplate, // Copy struct
		stderrLog: log.Logger{},
		marshaler: &jsonpb.Marshaler{},
	}
	result.stderrLog.SetOutput(os.Stderr)
	result.stderrLog.SetFlags(log.LstdFlags | log.Llongfile) // Log file/line

	// Add Job ID to log metadata
	result.template.JobID = req.JobID

	// Add inputs' details to log metadata, so we can find these logs later
	result.template.Data = make([]*pps.LogMessage_Datum, 0, len(req.Data))
	for i, d := range req.Data {
		result.template.Data = append(result.template.Data, new(pps.LogMessage_Datum))
		result.template.Data[i].Path = d.File.Path
		result.template.Data[i].Hash = d.Hash
	}
	return result
}

// Logf logs the line Sprintf(formatString, args...), but formatted as a json
// message and annotated with all of the metadata stored in 'loginfo'.
//
// Note: this is not thread-safe, as it modifies fields of 'logger.template'
func (logger *taggedLogger) Logf(formatString string, args ...interface{}) {
	logger.template.Message = fmt.Sprintf(formatString, args...)
	if ts, err := types.TimestampProto(time.Now()); err == nil {
		logger.template.Ts = ts
	} else {
		logger.stderrLog.Printf("could not generate logging timestamp: %s\n", err)
		return
	}
	bytes, err := logger.marshaler.MarshalToString(&logger.template)
	if err != nil {
		logger.stderrLog.Printf("could not marshal %v for logging: %s\n", &logger.template, err)
		return
	}
	fmt.Printf("%s\n", bytes)
}

func (logger *taggedLogger) userLogger() *taggedLogger {
	result := &taggedLogger{
		template:  logger.template, // Copy struct
		stderrLog: log.Logger{},
		marshaler: &jsonpb.Marshaler{},
	}
	result.template.User = true
	return result
}

// NewPipelineAPIServer creates an APIServer for a given pipeline
func NewPipelineAPIServer(pachClient *client.APIClient, pipelineInfo *pps.PipelineInfo, workerName string) *APIServer {
	server := &APIServer{
		Mutex:      sync.Mutex{},
		pachClient: pachClient,
		transform:  pipelineInfo.Transform,
		inputs:     make([]*Input, 0, len(pipelineInfo.Inputs)),
		logMsgTemplate: pps.LogMessage{
			PipelineName: pipelineInfo.Pipeline.Name,
			PipelineID:   pipelineInfo.ID,
			WorkerID:     os.Getenv(client.PPSPodNameEnv),
		},
		workerName: workerName,
	}
	for _, input := range pipelineInfo.Inputs {
		server.inputs = append(server.inputs, &Input{
			Name: input.Name,
			Lazy: input.Lazy,
		})
	}
	return server
}

// NewJobAPIServer creates an APIServer for a given pipeline
func NewJobAPIServer(pachClient *client.APIClient, jobInfo *pps.JobInfo, workerName string) *APIServer {
	server := &APIServer{
		Mutex:          sync.Mutex{},
		pachClient:     pachClient,
		transform:      jobInfo.Transform,
		inputs:         make([]*Input, 0, len(jobInfo.Inputs)),
		logMsgTemplate: pps.LogMessage{},
		workerName:     workerName,
	}
	for _, input := range jobInfo.Inputs {
		server.inputs = append(server.inputs, &Input{
			Name: input.Name,
			Lazy: input.Lazy,
		})
	}
	return server
}

func (a *APIServer) downloadData(data []*pfs.FileInfo, puller *filesync.Puller) error {
	for i, datum := range data {
		input := a.inputs[i]
		if err := puller.Pull(a.pachClient, filepath.Join(client.PPSInputPrefix, input.Name),
			datum.File.Commit.Repo.Name, datum.File.Commit.ID, datum.File.Path, input.Lazy, concurrency); err != nil {
			return err
		}
	}
	return nil
}

// Run user code and return the combined output of stdout and stderr.
func (a *APIServer) runUserCode(ctx context.Context, logger *taggedLogger) (string, error) {
	// Run user code
	transform := a.transform
	cmd := exec.Command(transform.Cmd[0], transform.Cmd[1:]...)
	cmd.Stdin = strings.NewReader(strings.Join(transform.Stdin, "\n") + "\n")
	var userlog bytes.Buffer
	cmd.Stdout = &userlog
	cmd.Stderr = &userlog
	err := cmd.Run()

	// Log output from user cmd, line-by-line, whether or not cmd errored
	logger.Logf("running user code")
	logscanner := bufio.NewScanner(&userlog)
	for logscanner.Scan() {
		logger.Logf(logscanner.Text())
	}

	// Return result
	if err == nil {
		return userlog.String(), nil
	}
	// (if err is an acceptable return code, don't return err)
	if exiterr, ok := err.(*exec.ExitError); ok {
		if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
			for _, returnCode := range transform.AcceptReturnCode {
				if int(returnCode) == status.ExitStatus() {
					return userlog.String(), nil
				}
			}
		}
	}
	return userlog.String(), err

}

func (a *APIServer) uploadOutput(ctx context.Context, tag string) error {
	// hashtree is not thread-safe--guard with 'lock'
	var lock sync.Mutex
	tree := hashtree.NewHashTree()

	// Upload all files in output directory
	var g errgroup.Group
	limiter := limit.New(concurrency)
	if err := filepath.Walk(client.PPSOutputPath, func(path string, info os.FileInfo, err error) error {
		g.Go(func() (retErr error) {
			limiter.Acquire()
			defer limiter.Release()
			if path == client.PPSOutputPath {
				return nil
			}

			relPath, err := filepath.Rel(client.PPSOutputPath, path)
			if err != nil {
				return err
			}

			// Put directory. Even if the directory is empty, that may be useful to
			// users
			// TODO(msteffen) write a test pipeline that outputs an empty directory and
			// make sure it's preserved
			if info.IsDir() {
				lock.Lock()
				defer lock.Unlock()
				tree.PutDir(relPath)
				return nil
			}

			f, err := os.Open(path)
			if err != nil {
				return err
			}
			defer func() {
				if err := f.Close(); err != nil && retErr == nil {
					retErr = err
				}
			}()

			object, size, err := a.pachClient.PutObject(f)
			if err != nil {
				return err
			}
			lock.Lock()
			defer lock.Unlock()
			return tree.PutFile(relPath, []*pfs.Object{object}, size)
		})
		return nil
	}); err != nil {
		return err
	}

	if err := g.Wait(); err != nil {
		return err
	}

	finTree, err := tree.Finish()
	if err != nil {
		return err
	}

	treeBytes, err := hashtree.Serialize(finTree)
	if err != nil {
		return err
	}

	if _, _, err := a.pachClient.PutObject(bytes.NewReader(treeBytes), tag); err != nil {
		return err
	}

	return nil
}

// cleanUpData removes everything under /pfs
//
// The reason we don't want to just os.RemoveAll(/pfs) is that we don't
// want to remove /pfs itself, since it's a symlink to the hostpath volume.
// We also don't want to remove /pfs and re-create the symlink, because for
// some reason that results in extremely pool performance.
//
// Most of the code is copied from os.RemoveAll().
func (a *APIServer) cleanUpData() error {
	path := filepath.Join(client.PPSHostPath, a.workerName)
	// Otherwise, is this a directory we need to recurse into?
	dir, serr := os.Lstat(path)
	if serr != nil {
		if serr, ok := serr.(*os.PathError); ok && (os.IsNotExist(serr.Err) || serr.Err == syscall.ENOTDIR) {
			return nil
		}
		return serr
	}
	if !dir.IsDir() {
		// Not a directory; return the error from Remove.
		return fmt.Errorf("%s is not a directory", path)
	}

	// Directory.
	fd, err := os.Open(path)
	if err != nil {
		if os.IsNotExist(err) {
			// Race. It was deleted between the Lstat and Open.
			// Return nil per RemoveAll's docs.
			return nil
		}
		return err
	}

	// Remove contents & return first error.
	err = nil
	for {
		names, err1 := fd.Readdirnames(100)
		for _, name := range names {
			err1 := os.RemoveAll(path + string(os.PathSeparator) + name)
			if err == nil {
				err = err1
			}
		}
		if err1 == io.EOF {
			break
		}
		// If Readdirnames returned an error, use it.
		if err == nil {
			err = err1
		}
		if len(names) == 0 {
			break
		}
	}

	// Close directory, because windows won't remove opened directory.
	fd.Close()
	return err
}

// HashDatum computes and returns the hash of a datum + pipeline.
func (a *APIServer) HashDatum(data []*pfs.FileInfo) (string, error) {
	hash := sha256.New()
	sort.Slice(data, func(i, j int) bool {
		return data[i].File.Path < data[j].File.Path
	})
	for i, fileInfo := range data {
		if _, err := hash.Write([]byte(a.inputs[i].Name)); err != nil {
			return "", err
		}
		if _, err := hash.Write([]byte(fileInfo.File.Path)); err != nil {
			return "", err
		}
		if _, err := hash.Write(fileInfo.Hash); err != nil {
			return "", err
		}
	}
	bytes, err := proto.Marshal(a.transform)
	if err != nil {
		return "", err
	}
	if _, err := hash.Write(bytes); err != nil {
		return "", err
	}
	return hex.EncodeToString(hash.Sum(nil)), nil
}

// Process processes a datum.
func (a *APIServer) Process(ctx context.Context, req *ProcessRequest) (resp *ProcessResponse, retErr error) {
	// We cannot run more than one user process at once; otherwise they'd be
	// writing to the same output directory. Acquire lock to make sure only one
	// user process runs at a time.
	a.Lock()
	defer a.Unlock()
	logger := a.getTaggedLogger(req)
	logger.Logf("Received request")

	// Hash inputs and check if output is in s3 already. Note: ppsserver sorts
	// inputs by input name for both jobs and pipelines, so this hash is stable
	// even if a.Inputs are reordered by the user
	tag, err := a.HashDatum(req.Data)
	if err != nil {
		return nil, err
	}
	if _, err := a.pachClient.InspectTag(ctx, &pfs.Tag{tag}); err == nil {
		// We've already computed the output for these inputs. Return immediately
		logger.Logf("skipping input, as it's already been processed")
		return &ProcessResponse{
			Tag: &pfs.Tag{tag},
		}, nil
	}

	// Download input data
	logger.Logf("input has not been processed, downloading data")
	puller := filesync.NewPuller()
	if err := a.downloadData(req.Data, puller); err != nil {
		return nil, err
	}
	defer func() {
		if err := puller.CleanUp(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	defer func() {
		if err := a.cleanUpData(); retErr == nil && err != nil {
			retErr = err
		}
	}()

	// Create output directory (currently /pfs/out) and run user code
	if err := os.MkdirAll(client.PPSOutputPath, 0666); err != nil {
		return nil, err
	}
	logger.Logf("beginning to process user input")
	userlog, err := a.runUserCode(ctx, logger)
	logger.Logf("finished processing user input")
	if err != nil {
		return &ProcessResponse{
			Log: userlog,
		}, nil
	}
	if err := a.uploadOutput(ctx, tag); err != nil {
		return nil, err
	}
	return &ProcessResponse{
		Tag: &pfs.Tag{tag},
	}, nil
}
<commit_msg>Use userLogger in correct places.<commit_after>package worker

import (
	"bufio"
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"syscall"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"

	"github.com/gogo/protobuf/jsonpb"
	"github.com/gogo/protobuf/proto"
	"github.com/gogo/protobuf/types"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/limit"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/hashtree"
	filesync "github.com/pachyderm/pachyderm/src/server/pkg/sync"
)

const (
	// The maximum number of concurrent download/upload operations
	concurrency = 100
	maxLogItems = 10
)

// Input is a generic input object that can either be a pipeline input or
// a job input.  It only defines the attributes that the worker cares about.
type Input struct {
	Name string
	Lazy bool
}

// APIServer implements the worker API
type APIServer struct {
	sync.Mutex
	pachClient *client.APIClient

	// Information needed to process input data and upload output
	transform *pps.Transform
	inputs    []*Input

	// Information attached to log lines
	logMsgTemplate pps.LogMessage

	// The k8s pod name of this worker
	workerName string
}

type taggedLogger struct {
	template  pps.LogMessage
	stderrLog log.Logger
	marshaler *jsonpb.Marshaler
}

func (a *APIServer) getTaggedLogger(req *ProcessRequest) *taggedLogger {
	result := &taggedLogger{
		template:  a.logMsgTemplate, // Copy struct
		stderrLog: log.Logger{},
		marshaler: &jsonpb.Marshaler{},
	}
	result.stderrLog.SetOutput(os.Stderr)
	result.stderrLog.SetFlags(log.LstdFlags | log.Llongfile) // Log file/line

	// Add Job ID to log metadata
	result.template.JobID = req.JobID

	// Add inputs' details to log metadata, so we can find these logs later
	result.template.Data = make([]*pps.LogMessage_Datum, 0, len(req.Data))
	for i, d := range req.Data {
		result.template.Data = append(result.template.Data, new(pps.LogMessage_Datum))
		result.template.Data[i].Path = d.File.Path
		result.template.Data[i].Hash = d.Hash
	}
	return result
}

// Logf logs the line Sprintf(formatString, args...), but formatted as a json
// message and annotated with all of the metadata stored in 'loginfo'.
//
// Note: this is not thread-safe, as it modifies fields of 'logger.template'
func (logger *taggedLogger) Logf(formatString string, args ...interface{}) {
	logger.template.Message = fmt.Sprintf(formatString, args...)
	if ts, err := types.TimestampProto(time.Now()); err == nil {
		logger.template.Ts = ts
	} else {
		logger.stderrLog.Printf("could not generate logging timestamp: %s\n", err)
		return
	}
	bytes, err := logger.marshaler.MarshalToString(&logger.template)
	if err != nil {
		logger.stderrLog.Printf("could not marshal %v for logging: %s\n", &logger.template, err)
		return
	}
	fmt.Printf("%s\n", bytes)
}

func (logger *taggedLogger) userLogger() *taggedLogger {
	result := &taggedLogger{
		template:  logger.template, // Copy struct
		stderrLog: log.Logger{},
		marshaler: &jsonpb.Marshaler{},
	}
	result.template.User = true
	return result
}

// NewPipelineAPIServer creates an APIServer for a given pipeline
func NewPipelineAPIServer(pachClient *client.APIClient, pipelineInfo *pps.PipelineInfo, workerName string) *APIServer {
	server := &APIServer{
		Mutex:      sync.Mutex{},
		pachClient: pachClient,
		transform:  pipelineInfo.Transform,
		inputs:     make([]*Input, 0, len(pipelineInfo.Inputs)),
		logMsgTemplate: pps.LogMessage{
			PipelineName: pipelineInfo.Pipeline.Name,
			PipelineID:   pipelineInfo.ID,
			WorkerID:     os.Getenv(client.PPSPodNameEnv),
		},
		workerName: workerName,
	}
	for _, input := range pipelineInfo.Inputs {
		server.inputs = append(server.inputs, &Input{
			Name: input.Name,
			Lazy: input.Lazy,
		})
	}
	return server
}

// NewJobAPIServer creates an APIServer for a given pipeline
func NewJobAPIServer(pachClient *client.APIClient, jobInfo *pps.JobInfo, workerName string) *APIServer {
	server := &APIServer{
		Mutex:          sync.Mutex{},
		pachClient:     pachClient,
		transform:      jobInfo.Transform,
		inputs:         make([]*Input, 0, len(jobInfo.Inputs)),
		logMsgTemplate: pps.LogMessage{},
		workerName:     workerName,
	}
	for _, input := range jobInfo.Inputs {
		server.inputs = append(server.inputs, &Input{
			Name: input.Name,
			Lazy: input.Lazy,
		})
	}
	return server
}

func (a *APIServer) downloadData(data []*pfs.FileInfo, puller *filesync.Puller) error {
	for i, datum := range data {
		input := a.inputs[i]
		if err := puller.Pull(a.pachClient, filepath.Join(client.PPSInputPrefix, input.Name),
			datum.File.Commit.Repo.Name, datum.File.Commit.ID, datum.File.Path, input.Lazy, concurrency); err != nil {
			return err
		}
	}
	return nil
}

// Run user code and return the combined output of stdout and stderr.
func (a *APIServer) runUserCode(ctx context.Context, logger *taggedLogger) (string, error) {
	// Run user code
	transform := a.transform
	cmd := exec.Command(transform.Cmd[0], transform.Cmd[1:]...)
	cmd.Stdin = strings.NewReader(strings.Join(transform.Stdin, "\n") + "\n")
	var userlog bytes.Buffer
	cmd.Stdout = &userlog
	cmd.Stderr = &userlog
	err := cmd.Run()

	// Log output from user cmd, line-by-line, whether or not cmd errored
	logger.Logf("running user code")
	logscanner := bufio.NewScanner(&userlog)
	userLogger := logger.userLogger()
	for logscanner.Scan() {
		userLogger.Logf(logscanner.Text())
	}

	// Return result
	if err == nil {
		return userlog.String(), nil
	}
	// (if err is an acceptable return code, don't return err)
	if exiterr, ok := err.(*exec.ExitError); ok {
		if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
			for _, returnCode := range transform.AcceptReturnCode {
				if int(returnCode) == status.ExitStatus() {
					return userlog.String(), nil
				}
			}
		}
	}
	return userlog.String(), err

}

func (a *APIServer) uploadOutput(ctx context.Context, tag string) error {
	// hashtree is not thread-safe--guard with 'lock'
	var lock sync.Mutex
	tree := hashtree.NewHashTree()

	// Upload all files in output directory
	var g errgroup.Group
	limiter := limit.New(concurrency)
	if err := filepath.Walk(client.PPSOutputPath, func(path string, info os.FileInfo, err error) error {
		g.Go(func() (retErr error) {
			limiter.Acquire()
			defer limiter.Release()
			if path == client.PPSOutputPath {
				return nil
			}

			relPath, err := filepath.Rel(client.PPSOutputPath, path)
			if err != nil {
				return err
			}

			// Put directory. Even if the directory is empty, that may be useful to
			// users
			// TODO(msteffen) write a test pipeline that outputs an empty directory and
			// make sure it's preserved
			if info.IsDir() {
				lock.Lock()
				defer lock.Unlock()
				tree.PutDir(relPath)
				return nil
			}

			f, err := os.Open(path)
			if err != nil {
				return err
			}
			defer func() {
				if err := f.Close(); err != nil && retErr == nil {
					retErr = err
				}
			}()

			object, size, err := a.pachClient.PutObject(f)
			if err != nil {
				return err
			}
			lock.Lock()
			defer lock.Unlock()
			return tree.PutFile(relPath, []*pfs.Object{object}, size)
		})
		return nil
	}); err != nil {
		return err
	}

	if err := g.Wait(); err != nil {
		return err
	}

	finTree, err := tree.Finish()
	if err != nil {
		return err
	}

	treeBytes, err := hashtree.Serialize(finTree)
	if err != nil {
		return err
	}

	if _, _, err := a.pachClient.PutObject(bytes.NewReader(treeBytes), tag); err != nil {
		return err
	}

	return nil
}

// cleanUpData removes everything under /pfs
//
// The reason we don't want to just os.RemoveAll(/pfs) is that we don't
// want to remove /pfs itself, since it's a symlink to the hostpath volume.
// We also don't want to remove /pfs and re-create the symlink, because for
// some reason that results in extremely pool performance.
//
// Most of the code is copied from os.RemoveAll().
func (a *APIServer) cleanUpData() error {
	path := filepath.Join(client.PPSHostPath, a.workerName)
	// Otherwise, is this a directory we need to recurse into?
	dir, serr := os.Lstat(path)
	if serr != nil {
		if serr, ok := serr.(*os.PathError); ok && (os.IsNotExist(serr.Err) || serr.Err == syscall.ENOTDIR) {
			return nil
		}
		return serr
	}
	if !dir.IsDir() {
		// Not a directory; return the error from Remove.
		return fmt.Errorf("%s is not a directory", path)
	}

	// Directory.
	fd, err := os.Open(path)
	if err != nil {
		if os.IsNotExist(err) {
			// Race. It was deleted between the Lstat and Open.
			// Return nil per RemoveAll's docs.
			return nil
		}
		return err
	}

	// Remove contents & return first error.
	err = nil
	for {
		names, err1 := fd.Readdirnames(100)
		for _, name := range names {
			err1 := os.RemoveAll(path + string(os.PathSeparator) + name)
			if err == nil {
				err = err1
			}
		}
		if err1 == io.EOF {
			break
		}
		// If Readdirnames returned an error, use it.
		if err == nil {
			err = err1
		}
		if len(names) == 0 {
			break
		}
	}

	// Close directory, because windows won't remove opened directory.
	fd.Close()
	return err
}

// HashDatum computes and returns the hash of a datum + pipeline.
func (a *APIServer) HashDatum(data []*pfs.FileInfo) (string, error) {
	hash := sha256.New()
	sort.Slice(data, func(i, j int) bool {
		return data[i].File.Path < data[j].File.Path
	})
	for i, fileInfo := range data {
		if _, err := hash.Write([]byte(a.inputs[i].Name)); err != nil {
			return "", err
		}
		if _, err := hash.Write([]byte(fileInfo.File.Path)); err != nil {
			return "", err
		}
		if _, err := hash.Write(fileInfo.Hash); err != nil {
			return "", err
		}
	}
	bytes, err := proto.Marshal(a.transform)
	if err != nil {
		return "", err
	}
	if _, err := hash.Write(bytes); err != nil {
		return "", err
	}
	return hex.EncodeToString(hash.Sum(nil)), nil
}

// Process processes a datum.
func (a *APIServer) Process(ctx context.Context, req *ProcessRequest) (resp *ProcessResponse, retErr error) {
	// We cannot run more than one user process at once; otherwise they'd be
	// writing to the same output directory. Acquire lock to make sure only one
	// user process runs at a time.
	a.Lock()
	defer a.Unlock()
	logger := a.getTaggedLogger(req)
	logger.Logf("Received request")

	// Hash inputs and check if output is in s3 already. Note: ppsserver sorts
	// inputs by input name for both jobs and pipelines, so this hash is stable
	// even if a.Inputs are reordered by the user
	tag, err := a.HashDatum(req.Data)
	if err != nil {
		return nil, err
	}
	if _, err := a.pachClient.InspectTag(ctx, &pfs.Tag{tag}); err == nil {
		// We've already computed the output for these inputs. Return immediately
		logger.Logf("skipping input, as it's already been processed")
		return &ProcessResponse{
			Tag: &pfs.Tag{tag},
		}, nil
	}

	// Download input data
	logger.Logf("input has not been processed, downloading data")
	puller := filesync.NewPuller()
	if err := a.downloadData(req.Data, puller); err != nil {
		return nil, err
	}
	defer func() {
		if err := puller.CleanUp(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	defer func() {
		if err := a.cleanUpData(); retErr == nil && err != nil {
			retErr = err
		}
	}()

	// Create output directory (currently /pfs/out) and run user code
	if err := os.MkdirAll(client.PPSOutputPath, 0666); err != nil {
		return nil, err
	}
	logger.Logf("beginning to process user input")
	userlog, err := a.runUserCode(ctx, logger)
	logger.Logf("finished processing user input")
	if err != nil {
		return &ProcessResponse{
			Log: userlog,
		}, nil
	}
	if err := a.uploadOutput(ctx, tag); err != nil {
		return nil, err
	}
	return &ProcessResponse{
		Tag: &pfs.Tag{tag},
	}, nil
}
<|endoftext|>


<commit_before>package linkedin

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strings"
)

// Flow:
// 1. Redirect user to RedirectUri, they confirm linkedin. This process generates an AuthToken
// 2. The AuthToken is exchanged for an AccessToken. Posting to _LI_VALIDATE returns the AccessToken
// 3. Requests for profile information must contain AccessToken paramater

const (
	_LI_AUTH_URL     = "https://www.linkedin.com/uas/oauth2/authorization?"
	_LI_VALIDATE_URL = "https://www.linkedin.com/uas/oauth2/accessToken?grant_type=authorization_code&code=%s&redirect_uri=%s&client_id=%s&client_secret=%s"
	_LI_PROFILE_URL  = "//api.linkedin.com/v1/people/~"
)

// Contains data for the linked in api
type LinkedIn struct {
	ApiKey    string
	ApiSecret string
	Redirect  string
	State     string
}

// Generate a an AuthUri - the user must be redirected here and accept
// the linkedin request. This process produces an AuthToken which must be
// sent to Validate quickly (under a minute) before linkedin expires it
func (l *LinkedIn) AuthUri(scope ...string) string {

	v := url.Values{}
	v.Add("response_type", "code")
	v.Add("client_id", l.ApiKey)
	v.Add("state", l.State)

	if len(scope) > 0 {
		v.Add("scope", strings.Join(scope, " "))
	}

	// the redirect_uri must be left unescaped

	return fmt.Sprint(_LI_AUTH_URL, v.Encode(), "&redirect_uri="+l.Redirect)
}

// Validate gets a (2 month) token that is used with Get to retreive
// profile information about the user. Entire response is returned, errors
// included (ie: linkedin reject is not an err)
func (l *LinkedIn) ValidateToken(authToken string) (data map[string]interface{}, err error) {

	postUrl := fmt.Sprintf(_LI_VALIDATE_URL, authToken, l.Redirect, l.ApiKey, l.ApiSecret)
	println(postUrl)
	//resp, err := http.PostForm(postUrl, url.Values{})

	req, _ := http.NewRequest("POST", postUrl, nil)
	req.Header.Add("x-li-format", "json")

	c := http.Client{}
	resp, err := c.Do(req)

	if err != nil {
		return
	}

	defer resp.Body.Close()

	data = map[string]interface{}{}
	dec := json.NewDecoder(resp.Body)
	dec.Decode(&data)

	return
}

// Get a user profile values from linkedin. If fields is blank the default
// linkedin response contains name and linkedinUri. Otherwise the selected
// fields are requested from linkedin.
func Get(access_token string, fields ...string) (data map[string]interface{}, err error) {

	v := url.Values{}
	v.Add("oauth2_access_token", access_token)

	// Since linkedin parameters are going to get escaped (http does not like parenthesis
	// in url) its best to make the request with no URL and create one manually.
	// The Opaque value below makes sure that the Get request works with linkedin (otherwise
	// go url-ecapes the parethesies and the other silly characters)
	req, _ := http.NewRequest("GET", "", nil)
	req.URL = &url.URL{
		Scheme:   "https",
		Host:     "api.linkedin.com",
		Opaque:   makeProfileQuery(fields...),
		RawQuery: v.Encode(),
	}
	req.Header.Add("x-li-format", "json")

	c := http.Client{}
	resp, err := c.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	data = map[string]interface{}{}
	dec := json.NewDecoder(resp.Body)
	dec.Decode(&data)

	return
}

// Get all the companies listed as current in the linkedin response. For
// proper results the passed argument should be a linkedin response of
// containing "positions" and "headline".
//
// Its up to the implementation to decide which to use.
func GetCurrentCompanies(lresp map[string]interface{}) (curPositions []map[string]string, headline string) {

	var positionsMap map[string]interface{}
	var ok bool

	if headline, ok = lresp["headline"].(string); !ok {
		headline = ""
	}

	if positionsMap, ok = lresp["positions"].(map[string]interface{}); !ok {
		return
	}

	for _, pItem := range positionsMap {
		positions, ok := pItem.([]interface{})

		if !ok {
			continue
		}

		for _, positionRaw := range positions {
			if position, ok := positionRaw.(map[string]interface{}); ok {
				if isCurrent, ok := position["isCurrent"].(bool); ok && isCurrent {
					data := position["company"].(map[string]interface{})
					pos := map[string]string{}

					// Get company name
					if val, ok := data["name"]; ok && val != nil {
						pos["company"] = fmt.Sprint(val)
					}

					// Get title
					if val, ok := position["title"]; ok && val != nil {
						pos["position"] = fmt.Sprint(val)
					}

					curPositions = append(curPositions, pos)

					/* BUGGY previous version:
					   curPositions = append(curPositions, map[string]string{
					       "company":  data["name"].(string),
					       "position": position["title"].(string),
					   })
					*/
				}
			}
		}
	}

	return
}

// Create LinkedIn-formatted profile request. LinkedIn format looks
// like [api-url]/~:(id, picture-url, etc)
func makeProfileQuery(params ...string) string {

	if len(params) == 0 {
		return _LI_PROFILE_URL
	}

	buf := bytes.NewBufferString(_LI_PROFILE_URL + ":(")

	last := len(params) - 1
	comma := []byte(",")
	for _, param := range params[:last] {
		buf.Write([]byte(param))
		buf.Write(comma)
	}

	buf.Write([]byte(params[last]))
	buf.Write([]byte(")"))

	return buf.String()
}
<commit_msg>linkedin api v2<commit_after>package linkedin

import (
	"bytes"
	"encoding/json"
	"fmt"
	"errors"
	"net/http"
	"net/url"
	"strings"
	"strconv"
	//"io/ioutil"
)

// Flow:
// 1. Redirect user to RedirectUri, they confirm linkedin. This process generates an AuthToken
// 2. The AuthToken is exchanged for an AccessToken. Posting to _LI_VALIDATE returns the AccessToken
// 3. Requests for profile information must contain AccessToken paramater

const (
	_LI_AUTH_URL     = "https://www.linkedin.com/oauth/v2/authorization?"
	_LI_VALIDATE_URL = "https://www.linkedin.com/oauth/v2/accessToken?grant_type=authorization_code&code=%s&redirect_uri=%s&client_id=%s&client_secret=%s"
	_LI_PROFILE_URL  = "https://api.linkedin.com/v2/me"
)

// Contains data for the linked in api
type LinkedIn struct {
	ApiKey    string
	ApiSecret string
	Redirect  string
	State     string
}

// LinkedIn member profile data
type MemberProfile struct {
	Id string
	FirstName string
	LastName string
	Photos []PhotoDescriptor
}

// Describes a single profile photo
type PhotoDescriptor struct {
	Width  int
	Height int
	Url    string
}

// Finds photo with size closest to given size.
// Otherwise will return the one with size slightly under.
// Otherwise will return nil.
func (mp *MemberProfile) PhotoLargerThan(width int) (descr *PhotoDescriptor) {

	// No width meants first photo, if available
	if width <= 0 && len(mp.Photos) > 0 {
		photo := mp.Photos[0]
		return &photo
	}

	// Find two sizes that are close to given width
	var smallestPositiveDelta int = 10000
	var smallestNegativeDelta int = -10000
	var smallestPositiveIdx int = -1
	var smallestNegativeIdx int = -1

	for i, photo := range mp.Photos {
		delta := photo.Width - width
		if delta == 0 {
			return &photo
		}
		if delta > 0 && delta < smallestPositiveDelta {
			smallestPositiveDelta = delta
			smallestPositiveIdx = i
		} else if delta > smallestNegativeDelta {
			smallestNegativeDelta = delta
			smallestNegativeIdx = i
		}
	}

	if 0 <= smallestPositiveIdx && smallestPositiveIdx < len(mp.Photos) {
		photo := mp.Photos[smallestPositiveIdx]
		return &photo
	} else if 0 <= smallestNegativeIdx && smallestNegativeIdx < len(mp.Photos) {
		photo := mp.Photos[smallestNegativeIdx]
		return &photo
	}

	return
}

// String function.
func (mp *MemberProfile) String() string {

	ss := []string{}

	ss = append(ss, fmt.Sprintf("ID: %v", mp.Id))
	ss = append(ss, fmt.Sprintf("FirstName: %v", mp.FirstName))
	ss = append(ss, fmt.Sprintf("LastName: %v", mp.LastName))
	ss = append(ss, fmt.Sprintf("LastName: %v", mp.LastName))
	ss = append(ss, fmt.Sprintf("Photos: %v", len(mp.Photos)))

	return strings.Join(ss, ", ")
}

// Generate a an AuthUri - the user must be redirected here and accept
// the linkedin request. This process produces an AuthToken which must be
// sent to Validate quickly (under a minute) before linkedin expires it
func (l *LinkedIn) AuthUri(scope ...string) string {

	v := url.Values{}
	v.Add("response_type", "code")
	v.Add("client_id", l.ApiKey)
	v.Add("state", l.State)

	if len(scope) > 0 {
		v.Add("scope", strings.Join(scope, " "))
	} else {
		v.Add("scope", "r_liteprofile")
		//v.Add("scope", "r_basicprofile")
		//v.Add("scope", "r_fullprofile")
	}

	// The redirect_uri must be left unescaped

	return fmt.Sprint(_LI_AUTH_URL, v.Encode(), "&redirect_uri="+l.Redirect)
}

// Validate gets a (2 month) token that is used with Get to retreive
// profile information about the user. Entire response is returned, errors
// included (ie: linkedin reject is not an err)
func (l *LinkedIn) ValidateToken(authToken string) (data map[string]interface{}, err error) {

	postUrl := fmt.Sprintf(_LI_VALIDATE_URL, authToken, l.Redirect, l.ApiKey, l.ApiSecret)
	//resp, err := http.PostForm(postUrl, url.Values{})

	req, _ := http.NewRequest("POST", postUrl, nil)
	req.Header.Add("x-li-format", "json")

	c := http.Client{}
	resp, err := c.Do(req)

	if err != nil {
		return
	}

	defer resp.Body.Close()

	data = map[string]interface{}{}
	dec := json.NewDecoder(resp.Body)
	dec.Decode(&data)

	return
}

// Get a user profile values from linkedin. If fields is blank the default
// linkedin response contains name and linkedinUri. Otherwise the selected
// fields are requested from linkedin.
func getUserProfile(access_token string) (data map[string]interface{}, err error) {

	v := url.Values{}
	v.Add("oauth2_access_token", access_token)

	// Since linkedin parameters are going to get escaped (http does not like parenthesis
	// in url) its best to make the request with no URL and create one manually.
	// The Opaque value below makes sure that the Get request works with linkedin (otherwise
	// go url-ecapes the parethesies and the other silly characters)
	req, _ := http.NewRequest("GET", "", nil)
	req.URL = &url.URL{
		Scheme:   "https",
		Host:     "api.linkedin.com",
		//Opaque:   makeProfileQuery(fields...),
		Opaque:    _LI_PROFILE_URL,
		RawQuery: v.Encode(),
	}
	//req.Header.Add("x-li-format", "json")
	req.Header.Add("X-RestLi-Protocol-Version", "2.0.0")

	c := http.Client{}
	resp, err := c.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	// DEBUG:
	/*
	fmt.Println()
	fmt.Println("--- LI HTTP REQUEST---")
	fmt.Println(req.URL)
	fmt.Println("--- LI HTTP RESPONSE---")
	fmt.Println("Error    =", err)
	fmt.Println("Response =", resp)
	fmt.Println()
	// If body has been read then it'll become emptied and
	// JSON decode below will fail
	//fmt.Println("Response String =")
	//bodyBytes, _ := ioutil.ReadAll(resp.Body)
    //fmt.Println(string(bodyBytes))
	*/

	// Parse JSON

	data = map[string]interface{}{}
	dec := json.NewDecoder(resp.Body)
	dec.Decode(&data)

	return
}

// Get a user profile associated with given access token.
func GetUserProfile(access_token string) (profile MemberProfile, err error) {

	// Get member profile

	var profileData map[string]interface{}
	profileData, err = getUserProfile(access_token)
	if err != nil {
		return
	}
	if profileData == nil {
		err = errors.New("LinkedIn returned empty data")
		return
	}

	// FUNC: Gets string value from map, or empty string.
	getStringValue := func(m map[string]interface{}, key string) (s string) {
		if v := m[key]; v != nil {
			return fmt.Sprint(v)
		}
		return
	}

	// FUNC: Gets lite profile field using default localization.
	// map[localized:map[en_US:SOME_VALUE] preferredLocale:map[country:US language:en]]
	getValue := func(key string) (s string) {
		if amap, ok := profileData[key].(map[string]interface{}); ok {
			if amap1, ok1 := amap["preferredLocale"].(map[string]interface{}); ok1 {
				country := getStringValue(amap1, "country")
				language := getStringValue(amap1, "language")
				localization := fmt.Sprintf("%v_%v", language, country)

				if amap2, ok2 := amap["localized"].(map[string]interface{}); ok2 {
					return getStringValue(amap2, localization)
				}
			}
		}

		return
	}

	// Parse member basic information

	if s := getValue("firstName"); s != "" {
		profile.FirstName = s
	}
	if s := getValue("lastName"); s != "" {
		profile.LastName = s
	}

	// Member ID

	var profileId string
	if v := profileData["id"]; v != nil {
		profileId = fmt.Sprint(v)
		profile.Id = profileId
	} else {
		return
	}

	// Get member photo URL

	var photos []PhotoDescriptor
	photos, err = GetProfilePhotos(access_token, profileId)
	if err != nil {
		return
	}

	profile.Photos = photos

	// Positions ?
	GetProfilePositions(access_token, profileId)

	return
}

// Get a user profile photo from linkedin for given ID.
// Will not work for r_liteprofile
func GetProfilePositions(access_token string, id string) (photoUrl string, err error) {

	v := url.Values{}
	v.Add("oauth2_access_token", access_token)
	v.Add("projection", "(id,positions,profilePicture)") // profilePicture added just for testing

	// Since linkedin parameters are going to get escaped (http does not like parenthesis
	// in url) its best to make the request with no URL and create one manually.
	// The Opaque value below makes sure that the Get request works with linkedin (otherwise
	// go url-ecapes the parethesies and the other silly characters)
	req, _ := http.NewRequest("GET", "", nil)
	req.URL = &url.URL{
		Scheme:   "https",
		Host:     "api.linkedin.com",
		Opaque:    _LI_PROFILE_URL,
		RawQuery: v.Encode(),
	}
	req.Header.Add("X-RestLi-Protocol-Version", "2.0.0")

	c := http.Client{}
	resp, err := c.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	// DEBUG:
	/*
	fmt.Println()
	fmt.Println("--- LI HTTP REQUEST---")
	fmt.Println(req.URL)
	fmt.Println("--- LI HTTP RESPONSE---")
	fmt.Println("Error    =", err)
	fmt.Println("Response =", resp)
	fmt.Println()
	// If body has been read then it'll become emptied and
	// JSON decode below will fail
	//fmt.Println("Response String =")
	//bodyBytes, _ := ioutil.ReadAll(resp.Body)
    //fmt.Println(string(bodyBytes))
	*/

	data := map[string]interface{}{}
	dec := json.NewDecoder(resp.Body)
	dec.Decode(&data)

	return
}

// Get a user profile photo from linkedin for given ID.
// Returns empty array on failure.
func GetProfilePhotos(access_token string, id string) (photos []PhotoDescriptor, err error) {

	v := url.Values{}
	v.Add("oauth2_access_token", access_token)
	v.Add("projection", "(id,profilePicture(displayImage~:playableStreams))")

	// Since linkedin parameters are going to get escaped (http does not like parenthesis
	// in url) its best to make the request with no URL and create one manually.
	// The Opaque value below makes sure that the Get request works with linkedin (otherwise
	// go url-ecapes the parethesies and the other silly characters)
	req, _ := http.NewRequest("GET", "", nil)
	req.URL = &url.URL{
		Scheme:   "https",
		Host:     "api.linkedin.com",
		Opaque:    _LI_PROFILE_URL,
		RawQuery: v.Encode(),
	}
	req.Header.Add("X-RestLi-Protocol-Version", "2.0.0")

	c := http.Client{}
	resp, err := c.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	// DEBUG:
	/*
	fmt.Println()
	fmt.Println("--- LI HTTP REQUEST---")
	fmt.Println(req.URL)
	fmt.Println("--- LI HTTP RESPONSE---")
	fmt.Println("Error    =", err)
	fmt.Println("Response =", resp)
	fmt.Println()
	*/

	data := map[string]interface{}{}
	dec := json.NewDecoder(resp.Body)
	dec.Decode(&data)

	// FUNC: Gets string value from map, or empty string.
	getStringValue := func(m map[string]interface{}, key string) (s string) {
		if v := m[key]; v != nil {
			return fmt.Sprint(v)
		}
		return
	}

	// FUNC: Gets int value from map, or 0.
	getIntValue := func(m map[string]interface{}, key string) (n int) {
		if v, ok := m[key]; ok && v != nil {
			if n, ok = v.(int); ok {
				return
			}
			if f, ok := v.(float64); ok {
				n = int(f)
				return
			}
			n, _ = strconv.Atoi(fmt.Sprint(v))
			return
		}
		return
	}

	// FUNC: Gets map value from map, or nil.
	getMapValue := func(m map[string]interface{}, key string) (m2 map[string]interface{}) {
		if v, ok := m[key].(map[string]interface{}); ok {
			return v
		}
		return
	}

	// FUNC: Gets array value from map, or nil.
	getArrayValue := func(m map[string]interface{}, key string) (arr []interface{}) {
		if v, ok := m[key].([]interface{}); ok {
			return v
		}
		return
	}

	// Parse photo data

	// Build array of all images from:
	// data.profilePicture["displayImage~"].elements[]

	dataProfilePicture := getMapValue(data, "profilePicture")
	if dataProfilePicture == nil {
		return
	}

	dataDisplayImage := getMapValue(dataProfilePicture, "displayImage~")
	if dataDisplayImage == nil {
		return
	}

	dataElements := getArrayValue(dataDisplayImage, "elements")
	if dataElements == nil {
		return
	}

	// For each element get size
	// elements[0].data["com.linkedin.digitalmedia.mediaartifact.StillImage"].displaySize.width
	// and URL
	// elements[0].identifiers[0].identifier

	//photos := []PhotoDescriptors{}

	for _, d := range dataElements {
		if dataElement, ok := d.(map[string]interface{}); ok {
			dataElementData := getMapValue(dataElement, "data")
			if dataElementData == nil {
				continue
			}

			dataStillImage := getMapValue(dataElementData, "com.linkedin.digitalmedia.mediaartifact.StillImage")
			if dataStillImage == nil {
				continue
			}

			dataDisplaySize := getMapValue(dataStillImage, "displaySize")
			if dataDisplaySize == nil {
				continue
			}

			w := getIntValue(dataDisplaySize, "width")
			h := getIntValue(dataDisplaySize, "height")

			dataIdentifiers := getArrayValue(dataElement, "identifiers")
			if dataIdentifiers == nil {
				continue
			}

			var url string
			for _, d1 := range dataIdentifiers {
				if dataIdentifier := d1.(map[string]interface{}); ok {
					if s := getStringValue(dataIdentifier, "identifier"); s != "" {
						url = s
						break
					}
				}
			}

			if url == "" {
				continue
			}

			ph := PhotoDescriptor{ Width: w, Height: h, Url: url }
			photos = append(photos, ph)
		}
	}

	return
}

// Get all the companies listed as current in the linkedin response. For
// proper results the passed argument should be a linkedin response of
// containing "positions" and "headline".
//
// Its up to the implementation to decide which to use.
func GetCurrentCompanies(lresp map[string]interface{}) (curPositions []map[string]string, headline string) {

	var positionsMap map[string]interface{}
	var ok bool

	if headline, ok = lresp["headline"].(string); !ok {
		headline = ""
	}

	if positionsMap, ok = lresp["positions"].(map[string]interface{}); !ok {
		return
	}

	for _, pItem := range positionsMap {
		positions, ok := pItem.([]interface{})

		if !ok {
			continue
		}

		for _, positionRaw := range positions {
			if position, ok := positionRaw.(map[string]interface{}); ok {
				if isCurrent, ok := position["isCurrent"].(bool); ok && isCurrent {
					data := position["company"].(map[string]interface{})
					pos := map[string]string{}

					// Get company name
					if val, ok := data["name"]; ok && val != nil {
						pos["company"] = fmt.Sprint(val)
					}

					// Get title
					if val, ok := position["title"]; ok && val != nil {
						pos["position"] = fmt.Sprint(val)
					}

					curPositions = append(curPositions, pos)

					/* BUGGY previous version:
					   curPositions = append(curPositions, map[string]string{
					       "company":  data["name"].(string),
					       "position": position["title"].(string),
					   })
					*/
				}
			}
		}
	}

	return
}

// Create LinkedIn-formatted profile request. LinkedIn format looks
// like [api-url]/~:(id, picture-url, etc)
func makeProfileQuery(params ...string) string {

	if len(params) == 0 {
		return _LI_PROFILE_URL
	}

	buf := bytes.NewBufferString(_LI_PROFILE_URL + ":(")

	last := len(params) - 1
	comma := []byte(",")
	for _, param := range params[:last] {
		buf.Write([]byte(param))
		buf.Write(comma)
	}

	buf.Write([]byte(params[last]))
	buf.Write([]byte(")"))

	return buf.String()
}

<|endoftext|>


<commit_before>package azurerm

import (
	"fmt"
	"net/http"

	"github.com/Azure/azure-sdk-for-go/arm/dns"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceArmDnsAAAARecord() *schema.Resource {
	return &schema.Resource{
		Create: resourceArmDnsAaaaRecordCreateOrUpdate,
		Read:   resourceArmDnsAaaaRecordRead,
		Update: resourceArmDnsAaaaRecordCreateOrUpdate,
		Delete: resourceArmDnsAaaaRecordDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"resource_group_name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"zone_name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},

			"records": &schema.Schema{
				Type:     schema.TypeSet,
				Required: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Set:      schema.HashString,
			},

			"ttl": &schema.Schema{
				Type:     schema.TypeInt,
				Required: true,
			},

			"etag": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"tags": tagsSchema(),
		},
	}
}

func resourceArmDnsAaaaRecordCreateOrUpdate(d *schema.ResourceData, meta interface{}) error {
	dnsClient := meta.(*ArmClient).dnsClient

	name := d.Get("name").(string)
	resGroup := d.Get("resource_group_name").(string)
	zoneName := d.Get("zone_name").(string)
	ttl := int64(d.Get("ttl").(int))
	eTag := d.Get("etag").(string)

	tags := d.Get("tags").(map[string]interface{})
	metadata := expandTags(tags)

	records, err := expandAzureRmDnsAaaaRecords(d)
	props := dns.RecordSetProperties{
		Metadata:    metadata,
		TTL:         &ttl,
		AaaaRecords: &records,
	}

	parameters := dns.RecordSet{
		Name:                &name,
		RecordSetProperties: &props,
	}

	//last parameter is set to empty to allow updates to records after creation
	// (per SDK, set it to '*' to prevent updates, all other values are ignored)
	resp, err := dnsClient.CreateOrUpdate(resGroup, zoneName, name, dns.AAAA, parameters, eTag, "")
	if err != nil {
		return err
	}

	if resp.ID == nil {
		return fmt.Errorf("Cannot read DNS AAAA Record %s (resource group %s) ID", name, resGroup)
	}

	d.SetId(*resp.ID)

	return resourceArmDnsAaaaRecordRead(d, meta)
}

func resourceArmDnsAaaaRecordRead(d *schema.ResourceData, meta interface{}) error {
	dnsClient := meta.(*ArmClient).dnsClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}

	resGroup := id.ResourceGroup
	name := id.Path["AAAA"]
	zoneName := id.Path["dnszones"]

	resp, err := dnsClient.Get(resGroup, zoneName, name, dns.AAAA)
	if err != nil {
		return fmt.Errorf("Error reading DNS AAAA record %s: %v", name, err)
	}
	if resp.StatusCode == http.StatusNotFound {
		d.SetId("")
		return nil
	}

	d.Set("name", name)
	d.Set("resource_group_name", resGroup)
	d.Set("zone_name", zoneName)
	d.Set("ttl", resp.TTL)
	d.Set("etag", resp.Etag)

	if err := d.Set("records", flattenAzureRmDnsAaaaRecords(resp.AaaaRecords)); err != nil {
		return err
	}
	flattenAndSetTags(d, resp.Metadata)

	return nil
}

func resourceArmDnsAaaaRecordDelete(d *schema.ResourceData, meta interface{}) error {
	dnsClient := meta.(*ArmClient).dnsClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}

	resGroup := id.ResourceGroup
	name := id.Path["AAAA"]
	zoneName := id.Path["dnszones"]

	resp, error := dnsClient.Delete(resGroup, zoneName, name, dns.AAAA, "")
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("Error deleting DNS AAAA Record %s: %s", name, error)
	}

	return nil
}

func flattenAzureRmDnsAaaaRecords(records *[]dns.AaaaRecord) []string {
	results := make([]string, 0, len(*records))

	if records != nil {
		for _, record := range *records {
			results = append(results, *record.Ipv6Address)
		}
	}

	return results
}

func expandAzureRmDnsAaaaRecords(d *schema.ResourceData) ([]dns.AaaaRecord, error) {
	recordStrings := d.Get("records").(*schema.Set).List()
	records := make([]dns.AaaaRecord, len(recordStrings))

	for i, v := range recordStrings {
		ipv6 := v.(string)
		records[i] = dns.AaaaRecord{
			Ipv6Address: &ipv6,
		}
	}

	return records, nil
}
<commit_msg>response to PR feedback<commit_after>package azurerm

import (
	"fmt"
	"net/http"

	"github.com/Azure/azure-sdk-for-go/arm/dns"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceArmDnsAAAARecord() *schema.Resource {
	return &schema.Resource{
		Create: resourceArmDnsAaaaRecordCreateOrUpdate,
		Read:   resourceArmDnsAaaaRecordRead,
		Update: resourceArmDnsAaaaRecordCreateOrUpdate,
		Delete: resourceArmDnsAaaaRecordDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"resource_group_name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"zone_name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},

			"records": &schema.Schema{
				Type:     schema.TypeSet,
				Required: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Set:      schema.HashString,
			},

			"ttl": &schema.Schema{
				Type:     schema.TypeInt,
				Required: true,
			},

			"tags": tagsSchema(),
		},
	}
}

func resourceArmDnsAaaaRecordCreateOrUpdate(d *schema.ResourceData, meta interface{}) error {
	dnsClient := meta.(*ArmClient).dnsClient

	name := d.Get("name").(string)
	resGroup := d.Get("resource_group_name").(string)
	zoneName := d.Get("zone_name").(string)
	ttl := int64(d.Get("ttl").(int))

	tags := d.Get("tags").(map[string]interface{})
	metadata := expandTags(tags)

	records, err := expandAzureRmDnsAaaaRecords(d)
	props := dns.RecordSetProperties{
		Metadata:    metadata,
		TTL:         &ttl,
		AaaaRecords: &records,
	}

	parameters := dns.RecordSet{
		Name:                &name,
		RecordSetProperties: &props,
	}

	//last parameter is set to empty to allow updates to records after creation
	// (per SDK, set it to '*' to prevent updates, all other values are ignored)
	resp, err := dnsClient.CreateOrUpdate(resGroup, zoneName, name, dns.AAAA, parameters, "", "")
	if err != nil {
		return err
	}

	if resp.ID == nil {
		return fmt.Errorf("Cannot read DNS AAAA Record %s (resource group %s) ID", name, resGroup)
	}

	d.SetId(*resp.ID)

	return resourceArmDnsAaaaRecordRead(d, meta)
}

func resourceArmDnsAaaaRecordRead(d *schema.ResourceData, meta interface{}) error {
	dnsClient := meta.(*ArmClient).dnsClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}

	resGroup := id.ResourceGroup
	name := id.Path["AAAA"]
	zoneName := id.Path["dnszones"]

	resp, err := dnsClient.Get(resGroup, zoneName, name, dns.AAAA)
	if err != nil {
		return fmt.Errorf("Error reading DNS AAAA record %s: %v", name, err)
	}
	if resp.StatusCode == http.StatusNotFound {
		d.SetId("")
		return nil
	}

	d.Set("name", name)
	d.Set("resource_group_name", resGroup)
	d.Set("zone_name", zoneName)
	d.Set("ttl", resp.TTL)

	if err := d.Set("records", flattenAzureRmDnsAaaaRecords(resp.AaaaRecords)); err != nil {
		return err
	}
	flattenAndSetTags(d, resp.Metadata)

	return nil
}

func resourceArmDnsAaaaRecordDelete(d *schema.ResourceData, meta interface{}) error {
	dnsClient := meta.(*ArmClient).dnsClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}

	resGroup := id.ResourceGroup
	name := id.Path["AAAA"]
	zoneName := id.Path["dnszones"]

	resp, error := dnsClient.Delete(resGroup, zoneName, name, dns.AAAA, "")
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("Error deleting DNS AAAA Record %s: %+v", name, error)
	}

	return nil
}

func flattenAzureRmDnsAaaaRecords(records *[]dns.AaaaRecord) []string {
	results := make([]string, 0, len(*records))

	if records != nil {
		for _, record := range *records {
			results = append(results, *record.Ipv6Address)
		}
	}

	return results
}

func expandAzureRmDnsAaaaRecords(d *schema.ResourceData) ([]dns.AaaaRecord, error) {
	recordStrings := d.Get("records").(*schema.Set).List()
	records := make([]dns.AaaaRecord, len(recordStrings))

	for i, v := range recordStrings {
		ipv6 := v.(string)
		records[i] = dns.AaaaRecord{
			Ipv6Address: &ipv6,
		}
	}

	return records, nil
}
<|endoftext|>


<commit_before>package log

import (
	"io"

	"gopkg.in/logfmt.v0"
)

type logfmtLogger struct {
	w io.Writer
}

// NewLogfmtLogger returns a basic logger that encodes keyvals as simple "k=v"
// pairs to the Writer.
func NewLogfmtLogger(w io.Writer) Logger {
	return &logfmtLogger{w}
}

func (l logfmtLogger) Log(keyvals ...interface{}) error {
	b, err := logfmt.MarshalKeyvals(keyvals...)
	if err != nil {
		return err
	}
	b = append(b, '\n')
	if _, err := l.w.Write(b); err != nil {
		return err
	}
	return nil
}
<commit_msg>Update docs.<commit_after>package log

import (
	"io"

	"gopkg.in/logfmt.v0"
)

type logfmtLogger struct {
	w io.Writer
}

// NewLogfmtLogger returns a logger that encodes keyvals to the Writer in
// logfmt format.
func NewLogfmtLogger(w io.Writer) Logger {
	return &logfmtLogger{w}
}

func (l logfmtLogger) Log(keyvals ...interface{}) error {
	b, err := logfmt.MarshalKeyvals(keyvals...)
	if err != nil {
		return err
	}
	b = append(b, '\n')
	if _, err := l.w.Write(b); err != nil {
		return err
	}
	return nil
}
<|endoftext|>