<commit_before>package term

import (
	"io"

	"golang.org/x/mobile/event/key"
	"golang.org/x/mobile/event/mouse"

	"github.com/gdamore/tcell"
	"github.com/mibk/syd/ui"
)

type UI struct {
	screen        tcell.Screen
	wasBtnPressed bool

	windows []*Window
}

func (t *UI) Init() error {
	sc, err := tcell.NewScreen()
	if err != nil {
		return err
	}
	if err := sc.Init(); err != nil {
		return err
	}
	sc.EnableMouse()
	t.screen = sc

	go t.translateEvents()
	return nil
}

func (t *UI) Close() error {
	t.screen.Fini()
	return nil
}

func (t *UI) Size() (w, h int) { return t.screen.Size() }

func (t *UI) NewWindow() *Window {
	head := &Text{
		frame: new(Frame),
		bgstyle: tcell.StyleDefault.
			Background(tcell.GetColor("#eaffff")),
		hlstyle: tcell.StyleDefault.
			Background(tcell.GetColor("#90e0e0")),
	}
	body := &Text{
		frame: new(Frame),
		bgstyle: tcell.StyleDefault.
			Background(tcell.GetColor("#ffffea")),
		hlstyle: tcell.StyleDefault.
			Background(tcell.GetColor("#e0e090")),
	}
	win := &Window{
		x: 1, y: 1, // For testing purposes.
		ui:     t,
		head:   head,
		body:   body,
		active: body,
	}
	head.win = win
	body.win = win
	t.windows = append(t.windows, win)
	return win
}

// TODO: This is for temporary reasons. Remove it.
func (t *UI) Push_Mouse_Event(ev mouse.Event) {
	win := t.windows[0] // TODO: It may not exist.
	if int(ev.Y) >= win.body.y {
		win.body.click(ev)
		win.active = win.body
	} else {
		win.head.click(ev)
		win.active = win.head
	}
}

func (t *UI) Push_Key_Event(ev key.Event) {
	t.windows[0].active.keyEventHandler(ev)
}

type Window struct {
	ui *UI

	width, height int
	x, y          int

	head   *Text
	body   *Text
	active *Text // will receive key events
}

func (win *Window) Size() (w, h int) {
	// TODO: Return the width and height of the window.
	w, h = win.ui.Size()
	return w / 2, h
}

func (win *Window) Head() *Text { return win.head }
func (win *Window) Body() *Text { return win.body }

func (win *Window) Clear() {
	win.head.clear()
	win.body.clear()
}

func (win *Window) Flush() {
	_, height := win.Size()
	win.head.x = win.x
	win.head.y = win.y
	win.body.x = win.x
	win.head.flush()
	win.body.y = win.y + len(win.head.frame.lines)
	win.ui.screen.HideCursor()
	win.body.flush()
	win.body.fill(height)
	win.ui.screen.Show()
}

type Text struct {
	win   *Window
	frame *Frame

	x, y int
	cur  struct {
		p0, p1 int // char position
		x, y   int // current position
	}

	// styles
	bgstyle tcell.Style
	hlstyle tcell.Style

	mouseEventHandler ui.MouseEventHandler
	keyEventHandler   ui.KeyEventHandler
}

func (t *Text) click(ev mouse.Event) {
	if t.mouseEventHandler == nil {
		return
	}
	p := t.frame.CharsUntilXY(int(ev.X)-t.x, int(ev.Y)-t.y)
	t.mouseEventHandler(p, ev)
}

func (t *Text) OnMouseEvent(h ui.MouseEventHandler) {
	t.mouseEventHandler = h
}

func (t *Text) OnKeyEvent(h ui.KeyEventHandler) {
	t.keyEventHandler = h
}

func (t *Text) clear() {
	*t.frame = Frame{
		lines:   make([][]rune, 1),
		wantCol: t.frame.wantCol,
	}
	t.cur.x, t.cur.y = 0, 0
	t.checkSelection()
}

func (t *Text) Select(p0, p1 int) { t.cur.p0, t.cur.p1 = p0, p1 }

func (t *Text) WriteRune(r rune) error {
	if r != '\n' {
		t.frame.lines[t.cur.y] = append(t.frame.lines[t.cur.y], r)
	}

	w, h := t.win.Size()
	if t.cur.x >= w || r == '\n' {
		t.cur.y++
		t.cur.x = 0
		t.frame.lines = append(t.frame.lines, nil)
		if t.cur.y == h {
			return io.EOF
		}
	} else if r == '\t' {
		t.cur.x += tabWidthForCol(t.cur.x)
	} else {
		t.cur.x++
	}
	t.frame.nchars++
	t.checkSelection()
	return nil
}

// checkSelection tries to line0, line1, and wantCol.
func (t *Text) checkSelection() {
	if t.cur.p0 == t.frame.nchars {
		t.frame.line0 = t.cur.y
		if t.frame.wantCol == ui.ColQ0 {
			t.frame.wantCol = t.cur.x
		}
	}
	if t.cur.p1 == t.frame.nchars {
		t.frame.line1 = t.cur.y
		if t.frame.wantCol == ui.ColQ1 {
			t.frame.wantCol = t.cur.x
		}
	}
}

var reverse = tcell.StyleDefault.Reverse(true)

func (t *Text) flush() {
	width, _ := t.win.Size()
	style := t.bgstyle
	selText := func(p, x, y int) {
		if p == t.cur.p0 && t.cur.p0 == t.cur.p1 {
			style = reverse
		} else if p >= t.cur.p0 && p < t.cur.p1 {
			style = t.hlstyle
		} else {
			style = t.bgstyle
		}
	}
	p := 0
	for y, l := range t.frame.lines {
		x := 0
		for _, r := range l {
			selText(p, x, y)
			w := 1
			if r == '\t' {
				r = ' '
				w = tabWidthForCol(x)

			}
			for i := 0; i < w; i++ {
				t.win.ui.screen.SetContent(t.x+x, t.y+y, r, nil, style)
				x += 1
				if style == reverse {
					style = t.bgstyle
				}
			}
			p++
		}
		selText(p, x, y)
		for ; x < width; x++ {
			t.win.ui.screen.SetContent(t.x+x, t.y+y, ' ', nil, style)
			if style == reverse {
				style = t.bgstyle
			}
		}
		p++
	}
}

func (t *Text) fill(height int) {
	width, _ := t.win.Size()
	for y := len(t.frame.lines) + t.y; y < height; y++ {
		for x := 0; x < width; x++ {
			t.win.ui.screen.SetContent(t.win.x+x, y, ' ', nil, t.bgstyle)
		}
	}
}

func (t *Text) Frame() *Frame { return t.frame }

type Frame struct {
	lines   [][]rune
	line0   int
	line1   int
	wantCol int
	nchars  int
}

func (f *Frame) Nchars() int                { return f.nchars }
func (f *Frame) SelectionLines() (int, int) { return f.line0, f.line1 }

func (f *Frame) CharsUntilXY(x, y int) int {
	if y >= len(f.lines) {
		return f.nchars
	}
	var p int
	for n, l := range f.lines {
		if n == y {
			return p + charsUntilX(l, x)
		}
		p += len(l) + 1 // + '\n'
	}
	return 0
}

func charsUntilX(s []rune, x int) int {
	var w int
	for i, r := range s {
		if r == '\t' {
			w += tabWidthForCol(w)
		} else {
			w += 1
		}
		if w > x {
			return i
		}
	}
	return len(s)
}

const tabStop = 8

func tabWidthForCol(col int) int {
	w := tabStop - col%tabStop
	if w == 0 {
		return tabStop
	}
	return w
}

func (f *Frame) Lines() int         { return len(f.lines) }
func (f *Frame) WantCol() int       { return f.wantCol }
func (f *Frame) SetWantCol(col int) { f.wantCol = col }
<commit_msg>ui/term: Fix wrapping of long lines<commit_after>package term

import (
	"io"

	"golang.org/x/mobile/event/key"
	"golang.org/x/mobile/event/mouse"

	"github.com/gdamore/tcell"
	"github.com/mibk/syd/ui"
)

type UI struct {
	screen        tcell.Screen
	wasBtnPressed bool

	windows []*Window
}

func (t *UI) Init() error {
	sc, err := tcell.NewScreen()
	if err != nil {
		return err
	}
	if err := sc.Init(); err != nil {
		return err
	}
	sc.EnableMouse()
	t.screen = sc

	go t.translateEvents()
	return nil
}

func (t *UI) Close() error {
	t.screen.Fini()
	return nil
}

func (t *UI) Size() (w, h int) { return t.screen.Size() }

func (t *UI) NewWindow() *Window {
	head := &Text{
		frame: new(Frame),
		bgstyle: tcell.StyleDefault.
			Background(tcell.GetColor("#eaffff")),
		hlstyle: tcell.StyleDefault.
			Background(tcell.GetColor("#90e0e0")),
	}
	body := &Text{
		frame: new(Frame),
		bgstyle: tcell.StyleDefault.
			Background(tcell.GetColor("#ffffea")),
		hlstyle: tcell.StyleDefault.
			Background(tcell.GetColor("#e0e090")),
	}
	win := &Window{
		x: 1, y: 1, // For testing purposes.
		ui:     t,
		head:   head,
		body:   body,
		active: body,
	}
	head.win = win
	body.win = win
	t.windows = append(t.windows, win)
	return win
}

// TODO: This is for temporary reasons. Remove it.
func (t *UI) Push_Mouse_Event(ev mouse.Event) {
	win := t.windows[0] // TODO: It may not exist.
	if int(ev.Y) >= win.body.y {
		win.body.click(ev)
		win.active = win.body
	} else {
		win.head.click(ev)
		win.active = win.head
	}
}

func (t *UI) Push_Key_Event(ev key.Event) {
	t.windows[0].active.keyEventHandler(ev)
}

type Window struct {
	ui *UI

	width, height int
	x, y          int

	head   *Text
	body   *Text
	active *Text // will receive key events
}

func (win *Window) Size() (w, h int) {
	// TODO: Return the width and height of the window.
	w, h = win.ui.Size()
	return w / 2, h
}

func (win *Window) Head() *Text { return win.head }
func (win *Window) Body() *Text { return win.body }

func (win *Window) Clear() {
	win.head.clear()
	win.body.clear()
}

func (win *Window) Flush() {
	_, height := win.Size()
	win.head.x = win.x
	win.head.y = win.y
	win.body.x = win.x
	win.head.flush()
	win.body.y = win.y + len(win.head.frame.lines)
	win.ui.screen.HideCursor()
	win.body.flush()
	win.body.fill(height)
	win.ui.screen.Show()
}

type Text struct {
	win   *Window
	frame *Frame

	x, y int
	cur  struct {
		p0, p1 int // char position
		x, y   int // current position
	}

	// styles
	bgstyle tcell.Style
	hlstyle tcell.Style

	mouseEventHandler ui.MouseEventHandler
	keyEventHandler   ui.KeyEventHandler
}

func (t *Text) click(ev mouse.Event) {
	if t.mouseEventHandler == nil {
		return
	}
	p := t.frame.CharsUntilXY(int(ev.X)-t.x, int(ev.Y)-t.y)
	t.mouseEventHandler(p, ev)
}

func (t *Text) OnMouseEvent(h ui.MouseEventHandler) {
	t.mouseEventHandler = h
}

func (t *Text) OnKeyEvent(h ui.KeyEventHandler) {
	t.keyEventHandler = h
}

func (t *Text) clear() {
	*t.frame = Frame{
		lines:   make([][]rune, 1),
		wantCol: t.frame.wantCol,
	}
	t.cur.x, t.cur.y = 0, 0
	t.checkSelection()
}

func (t *Text) Select(p0, p1 int) { t.cur.p0, t.cur.p1 = p0, p1 }

func (t *Text) WriteRune(r rune) error {
	t.frame.lines[t.cur.y] = append(t.frame.lines[t.cur.y], r)
	if r == '\t' {
		t.cur.x += tabWidthForCol(t.cur.x)
	} else {
		t.cur.x++
	}

	w, h := t.win.Size()
	if t.cur.x >= w || r == '\n' {
		t.cur.y++
		t.cur.x = 0
		t.frame.lines = append(t.frame.lines, nil)
		if t.cur.y == h {
			return io.EOF
		}
	}

	t.frame.nchars++
	t.checkSelection()
	return nil
}

// checkSelection tries to line0, line1, and wantCol.
func (t *Text) checkSelection() {
	if t.cur.p0 == t.frame.nchars {
		t.frame.line0 = t.cur.y
		if t.frame.wantCol == ui.ColQ0 {
			t.frame.wantCol = t.cur.x
		}
	}
	if t.cur.p1 == t.frame.nchars {
		t.frame.line1 = t.cur.y
		if t.frame.wantCol == ui.ColQ1 {
			t.frame.wantCol = t.cur.x
		}
	}
}

var reverse = tcell.StyleDefault.Reverse(true)

func (t *Text) flush() {
	width, _ := t.win.Size()
	style := t.bgstyle
	selStyle := func(p int) {
		if p == t.cur.p0 && t.cur.p0 == t.cur.p1 {
			style = reverse
		} else if p >= t.cur.p0 && p < t.cur.p1 {
			style = t.hlstyle
		} else {
			style = t.bgstyle
		}
	}
	p := 0
	for y, l := range t.frame.lines {
		x := 0
		for _, r := range l {
			selStyle(p)
			p++
			if r == '\n' {
				goto fill
			}
			w := 1
			if r == '\t' {
				r = ' '
				w = tabWidthForCol(x)
			}
			for i := 0; i < w && x < width; i++ {
				// TODO: Should the rest of the tab at the end of a
				// line span the begining of the next line?
				t.win.ui.screen.SetContent(t.x+x, t.y+y, r, nil, style)
				x++
				if style == reverse {
					style = t.bgstyle
				}
			}
		}
		selStyle(p)
	fill:
		for ; x < width; x++ {
			t.win.ui.screen.SetContent(t.x+x, t.y+y, ' ', nil, style)
			if style == reverse {
				style = t.bgstyle
			}
		}
	}
}

func (t *Text) fill(height int) {
	width, _ := t.win.Size()
	for y := len(t.frame.lines) + t.y; y < height; y++ {
		for x := 0; x < width; x++ {
			t.win.ui.screen.SetContent(t.win.x+x, y, ' ', nil, t.bgstyle)
		}
	}
}

func (t *Text) Frame() *Frame { return t.frame }

type Frame struct {
	lines   [][]rune
	line0   int
	line1   int
	wantCol int
	nchars  int
}

func (f *Frame) Nchars() int                { return f.nchars }
func (f *Frame) SelectionLines() (int, int) { return f.line0, f.line1 }

func (f *Frame) CharsUntilXY(x, y int) int {
	if y >= len(f.lines) {
		return f.nchars
	}
	var p int
	for n, l := range f.lines {
		if n == y {
			return p + charsUntilX(l, x)
		}
		p += len(l)
	}
	return 0
}

func charsUntilX(s []rune, x int) int {
	if len(s) == 0 {
		return 0
	}
	var w int
	for i, r := range s {
		if r == '\t' {
			w += tabWidthForCol(w)
		} else {
			w += 1
		}
		if w > x {
			return i
		}
	}
	if s[len(s)-1] == '\n' {
		return len(s) - 1
	}
	return len(s)
}

const tabStop = 8

func tabWidthForCol(col int) int {
	w := tabStop - col%tabStop
	if w == 0 {
		return tabStop
	}
	return w
}

func (f *Frame) Lines() int         { return len(f.lines) }
func (f *Frame) WantCol() int       { return f.wantCol }
func (f *Frame) SetWantCol(col int) { f.wantCol = col }
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"os/exec"
	"strings"
	"time"
)

type tarsnap struct {
	config *Config
}

func (t *tarsnap) backup(s *Set) error {
	ca := time.Now().UTC().Format(time.RFC3339)

	// archive is named archiveName_currentTime
	name := strings.Join([]string{s.Name, nameTimeSep, ca}, "")

	ops := append([]string{"-cf", name}, s.Dirs...)
	if err := exec.Command(t.config.TarsnapLoc, ops...).Run(); err != nil {
		return err
	}

	return nil
}

func (t *tarsnap) archives() ([]*archive, error) {
	cmd := exec.Command(t.config.TarsnapLoc, "--list-archives")

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	if err := cmd.Start(); err != nil {
		return nil, err
	}

	archives := []*archive{}

	scn := bufio.NewScanner(stdout)
	for scn.Scan() {
		arc := &archive{}
		if err := arc.decodeName(scn.Text()); err != nil {
			continue
		}

		archives = append(archives, arc)
	}

	return archives, nil
}

func (t *tarsnap) delete(a *archive) error {
	cmd := exec.Command(t.config.TarsnapLoc, "-df", a.archiveName())
	if err := cmd.Start(); err != nil {
		return err
	}

	return nil
}
<commit_msg>Fix archive deletion<commit_after>package main

import (
	"bufio"
	"os/exec"
	"strings"
	"time"
)

type tarsnap struct {
	config *Config
}

func (t *tarsnap) backup(s *Set) error {
	ca := time.Now().UTC().Format(time.RFC3339)

	// archive is named archiveName_currentTime
	name := strings.Join([]string{s.Name, nameTimeSep, ca}, "")

	ops := append([]string{"-cf", name}, s.Dirs...)
	if err := exec.Command(t.config.TarsnapLoc, ops...).Run(); err != nil {
		return err
	}

	return nil
}

func (t *tarsnap) archives() ([]*archive, error) {
	cmd := exec.Command(t.config.TarsnapLoc, "--list-archives")
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	if err := cmd.Start(); err != nil {
		return nil, err
	}

	archives := []*archive{}

	scn := bufio.NewScanner(stdout)
	for scn.Scan() {
		arc := &archive{}
		if err := arc.decodeName(scn.Text()); err != nil {
			continue
		}

		archives = append(archives, arc)
	}

	return archives, nil
}

func (t *tarsnap) delete(a *archive) error {
	cmd := exec.Command(t.config.TarsnapLoc, "-df", a.archiveName())
	if err := cmd.Run(); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package docker

import (
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"net/http"
	"net/url"
	"os"
	"strconv"

	"github.com/Sirupsen/logrus"
	"github.com/containers/image/manifest"
	"github.com/containers/image/types"
)

// ErrFetchManifest provides the error when fetching the manifest fails
type ErrFetchManifest struct {
	statusCode int
	body       []byte
}

func (e ErrFetchManifest) Error() string {
	return fmt.Sprintf("error fetching manifest: status code: %d, body: %s", e.statusCode, string(e.body))
}

type dockerImageSource struct {
	ref                        dockerReference
	requestedManifestMIMETypes []string
	c                          *dockerClient
	// State
	cachedManifest         []byte // nil if not loaded yet
	cachedManifestMIMEType string // Only valid if cachedManifest != nil
}

// newImageSource creates a new ImageSource for the specified image reference,
// asking the backend to use a manifest from requestedManifestMIMETypes if possible.
// nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes.
// The caller must call .Close() on the returned ImageSource.
func newImageSource(ctx *types.SystemContext, ref dockerReference, requestedManifestMIMETypes []string) (*dockerImageSource, error) {
	c, err := newDockerClient(ctx, ref, false)
	if err != nil {
		return nil, err
	}
	if requestedManifestMIMETypes == nil {
		requestedManifestMIMETypes = manifest.DefaultRequestedManifestMIMETypes
	}
	return &dockerImageSource{
		ref: ref,
		requestedManifestMIMETypes: requestedManifestMIMETypes,
		c: c,
	}, nil
}

// Reference returns the reference used to set up this source, _as specified by the user_
// (not as the image itself, or its underlying storage, claims).  This can be used e.g. to determine which public keys are trusted for this image.
func (s *dockerImageSource) Reference() types.ImageReference {
	return s.ref
}

// Close removes resources associated with an initialized ImageSource, if any.
func (s *dockerImageSource) Close() {
}

// simplifyContentType drops parameters from a HTTP media type (see https://tools.ietf.org/html/rfc7231#section-3.1.1.1)
// Alternatively, an empty string is returned unchanged, and invalid values are "simplified" to an empty string.
func simplifyContentType(contentType string) string {
	if contentType == "" {
		return contentType
	}
	mimeType, _, err := mime.ParseMediaType(contentType)
	if err != nil {
		return ""
	}
	return mimeType
}

func (s *dockerImageSource) GetManifest() ([]byte, string, error) {
	err := s.ensureManifestIsLoaded()
	if err != nil {
		return nil, "", err
	}
	return s.cachedManifest, s.cachedManifestMIMEType, nil
}

func (s *dockerImageSource) fetchManifest(tagOrDigest string) ([]byte, string, error) {
	url := fmt.Sprintf(manifestURL, s.ref.ref.RemoteName(), tagOrDigest)
	headers := make(map[string][]string)
	headers["Accept"] = s.requestedManifestMIMETypes
	res, err := s.c.makeRequest("GET", url, headers, nil)
	if err != nil {
		return nil, "", err
	}
	defer res.Body.Close()
	manblob, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, "", err
	}
	if res.StatusCode != http.StatusOK {
		return nil, "", ErrFetchManifest{res.StatusCode, manblob}
	}
	return manblob, simplifyContentType(res.Header.Get("Content-Type")), nil
}

// GetTargetManifest returns an image's manifest given a digest.
// This is mainly used to retrieve a single image's manifest out of a manifest list.
func (s *dockerImageSource) GetTargetManifest(digest string) ([]byte, string, error) {
	return s.fetchManifest(digest)
}

// ensureManifestIsLoaded sets s.cachedManifest and s.cachedManifestMIMEType
//
// ImageSource implementations are not required or expected to do any caching,
// but because our signatures are “attached” to the manifest digest,
// we need to ensure that the digest of the manifest returned by GetManifest
// and used by GetSignatures are consistent, otherwise we would get spurious
// signature verification failures when pulling while a tag is being updated.
func (s *dockerImageSource) ensureManifestIsLoaded() error {
	if s.cachedManifest != nil {
		return nil
	}

	reference, err := s.ref.tagOrDigest()
	if err != nil {
		return err
	}

	manblob, mt, err := s.fetchManifest(reference)
	if err != nil {
		return err
	}
	// We might validate manblob against the Docker-Content-Digest header here to protect against transport errors.
	s.cachedManifest = manblob
	s.cachedManifestMIMEType = mt
	return nil
}

// GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown).
func (s *dockerImageSource) GetBlob(digest string) (io.ReadCloser, int64, error) {
	url := fmt.Sprintf(blobsURL, s.ref.ref.RemoteName(), digest)
	logrus.Debugf("Downloading %s", url)
	res, err := s.c.makeRequest("GET", url, nil, nil)
	if err != nil {
		return nil, 0, err
	}
	if res.StatusCode != http.StatusOK {
		// print url also
		return nil, 0, fmt.Errorf("Invalid status code returned when fetching blob %d", res.StatusCode)
	}
	size, err := strconv.ParseInt(res.Header.Get("Content-Length"), 10, 64)
	if err != nil {
		size = -1
	}
	return res.Body, size, nil
}

func (s *dockerImageSource) GetSignatures() ([][]byte, error) {
	if s.c.signatureBase == nil { // Skip dealing with the manifest digest if not necessary.
		return [][]byte{}, nil
	}

	if err := s.ensureManifestIsLoaded(); err != nil {
		return nil, err
	}
	manifestDigest, err := manifest.Digest(s.cachedManifest)
	if err != nil {
		return nil, err
	}

	signatures := [][]byte{}
	for i := 0; ; i++ {
		url := signatureStorageURL(s.c.signatureBase, manifestDigest, i)
		if url == nil {
			return nil, fmt.Errorf("Internal error: signatureStorageURL with non-nil base returned nil")
		}
		signature, missing, err := s.getOneSignature(url)
		if err != nil {
			return nil, err
		}
		if missing {
			break
		}
		signatures = append(signatures, signature)
	}
	return signatures, nil
}

// getOneSignature downloads one signature from url.
// If it successfully determines that the signature does not exist, returns with missing set to true and error set to nil.
func (s *dockerImageSource) getOneSignature(url *url.URL) (signature []byte, missing bool, err error) {
	switch url.Scheme {
	case "file":
		logrus.Debugf("Reading %s", url.Path)
		sig, err := ioutil.ReadFile(url.Path)
		if err != nil {
			if os.IsNotExist(err) {
				return nil, true, nil
			}
			return nil, false, err
		}
		return sig, false, nil

	case "http", "https":
		logrus.Debugf("GET %s", url)
		res, err := s.c.client.Get(url.String())
		if err != nil {
			return nil, false, err
		}
		defer res.Body.Close()
		if res.StatusCode == http.StatusNotFound {
			return nil, true, nil
		} else if res.StatusCode != http.StatusOK {
			return nil, false, fmt.Errorf("Error reading signature from %s: status %d", url.String(), res.StatusCode)
		}
		sig, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return nil, false, err
		}
		return sig, false, nil

	default:
		return nil, false, fmt.Errorf("Unsupported scheme when reading signature from %s", url.String())
	}
}

// deleteImage deletes the named image from the registry, if supported.
func deleteImage(ctx *types.SystemContext, ref dockerReference) error {
	c, err := newDockerClient(ctx, ref, true)
	if err != nil {
		return err
	}

	// When retrieving the digest from a registry >= 2.3 use the following header:
	//   "Accept": "application/vnd.docker.distribution.manifest.v2+json"
	headers := make(map[string][]string)
	headers["Accept"] = []string{manifest.DockerV2Schema2MediaType}

	reference, err := ref.tagOrDigest()
	if err != nil {
		return err
	}
	getURL := fmt.Sprintf(manifestURL, ref.ref.RemoteName(), reference)
	get, err := c.makeRequest("GET", getURL, headers, nil)
	if err != nil {
		return err
	}
	defer get.Body.Close()
	manifestBody, err := ioutil.ReadAll(get.Body)
	if err != nil {
		return err
	}
	switch get.StatusCode {
	case http.StatusOK:
	case http.StatusNotFound:
		return fmt.Errorf("Unable to delete %v. Image may not exist or is not stored with a v2 Schema in a v2 registry.", ref.ref)
	default:
		return fmt.Errorf("Failed to delete %v: %s (%v)", ref.ref, manifestBody, get.Status)
	}

	digest := get.Header.Get("Docker-Content-Digest")
	deleteURL := fmt.Sprintf(manifestURL, ref.ref.RemoteName(), digest)

	// When retrieving the digest from a registry >= 2.3 use the following header:
	//   "Accept": "application/vnd.docker.distribution.manifest.v2+json"
	delete, err := c.makeRequest("DELETE", deleteURL, headers, nil)
	if err != nil {
		return err
	}
	defer delete.Body.Close()

	body, err := ioutil.ReadAll(delete.Body)
	if err != nil {
		return err
	}
	if delete.StatusCode != http.StatusAccepted {
		return fmt.Errorf("Failed to delete %v: %s (%v)", deleteURL, string(body), delete.Status)
	}

	if c.signatureBase != nil {
		manifestDigest, err := manifest.Digest(manifestBody)
		if err != nil {
			return err
		}

		for i := 0; ; i++ {
			url := signatureStorageURL(c.signatureBase, manifestDigest, i)
			if url == nil {
				return fmt.Errorf("Internal error: signatureStorageURL with non-nil base returned nil")
			}
			missing, err := c.deleteOneSignature(url)
			if err != nil {
				return err
			}
			if missing {
				break
			}
		}
	}

	return nil
}
<commit_msg>docker: reuse docker/distribution registry errros<commit_after>package docker

import (
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"net/http"
	"net/url"
	"os"
	"strconv"

	"github.com/Sirupsen/logrus"
	"github.com/containers/image/manifest"
	"github.com/containers/image/types"
	"github.com/docker/distribution/registry/client"
)

type dockerImageSource struct {
	ref                        dockerReference
	requestedManifestMIMETypes []string
	c                          *dockerClient
	// State
	cachedManifest         []byte // nil if not loaded yet
	cachedManifestMIMEType string // Only valid if cachedManifest != nil
}

// newImageSource creates a new ImageSource for the specified image reference,
// asking the backend to use a manifest from requestedManifestMIMETypes if possible.
// nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes.
// The caller must call .Close() on the returned ImageSource.
func newImageSource(ctx *types.SystemContext, ref dockerReference, requestedManifestMIMETypes []string) (*dockerImageSource, error) {
	c, err := newDockerClient(ctx, ref, false)
	if err != nil {
		return nil, err
	}
	if requestedManifestMIMETypes == nil {
		requestedManifestMIMETypes = manifest.DefaultRequestedManifestMIMETypes
	}
	return &dockerImageSource{
		ref: ref,
		requestedManifestMIMETypes: requestedManifestMIMETypes,
		c: c,
	}, nil
}

// Reference returns the reference used to set up this source, _as specified by the user_
// (not as the image itself, or its underlying storage, claims).  This can be used e.g. to determine which public keys are trusted for this image.
func (s *dockerImageSource) Reference() types.ImageReference {
	return s.ref
}

// Close removes resources associated with an initialized ImageSource, if any.
func (s *dockerImageSource) Close() {
}

// simplifyContentType drops parameters from a HTTP media type (see https://tools.ietf.org/html/rfc7231#section-3.1.1.1)
// Alternatively, an empty string is returned unchanged, and invalid values are "simplified" to an empty string.
func simplifyContentType(contentType string) string {
	if contentType == "" {
		return contentType
	}
	mimeType, _, err := mime.ParseMediaType(contentType)
	if err != nil {
		return ""
	}
	return mimeType
}

func (s *dockerImageSource) GetManifest() ([]byte, string, error) {
	err := s.ensureManifestIsLoaded()
	if err != nil {
		return nil, "", err
	}
	return s.cachedManifest, s.cachedManifestMIMEType, nil
}

func (s *dockerImageSource) fetchManifest(tagOrDigest string) ([]byte, string, error) {
	url := fmt.Sprintf(manifestURL, s.ref.ref.RemoteName(), tagOrDigest)
	headers := make(map[string][]string)
	headers["Accept"] = s.requestedManifestMIMETypes
	res, err := s.c.makeRequest("GET", url, headers, nil)
	if err != nil {
		return nil, "", err
	}
	defer res.Body.Close()
	if res.StatusCode != http.StatusOK {
		return nil, "", client.HandleErrorResponse(res)
	}
	manblob, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, "", err
	}
	return manblob, simplifyContentType(res.Header.Get("Content-Type")), nil
}

// GetTargetManifest returns an image's manifest given a digest.
// This is mainly used to retrieve a single image's manifest out of a manifest list.
func (s *dockerImageSource) GetTargetManifest(digest string) ([]byte, string, error) {
	return s.fetchManifest(digest)
}

// ensureManifestIsLoaded sets s.cachedManifest and s.cachedManifestMIMEType
//
// ImageSource implementations are not required or expected to do any caching,
// but because our signatures are “attached” to the manifest digest,
// we need to ensure that the digest of the manifest returned by GetManifest
// and used by GetSignatures are consistent, otherwise we would get spurious
// signature verification failures when pulling while a tag is being updated.
func (s *dockerImageSource) ensureManifestIsLoaded() error {
	if s.cachedManifest != nil {
		return nil
	}

	reference, err := s.ref.tagOrDigest()
	if err != nil {
		return err
	}

	manblob, mt, err := s.fetchManifest(reference)
	if err != nil {
		return err
	}
	// We might validate manblob against the Docker-Content-Digest header here to protect against transport errors.
	s.cachedManifest = manblob
	s.cachedManifestMIMEType = mt
	return nil
}

// GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown).
func (s *dockerImageSource) GetBlob(digest string) (io.ReadCloser, int64, error) {
	url := fmt.Sprintf(blobsURL, s.ref.ref.RemoteName(), digest)
	logrus.Debugf("Downloading %s", url)
	res, err := s.c.makeRequest("GET", url, nil, nil)
	if err != nil {
		return nil, 0, err
	}
	if res.StatusCode != http.StatusOK {
		// print url also
		return nil, 0, fmt.Errorf("Invalid status code returned when fetching blob %d", res.StatusCode)
	}
	size, err := strconv.ParseInt(res.Header.Get("Content-Length"), 10, 64)
	if err != nil {
		size = -1
	}
	return res.Body, size, nil
}

func (s *dockerImageSource) GetSignatures() ([][]byte, error) {
	if s.c.signatureBase == nil { // Skip dealing with the manifest digest if not necessary.
		return [][]byte{}, nil
	}

	if err := s.ensureManifestIsLoaded(); err != nil {
		return nil, err
	}
	manifestDigest, err := manifest.Digest(s.cachedManifest)
	if err != nil {
		return nil, err
	}

	signatures := [][]byte{}
	for i := 0; ; i++ {
		url := signatureStorageURL(s.c.signatureBase, manifestDigest, i)
		if url == nil {
			return nil, fmt.Errorf("Internal error: signatureStorageURL with non-nil base returned nil")
		}
		signature, missing, err := s.getOneSignature(url)
		if err != nil {
			return nil, err
		}
		if missing {
			break
		}
		signatures = append(signatures, signature)
	}
	return signatures, nil
}

// getOneSignature downloads one signature from url.
// If it successfully determines that the signature does not exist, returns with missing set to true and error set to nil.
func (s *dockerImageSource) getOneSignature(url *url.URL) (signature []byte, missing bool, err error) {
	switch url.Scheme {
	case "file":
		logrus.Debugf("Reading %s", url.Path)
		sig, err := ioutil.ReadFile(url.Path)
		if err != nil {
			if os.IsNotExist(err) {
				return nil, true, nil
			}
			return nil, false, err
		}
		return sig, false, nil

	case "http", "https":
		logrus.Debugf("GET %s", url)
		res, err := s.c.client.Get(url.String())
		if err != nil {
			return nil, false, err
		}
		defer res.Body.Close()
		if res.StatusCode == http.StatusNotFound {
			return nil, true, nil
		} else if res.StatusCode != http.StatusOK {
			return nil, false, fmt.Errorf("Error reading signature from %s: status %d", url.String(), res.StatusCode)
		}
		sig, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return nil, false, err
		}
		return sig, false, nil

	default:
		return nil, false, fmt.Errorf("Unsupported scheme when reading signature from %s", url.String())
	}
}

// deleteImage deletes the named image from the registry, if supported.
func deleteImage(ctx *types.SystemContext, ref dockerReference) error {
	c, err := newDockerClient(ctx, ref, true)
	if err != nil {
		return err
	}

	// When retrieving the digest from a registry >= 2.3 use the following header:
	//   "Accept": "application/vnd.docker.distribution.manifest.v2+json"
	headers := make(map[string][]string)
	headers["Accept"] = []string{manifest.DockerV2Schema2MediaType}

	reference, err := ref.tagOrDigest()
	if err != nil {
		return err
	}
	getURL := fmt.Sprintf(manifestURL, ref.ref.RemoteName(), reference)
	get, err := c.makeRequest("GET", getURL, headers, nil)
	if err != nil {
		return err
	}
	defer get.Body.Close()
	manifestBody, err := ioutil.ReadAll(get.Body)
	if err != nil {
		return err
	}
	switch get.StatusCode {
	case http.StatusOK:
	case http.StatusNotFound:
		return fmt.Errorf("Unable to delete %v. Image may not exist or is not stored with a v2 Schema in a v2 registry.", ref.ref)
	default:
		return fmt.Errorf("Failed to delete %v: %s (%v)", ref.ref, manifestBody, get.Status)
	}

	digest := get.Header.Get("Docker-Content-Digest")
	deleteURL := fmt.Sprintf(manifestURL, ref.ref.RemoteName(), digest)

	// When retrieving the digest from a registry >= 2.3 use the following header:
	//   "Accept": "application/vnd.docker.distribution.manifest.v2+json"
	delete, err := c.makeRequest("DELETE", deleteURL, headers, nil)
	if err != nil {
		return err
	}
	defer delete.Body.Close()

	body, err := ioutil.ReadAll(delete.Body)
	if err != nil {
		return err
	}
	if delete.StatusCode != http.StatusAccepted {
		return fmt.Errorf("Failed to delete %v: %s (%v)", deleteURL, string(body), delete.Status)
	}

	if c.signatureBase != nil {
		manifestDigest, err := manifest.Digest(manifestBody)
		if err != nil {
			return err
		}

		for i := 0; ; i++ {
			url := signatureStorageURL(c.signatureBase, manifestDigest, i)
			if url == nil {
				return fmt.Errorf("Internal error: signatureStorageURL with non-nil base returned nil")
			}
			missing, err := c.deleteOneSignature(url)
			if err != nil {
				return err
			}
			if missing {
				break
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package shell

import (
	"io/ioutil"
	"os"
	"runtime"
	"testing"

	"github.com/buildkite/agent/bootstrap/shell"
	"github.com/buildkite/agent/v3/env"
)

// NewTestShell creates a minimal shell suitable for tests.
func NewTestShell(t *testing.T) *Shell {
	sh, err := New()
	if err != nil {
		t.Fatal(err)
	}

	sh.Logger = shell.DiscardLogger
	sh.Writer = ioutil.Discard

	if os.Getenv(`DEBUG_SHELL`) == "1" {
		sh.Logger = shell.TestingLogger{T: t}
	}

	// Windows requires certain env variables to be present
	if runtime.GOOS == "windows" {
		sh.Env = env.FromSlice([]string{
			//			"PATH=" + os.Getenv("PATH"),
			"SystemRoot=" + os.Getenv("SystemRoot"),
			"WINDIR=" + os.Getenv("WINDIR"),
			"COMSPEC=" + os.Getenv("COMSPEC"),
			"PATHEXT=" + os.Getenv("PATHEXT"),
			"TMP=" + os.Getenv("TMP"),
			"TEMP=" + os.Getenv("TEMP"),
			"ProgramData=" + os.Getenv("ProgramData"),
		})
	} else {
		sh.Env = env.New()
	}

	return sh
}
<commit_msg>NewTestShell: remove self-import.<commit_after>package shell

import (
	"io/ioutil"
	"os"
	"runtime"
	"testing"

	"github.com/buildkite/agent/v3/env"
)

// NewTestShell creates a minimal shell suitable for tests.
func NewTestShell(t *testing.T) *Shell {
	sh, err := New()
	if err != nil {
		t.Fatal(err)
	}

	sh.Logger = DiscardLogger
	sh.Writer = ioutil.Discard

	if os.Getenv(`DEBUG_SHELL`) == "1" {
		sh.Logger = TestingLogger{T: t}
	}

	// Windows requires certain env variables to be present
	if runtime.GOOS == "windows" {
		sh.Env = env.FromSlice([]string{
			//			"PATH=" + os.Getenv("PATH"),
			"SystemRoot=" + os.Getenv("SystemRoot"),
			"WINDIR=" + os.Getenv("WINDIR"),
			"COMSPEC=" + os.Getenv("COMSPEC"),
			"PATHEXT=" + os.Getenv("PATHEXT"),
			"TMP=" + os.Getenv("TMP"),
			"TEMP=" + os.Getenv("TEMP"),
			"ProgramData=" + os.Getenv("ProgramData"),
		})
	} else {
		sh.Env = env.New()
	}

	return sh
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"time"

	"github.com/mailgun/oxy/forward"
	"github.com/mailgun/oxy/roundrobin"
	"github.com/mailgun/oxy/testutils"
)

type MarathonTasks struct {
	Tasks []struct {
		AppId              string `json:"appId"`
		HealthCheckResults []struct {
			Alive bool `json:"alive"`
		} `json:"healthCheckResults"`
		Host         string  `json:"host"`
		Id           string  `json:"id"`
		Ports        []int64 `json:"ports"`
		ServicePorts []int64 `json:"servicePorts"`
		StagedAt     string  `json:"stagedAt"`
		StartedAt    string  `json:"startedAt"`
		Version      string  `json:"version"`
	} `json:"tasks"`
}

type MarathonApps struct {
	Apps []struct {
		Id           string            `json:"id"`
		Labels       map[string]string `json:"labels"`
		HealthChecks []interface{}     `json:"healthChecks"`
	} `json:"apps"`
}

// buffer of two, because we dont really need more.
var callbackqueue = make(chan bool, 2)

func callbackworker() {
	// a ticker channel to limit reloads to marathon, 1s is enough for now.
	ticker := time.NewTicker(1000 * time.Millisecond)
	go func() {
		for {
			select {
			case <-ticker.C:
				<-callbackqueue
				err := reload()
				if err != nil {
					log.Println(err.Error())
				} else {
					log.Println("config updated")
				}
			}
		}
	}()
}

func createBackup(jsontasks *MarathonTasks, jsonapps *MarathonApps) error {
	backup, err := json.MarshalIndent(jsontasks, "", "  ")
	if err != nil {
		return err
	}
	err = ioutil.WriteFile("/tmp/.moxy.tasks.tmp", backup, 0644)
	if err != nil {
		return err
	}
	backup, err = json.MarshalIndent(jsonapps, "", "  ")
	if err != nil {
		return err
	}
	err = ioutil.WriteFile("/tmp/.moxy.apps.tmp", backup, 0644)
	if err != nil {
		return err
	}
	return nil
}

func loadBackup(jsontasks *MarathonTasks, jsonapps *MarathonApps) error {
	file, err := ioutil.ReadFile("/tmp/.moxy.tasks.tmp")
	if err != nil {
		return err
	}
	err = json.Unmarshal(file, jsontasks)
	if err != nil {
		return err
	}
	file, err = ioutil.ReadFile("/tmp/.moxy.apps.tmp")
	if err != nil {
		return err
	}
	err = json.Unmarshal(file, jsonapps)
	if err != nil {
		return err
	}
	return nil
}

func fetchApps(jsontasks *MarathonTasks, jsonapps *MarathonApps) error {
	client := &http.Client{
		Timeout: 3 * time.Second,
	}
	r, _ := http.NewRequest("GET", config.Marathon+"/v2/tasks", nil)
	r.Header.Set("Accept", "application/json")
	resp, err := client.Do(r)
	if err != nil {
		return err
	}
	decoder := json.NewDecoder(resp.Body)
	err = decoder.Decode(&jsontasks)
	if err != nil {
		return err
	}
	r, _ = http.NewRequest("GET", config.Marathon+"/v2/apps", nil)
	r.Header.Set("Accept", "application/json")
	resp, err = client.Do(r)
	if err != nil {
		return err
	}
	decoder = json.NewDecoder(resp.Body)
	err = decoder.Decode(&jsonapps)
	if err != nil {
		return err
	}
	return nil
}

func syncApps(jsontasks *MarathonTasks, jsonapps *MarathonApps) {
	apps = Apps{Apps: make(map[string]App)}
	apps.Lock()
	defer apps.Unlock()
	for _, task := range jsontasks.Tasks {
		// Use regex to remove characters that are not allowed in hostnames
		re := regexp.MustCompile("[^0-9a-z-]")
		appid := re.ReplaceAllLiteralString(task.AppId, "")
		apphealth := false
		for _, v := range jsonapps.Apps {
			if v.Id == task.AppId {
				if s, ok := v.Labels["moxy_subdomain"]; ok {
					appid = s
				}
				if len(v.HealthChecks) > 0 {
					apphealth = true
				}
			}
		}
		if apphealth {
			if len(task.HealthCheckResults) == 0 {
				// this means tasks is being deployed but not yet monitored as alive. Assume down.
				continue
			}
			alive := true
			for _, health := range task.HealthCheckResults {
				// check if health check is alive
				if health.Alive == false {
					alive = false
				}
			}
			if alive != true {
				// at least one health check has failed. Assume down.
				continue
			}
		}
		if s, ok := apps.Apps[appid]; ok {
			s.Lb.UpsertServer(testutils.ParseURI("http://" + task.Host + ":" + strconv.FormatInt(task.Ports[0], 10)))
			s.Tasks = append(s.Tasks, task.Host+":"+strconv.FormatInt(task.Ports[0], 10))
			apps.Apps[appid] = s
		} else {
			var s = App{}
			s.Fwd, _ = forward.New(forward.PassHostHeader(true))
			s.Lb, _ = roundrobin.New(s.Fwd)
			s.Lb.UpsertServer(testutils.ParseURI("http://" + task.Host + ":" + strconv.FormatInt(task.Ports[0], 10)))
			s.Tasks = []string{task.Host + ":" + strconv.FormatInt(task.Ports[0], 10)}
			apps.Apps[appid] = s
		}
	}
}

func reload() error {
	jsontasks := MarathonTasks{}
	jsonapps := MarathonApps{}
	err := fetchApps(&jsontasks, &jsonapps)
	if err != nil {
		log.Println("Unable to sync from Marathon:", err)
		err = loadBackup(&jsontasks, &jsonapps)
		if err != nil {
			return err
		}
		log.Println("successfully loaded backup config.")
	}
	syncApps(&jsontasks, &jsonapps)
	// We write a backup to disk, this permits us to restart moxy even if Marathon is down using last working config.
	err = createBackup(&jsontasks, &jsonapps)
	if err != nil {
		log.Println("Unable to create backup:", err)
	}
	return nil
}
<commit_msg>closing the response body, possible memory leak.<commit_after>package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"time"

	"github.com/mailgun/oxy/forward"
	"github.com/mailgun/oxy/roundrobin"
	"github.com/mailgun/oxy/testutils"
)

type MarathonTasks struct {
	Tasks []struct {
		AppId              string `json:"appId"`
		HealthCheckResults []struct {
			Alive bool `json:"alive"`
		} `json:"healthCheckResults"`
		Host         string  `json:"host"`
		Id           string  `json:"id"`
		Ports        []int64 `json:"ports"`
		ServicePorts []int64 `json:"servicePorts"`
		StagedAt     string  `json:"stagedAt"`
		StartedAt    string  `json:"startedAt"`
		Version      string  `json:"version"`
	} `json:"tasks"`
}

type MarathonApps struct {
	Apps []struct {
		Id           string            `json:"id"`
		Labels       map[string]string `json:"labels"`
		HealthChecks []interface{}     `json:"healthChecks"`
	} `json:"apps"`
}

// buffer of two, because we dont really need more.
var callbackqueue = make(chan bool, 2)

func callbackworker() {
	// a ticker channel to limit reloads to marathon, 1s is enough for now.
	ticker := time.NewTicker(1000 * time.Millisecond)
	go func() {
		for {
			select {
			case <-ticker.C:
				<-callbackqueue
				err := reload()
				if err != nil {
					log.Println(err.Error())
				} else {
					log.Println("config updated")
				}
			}
		}
	}()
}

func createBackup(jsontasks *MarathonTasks, jsonapps *MarathonApps) error {
	backup, err := json.MarshalIndent(jsontasks, "", "  ")
	if err != nil {
		return err
	}
	err = ioutil.WriteFile("/tmp/.moxy.tasks.tmp", backup, 0644)
	if err != nil {
		return err
	}
	backup, err = json.MarshalIndent(jsonapps, "", "  ")
	if err != nil {
		return err
	}
	err = ioutil.WriteFile("/tmp/.moxy.apps.tmp", backup, 0644)
	if err != nil {
		return err
	}
	return nil
}

func loadBackup(jsontasks *MarathonTasks, jsonapps *MarathonApps) error {
	file, err := ioutil.ReadFile("/tmp/.moxy.tasks.tmp")
	if err != nil {
		return err
	}
	err = json.Unmarshal(file, jsontasks)
	if err != nil {
		return err
	}
	file, err = ioutil.ReadFile("/tmp/.moxy.apps.tmp")
	if err != nil {
		return err
	}
	err = json.Unmarshal(file, jsonapps)
	if err != nil {
		return err
	}
	return nil
}

func fetchApps(jsontasks *MarathonTasks, jsonapps *MarathonApps) error {
	client := &http.Client{
		Timeout: 3 * time.Second,
	}
	r, _ := http.NewRequest("GET", config.Marathon+"/v2/tasks", nil)
	r.Header.Set("Accept", "application/json")
	resp, err := client.Do(r)
	defer resp.Body.Close()
	if err != nil {
		return err
	}
	decoder := json.NewDecoder(resp.Body)
	err = decoder.Decode(&jsontasks)
	if err != nil {
		return err
	}
	r, _ = http.NewRequest("GET", config.Marathon+"/v2/apps", nil)
	r.Header.Set("Accept", "application/json")
	resp, err = client.Do(r)
	defer resp.Body.Close()
	if err != nil {
		return err
	}
	decoder = json.NewDecoder(resp.Body)
	err = decoder.Decode(&jsonapps)
	if err != nil {
		return err
	}
	return nil
}

func syncApps(jsontasks *MarathonTasks, jsonapps *MarathonApps) {
	apps = Apps{Apps: make(map[string]App)}
	apps.Lock()
	defer apps.Unlock()
	for _, task := range jsontasks.Tasks {
		// Use regex to remove characters that are not allowed in hostnames
		re := regexp.MustCompile("[^0-9a-z-]")
		appid := re.ReplaceAllLiteralString(task.AppId, "")
		apphealth := false
		for _, v := range jsonapps.Apps {
			if v.Id == task.AppId {
				if s, ok := v.Labels["moxy_subdomain"]; ok {
					appid = s
				}
				if len(v.HealthChecks) > 0 {
					apphealth = true
				}
			}
		}
		if apphealth {
			if len(task.HealthCheckResults) == 0 {
				// this means tasks is being deployed but not yet monitored as alive. Assume down.
				continue
			}
			alive := true
			for _, health := range task.HealthCheckResults {
				// check if health check is alive
				if health.Alive == false {
					alive = false
				}
			}
			if alive != true {
				// at least one health check has failed. Assume down.
				continue
			}
		}
		if s, ok := apps.Apps[appid]; ok {
			s.Lb.UpsertServer(testutils.ParseURI("http://" + task.Host + ":" + strconv.FormatInt(task.Ports[0], 10)))
			s.Tasks = append(s.Tasks, task.Host+":"+strconv.FormatInt(task.Ports[0], 10))
			apps.Apps[appid] = s
		} else {
			var s = App{}
			s.Fwd, _ = forward.New(forward.PassHostHeader(true))
			s.Lb, _ = roundrobin.New(s.Fwd)
			s.Lb.UpsertServer(testutils.ParseURI("http://" + task.Host + ":" + strconv.FormatInt(task.Ports[0], 10)))
			s.Tasks = []string{task.Host + ":" + strconv.FormatInt(task.Ports[0], 10)}
			apps.Apps[appid] = s
		}
	}
}

func reload() error {
	jsontasks := MarathonTasks{}
	jsonapps := MarathonApps{}
	err := fetchApps(&jsontasks, &jsonapps)
	if err != nil {
		log.Println("Unable to sync from Marathon:", err)
		err = loadBackup(&jsontasks, &jsonapps)
		if err != nil {
			return err
		}
		log.Println("successfully loaded backup config.")
	}
	syncApps(&jsontasks, &jsonapps)
	// We write a backup to disk, this permits us to restart moxy even if Marathon is down using last working config.
	err = createBackup(&jsontasks, &jsonapps)
	if err != nil {
		log.Println("Unable to create backup:", err)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"sync"

	"github.com/vbauerster/mpb"
	"github.com/vbauerster/mpb/decor"
)

func main() {
	log.SetOutput(os.Stderr)

	url1 := "https://homebrew.bintray.com/bottles/youtube-dl-2016.12.12.sierra.bottle.tar.gz"
	url2 := "https://homebrew.bintray.com/bottles/libtiff-4.0.7.sierra.bottle.tar.gz"

	var wg sync.WaitGroup
	p := mpb.New(mpb.WithWaitGroup(&wg))

	for i, url := range [...]string{url1, url2} {
		wg.Add(1)
		name := fmt.Sprintf("url%d:", i+1)
		go download(&wg, p, name, url)
	}

	p.Stop()
	fmt.Println("Finished")
}

func download(wg *sync.WaitGroup, p *mpb.Progress, name, url string) {
	defer wg.Done()
	resp, err := http.Get(url)
	if err != nil {
		log.Printf("%s: %v", name, err)
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		err = fmt.Errorf("non-200 status: %s", resp.Status)
		log.Printf("%s: %v", name, err)
		return
	}

	size := resp.ContentLength

	// create dest
	destName := filepath.Base(url)
	dest, err := os.Create(destName)
	if err != nil {
		err = fmt.Errorf("Can't create %s: %v", destName, err)
		log.Printf("%s: %v", name, err)
		return
	}

	// create bar with appropriate decorators
	bar := p.AddBar(size,
		mpb.PrependDecorators(
			decor.StaticName(name, 0, 0),
			decor.CountersKibiByte("%6.1f / %6.1f", 18, 0),
		),
		mpb.AppendDecorators(decor.ETA(5, decor.DwidthSync)),
	)

	// create proxy reader
	reader := bar.ProxyReader(resp.Body)
	// and copy from reader
	_, err = io.Copy(dest, reader)

	if closeErr := dest.Close(); err == nil {
		err = closeErr
	}
	if err != nil {
		log.Printf("%s: %v", name, err)
	}
}
<commit_msg>Preserve bar's ordrer showcase<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"sync"

	"github.com/vbauerster/mpb"
	"github.com/vbauerster/mpb/decor"
)

func main() {
	log.SetOutput(os.Stderr)

	url1 := "https://homebrew.bintray.com/bottles/youtube-dl-2016.12.12.sierra.bottle.tar.gz"
	url2 := "https://homebrew.bintray.com/bottles/libtiff-4.0.7.sierra.bottle.tar.gz"

	var wg sync.WaitGroup
	p := mpb.New(mpb.WithWaitGroup(&wg))

	for i, url := range [...]string{url1, url2} {
		wg.Add(1)
		name := fmt.Sprintf("url%d:", i+1)
		go download(&wg, p, name, url, i)
	}

	p.Stop()
	fmt.Println("Finished")
}

func download(wg *sync.WaitGroup, p *mpb.Progress, name, url string, n int) {
	defer wg.Done()
	resp, err := http.Get(url)
	if err != nil {
		log.Printf("%s: %v", name, err)
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		err = fmt.Errorf("non-200 status: %s", resp.Status)
		log.Printf("%s: %v", name, err)
		return
	}

	size := resp.ContentLength

	// create dest
	destName := filepath.Base(url)
	dest, err := os.Create(destName)
	if err != nil {
		err = fmt.Errorf("Can't create %s: %v", destName, err)
		log.Printf("%s: %v", name, err)
		return
	}

	// create bar with appropriate decorators
	bar := p.AddBar(size,
		mpb.PrependDecorators(
			decor.StaticName(name, 0, 0),
			decor.CountersKibiByte("%6.1f / %6.1f", 18, 0),
		),
		mpb.AppendDecorators(decor.ETA(5, decor.DwidthSync)),
	)
	// Respect the order
	p.UpdateBarPriority(bar, n)

	// create proxy reader
	reader := bar.ProxyReader(resp.Body)
	// and copy from reader
	_, err = io.Copy(dest, reader)

	if e := dest.Close(); err == nil {
		err = e
	}
	if err != nil {
		log.Printf("%s: %v", name, err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	tc "github.com/guidewire/teamcity-go-bindings"
	"github.com/orcaman/concurrent-map"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/version"
	"github.com/sirupsen/logrus"
	"net/http"
	_ "net/http/pprof"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	namespace = "teamcity"
)

var metricsStorage = cmap.New()

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	prometheus.MustRegister(version.NewCollector("teamcity_exporter"))
}

func main() {
	var (
		showVersion   = flag.Bool("version", false, "Print version information")
		listenAddress = flag.String("web.listen-address", ":9107", "Address to listen on for web interface and telemetry")
		metricsPath   = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics")
		logLevel      = flag.String("logLevel", "info", "Changes log level, available values: info/debug")
		configPath    = flag.String("config", "config.yaml", "Path to configuration file")
	)
	flag.Parse()

	switch {
	case *logLevel == "debug":
		logrus.SetLevel(logrus.DebugLevel)
		logrus.Debug("Debug logging was enabled")
	default:
		logrus.SetLevel(logrus.InfoLevel)
	}

	if *showVersion {
		fmt.Fprintln(os.Stdout, version.Print("teamcity_exporter"))
		os.Exit(0)
	}

	config := Configuration{}
	err := config.parseConfig(*configPath)
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"config": *configPath,
		}).Error(err)
		os.Exit(1)
	}

	logrus.Info("Starting teamcity_exporter" + version.Info())
	logrus.Info("Build context", version.BuildContext())

	collector := NewCollector()
	prometheus.MustRegister(collector)

	for i := range config.Instances {
		logrus.WithFields(logrus.Fields{
			"instance":       config.Instances[i].Name,
			"scrapeInterval": config.Instances[i].ScrapeInterval,
		}).Debug("Found Teamcity instance, preparing for metrics collection")
		go collectInstancesStat(config.Instances[i])
	}

	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
					 <title>Teamcity Exporter</title>

Teamcity Exporter

<a href='` + *metricsPath + `'>Metrics

					 `))
	})
	log.Infoln("Listening on", *listenAddress)
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}

func collectInstancesStat(i Instance) {
	client := tc.New(i.URL, i.Username, i.Password)
	wg := &sync.WaitGroup{}

	ticker := newTicker(time.Duration(i.ScrapeInterval) * time.Second)
	for _ = range ticker.C {
		logrus.WithFields(logrus.Fields{
			"instance":    i.Name,
			"timestamp":   time.Now().Unix(),
			"storageSize": len(metricsStorage.Keys()),
		}).Debug("Got message from ticker, starting metrics collection")
		logrus.WithFields(logrus.Fields{
			"instance": i.Name,
		}).Debug(fmt.Sprintf("Found %d build filters for instance, looping against filters", len(i.BuildsFilters)))
		for v := range i.BuildsFilters {
			logrus.WithFields(logsFormatter(i.BuildsFilters[v].Filter)).WithFields(logrus.Fields{
				"instance": i.Name,
			}).Debug("Found build filter, preparing request for Teamcity")
			if i.BuildsFilters[v].Filter.BuildType != "" {
				wg.Add(1)
				go collectBuildsStat(client, i.Name, i.BuildsFilters[v], wg)
			} else {
				logrus.WithFields(logrus.Fields{
					"instance": i.Name,
				}).Debug("Filter has no build configuration specified, will run against all available configurations")
				buildCfgs, err := client.GetAllBuildConfigurations()
				if err != nil {
					logrus.WithFields(logrus.Fields{
						"instance": i.URL,
					}).Error(err)
				}
				for z := range buildCfgs.BuildType {
					f := i.BuildsFilters[v]
					f.Filter.BuildType = buildCfgs.BuildType[z].ID
					logrus.WithFields(logsFormatter(f)).WithFields(logrus.Fields{
						"instance": i.Name,
					}).Debug("Found build configuration")
					wg.Add(1)
					go collectBuildsStat(client, i.Name, f, wg)
				}
			}
		}
		wg.Wait()
		logrus.WithFields(logrus.Fields{
			"instance": i.Name,
		}).Debug("Scraping job finished, waiting for a signal from ticker")
	}
}

func collectBuildsStat(c *tc.Client, inst string, filter BuildFilter, wg *sync.WaitGroup) {
	defer wg.Done()

	stat, err := c.GetBuildStatistics(filter.Filter)
	if err != nil {
		logrus.WithFields(logsFormatter(filter.Filter)).WithFields(logrus.Fields{
			"instance": inst,
			"filter":   filter.Name,
		}).Error(err)
	}

	logrus.WithFields(logsFormatter(filter.Filter)).WithFields(logrus.Fields{
		"instance":        inst,
		"filter":          filter.Name,
		"metricsGathered": stat.Count,
	}).Debug("Gathered build metrics based on provided filter")

	for i := range stat.Property {
		value, _ := strconv.ParseFloat(stat.Property[i].Value, 64)
		metric := strings.SplitN(stat.Property[i].Name, ":", 2)
		title := toSnakeCase(metric[0])

		labels := []Label{
			{"exporter_instance", inst},
			{"exporter_filter", filter.Name},
			{"build_configuration", stat.UsedFilter.BuildType},
		}
		if len(metric) > 1 {
			labels = append(labels, Label{"other", metric[1]})
		}

		labelsTitles, labelsValues := []string{}, []string{}
		for i := range labels {
			labelsTitles = append(labelsTitles, labels[i].Name)
			labelsValues = append(labelsValues, labels[i].Value)
		}

		desc := prometheus.NewDesc(title, title, labelsTitles, nil)
		metricsStorage.Set(getHash(title, labelsValues...), prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, value, labelsValues...))
		logrus.WithFields(logrus.Fields{
			"name":  title,
			"value": value,
		}).Debug("Saving metric to temporary storage")
	}
}
<commit_msg>Fix -log.level flag<commit_after>package main

import (
	"flag"
	"fmt"
	tc "github.com/guidewire/teamcity-go-bindings"
	"github.com/orcaman/concurrent-map"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/version"
	"github.com/sirupsen/logrus"
	"net/http"
	_ "net/http/pprof"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	namespace = "teamcity"
)

var metricsStorage = cmap.New()

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	prometheus.MustRegister(version.NewCollector("teamcity_exporter"))
}

func main() {
	var (
		showVersion   = flag.Bool("version", false, "Print version information")
		listenAddress = flag.String("web.listen-address", ":9107", "Address to listen on for web interface and telemetry")
		metricsPath   = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics")
		configPath    = flag.String("config", "config.yaml", "Path to configuration file")
	)
	flag.Parse()

	logrus.Info("Starting teamcity_exporter" + version.Info())
	logrus.Info("Build context", version.BuildContext())

	level, err := logrus.ParseLevel(strings.Replace(flag.Lookup("log.level").Value.String(), "\"", "", -1))
	if err != nil {
		logrus.Error(err)
		os.Exit(1)
	}
	logrus.SetLevel(level)
	logrus.Infof("Log level was set to %s", level.String())

	if *showVersion {
		fmt.Fprintln(os.Stdout, version.Print("teamcity_exporter"))
		os.Exit(0)
	}

	config := Configuration{}
	err = config.parseConfig(*configPath)
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"config": *configPath,
		}).Error(err)
		os.Exit(1)
	}

	collector := NewCollector()
	prometheus.MustRegister(collector)

	for i := range config.Instances {
		logrus.WithFields(logrus.Fields{
			"instance":       config.Instances[i].Name,
			"scrapeInterval": config.Instances[i].ScrapeInterval,
		}).Debug("Found Teamcity instance, preparing for metrics collection")
		go collectInstancesStat(config.Instances[i])
	}

	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
					 <title>Teamcity Exporter</title>

Teamcity Exporter

<a href='` + *metricsPath + `'>Metrics

					 `))
	})
	log.Infoln("Listening on", *listenAddress)
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}

func collectInstancesStat(i Instance) {
	client := tc.New(i.URL, i.Username, i.Password)
	wg := &sync.WaitGroup{}

	ticker := newTicker(time.Duration(i.ScrapeInterval) * time.Second)
	for _ = range ticker.C {
		logrus.WithFields(logrus.Fields{
			"instance":    i.Name,
			"timestamp":   time.Now().Unix(),
			"storageSize": len(metricsStorage.Keys()),
		}).Debug("Got message from ticker, starting metrics collection")
		logrus.WithFields(logrus.Fields{
			"instance": i.Name,
		}).Debug(fmt.Sprintf("Found %d build filters for instance, looping against filters", len(i.BuildsFilters)))
		for v := range i.BuildsFilters {
			logrus.WithFields(logsFormatter(i.BuildsFilters[v].Filter)).WithFields(logrus.Fields{
				"instance": i.Name,
			}).Debug("Found build filter, preparing request for Teamcity")
			if i.BuildsFilters[v].Filter.BuildType != "" {
				wg.Add(1)
				go collectBuildsStat(client, i.Name, i.BuildsFilters[v], wg)
			} else {
				logrus.WithFields(logrus.Fields{
					"instance": i.Name,
				}).Debug("Filter has no build configuration specified, will run against all available configurations")
				buildCfgs, err := client.GetAllBuildConfigurations()
				if err != nil {
					logrus.WithFields(logrus.Fields{
						"instance": i.URL,
					}).Error(err)
				}
				for z := range buildCfgs.BuildType {
					f := i.BuildsFilters[v]
					f.Filter.BuildType = buildCfgs.BuildType[z].ID
					logrus.WithFields(logsFormatter(f)).WithFields(logrus.Fields{
						"instance": i.Name,
					}).Debug("Found build configuration")
					wg.Add(1)
					go collectBuildsStat(client, i.Name, f, wg)
				}
			}
		}
		wg.Wait()
		logrus.WithFields(logrus.Fields{
			"instance": i.Name,
		}).Debug("Scraping job finished, waiting for a signal from ticker")
	}
}

func collectBuildsStat(c *tc.Client, inst string, filter BuildFilter, wg *sync.WaitGroup) {
	defer wg.Done()

	stat, err := c.GetBuildStatistics(filter.Filter)
	if err != nil {
		logrus.WithFields(logsFormatter(filter.Filter)).WithFields(logrus.Fields{
			"instance": inst,
			"filter":   filter.Name,
		}).Error(err)
	}

	logrus.WithFields(logsFormatter(filter.Filter)).WithFields(logrus.Fields{
		"instance":        inst,
		"filter":          filter.Name,
		"metricsGathered": stat.Count,
	}).Debug("Gathered build metrics based on provided filter")

	for i := range stat.Property {
		value, _ := strconv.ParseFloat(stat.Property[i].Value, 64)
		metric := strings.SplitN(stat.Property[i].Name, ":", 2)
		title := toSnakeCase(metric[0])

		labels := []Label{
			{"exporter_instance", inst},
			{"exporter_filter", filter.Name},
			{"build_configuration", stat.UsedFilter.BuildType},
		}
		if len(metric) > 1 {
			labels = append(labels, Label{"other", metric[1]})
		}

		labelsTitles, labelsValues := []string{}, []string{}
		for i := range labels {
			labelsTitles = append(labelsTitles, labels[i].Name)
			labelsValues = append(labelsValues, labels[i].Value)
		}

		desc := prometheus.NewDesc(title, title, labelsTitles, nil)
		metricsStorage.Set(getHash(title, labelsValues...), prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, value, labelsValues...))
		logrus.WithFields(logrus.Fields{
			"name":  title,
			"value": value,
		}).Debug("Saving metric to temporary storage")
	}
}
<|endoftext|>


<commit_before>// Package tinkerforge implements the tinkerforge protocol
// Author: Tim Scheuermann (https://github.com/noxer)
package tinkerforge

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"net"
	"sync"
	"time"
)

// Handler to get callbacks
type Handler interface {
	Handle(packet *Packet)
}

// respHandler for getting responses back
type respHandler struct {
	c chan *Packet
	t time.Duration
}

// Handles responses
func (r respHandler) Handle(p *Packet) {
	// No timeout provided (don't!)
	if r.t == 0 {
		r.c <- p
		return
	}

	timer := time.NewTimer(r.t)

	select {
	case r.c <- p:
		timer.Stop()
	case <-timer.C:
		close(r.c)
	}
}

// Tinkerforge interface
type Tinkerforge interface {
	io.Closer
	Handler(uid uint32, funcID uint8, handler Handler)
	Send(packet *Packet) (*Packet, error)
}

// Tinkerforge structure
type tinkerforge struct {
	conn          io.ReadWriteCloser
	seqNum        chan byte
	handlers      map[handlerID]Handler
	handlersMutex sync.RWMutex

	sendQueue chan func()

	done chan struct{}
	wait sync.WaitGroup

	Timeout time.Duration
}

type handlerID struct {
	uid    uint32
	funcID uint8
	seqNum uint8
}

var (
	// ErrTimeout represents a timeout while waiting for a callback
	ErrTimeout = errors.New("Timeout while waiting for callback")
)

// New creates a new tinkerforge client
func New(host string) (Tinkerforge, error) {

	fmt.Println("New.")

	// Set standard host
	if host == "" {
		host = "localhost:4223"
	}

	// Resolve service address
	addr, err := net.ResolveTCPAddr("tcp", host)
	if err != nil {
		return nil, err
	}

	// Connect to service
	conn, err := net.DialTCP("tcp", nil, addr)
	if err != nil {
		return nil, err
	}

	// Build up structure
	tf := &tinkerforge{
		conn:      conn,
		seqNum:    make(chan byte, 8),
		handlers:  make(map[handlerID]Handler),
		sendQueue: make(chan func(), 8),
		done:      make(chan struct{}),
		Timeout:   10 * time.Second,
	}

	// Start the go routines
	tf.wait.Add(3)
	go tf.seqNumGenerator() // Sequence number generator
	go tf.sender()          // Sender (queue for functions to send packets)
	go tf.receiver()        // Receiver

	// Okay
	return tf, nil
}

// Close closes the connection to the tinkerforge service
func (t *tinkerforge) Close() error {

	fmt.Println("Close.")

	// Close the channels
	close(t.done)
	close(t.sendQueue)

	// Close the tcp connection
	if err := t.conn.Close(); err != nil {
		return err
	}

	// Wait for termination
	t.wait.Wait()
	return nil
}

// Send sends a new packet to the service and returns the answer (if an answer is expected)
func (t *tinkerforge) Send(p *Packet) (*Packet, error) {

	fmt.Println("Send.")

	var packets chan *Packet

	errors := make(chan error, 1)

	// Create response channel in case we need it
	if p.ResponseExpected() {
		packets = make(chan *Packet, 1)
	}

	f := func() {

		fmt.Println("Send f.")

		// Generate sequence number
		seqNum := <-t.seqNum

		// Register callback for expected response (if any)
		if p.ResponseExpected() {
			t.handler(p.UID(), p.FunctionID(), seqNum, respHandler{c: packets, t: t.Timeout})
		}

		// Send packet
		if err := p.Serialize(t.conn, seqNum); err != nil {
			errors <- err
			return
		}

		// Close the error channel
		close(errors)
	}

	// Dispatch f
	t.sendQueue <- f

	// An error occurred
	if err := <-errors; err != nil {
		return nil, err
	}

	// Return depeneding of the expected response
	if p.ResponseExpected() {
		result, ok := <-packets
		if ok {
			return result, nil
		}
		// Timeout
		return nil, ErrTimeout
	}

	return nil, nil
}

// Handler registers a new handler for a packet
func (t *tinkerforge) Handler(uid uint32, funcID uint8, h Handler) {
	t.handler(uid, funcID, 0, h)
}

// handler registers any handler (internal)
func (t *tinkerforge) handler(uid uint32, funcID, seqNum uint8, h Handler) {

	fmt.Println("handler.")

	t.handlersMutex.Lock()
	defer t.handlersMutex.Unlock()

	// Make the handler removable
	if h == nil {
		delete(t.handlers, handlerIDFromParam(uid, funcID, seqNum))
		return
	}

	// Add handler
	t.handlers[handlerIDFromParam(uid, funcID, seqNum)] = h
}

// Sequence number generator
func (t *tinkerforge) seqNumGenerator() {
	defer t.wait.Done()

	num := byte(1)
	for {
		select {
		case t.seqNum <- num:
		case <-t.done:
			return
		}
		num++
		if num == 16 {
			num = 1
		}
	}
}

// Sender executes the funktions in sendQueue
func (t *tinkerforge) sender() {
	defer t.wait.Done()

	// Execute all functions
	for f := range t.sendQueue {
		f()
	}
}

// Receiver listens on the TCP connection and exeecutes the handlers accordingly
func (t *tinkerforge) receiver() {
	defer t.wait.Done()

	// Set up scanner
	scanner := bufio.NewScanner(t.conn)
	scanner.Split(scanPacket)

	// Scan for packets
	for scanner.Scan() {
		// Parse the packet
		p, err := readPacket(scanner.Bytes())
		if err != nil {
			fmt.Println(err.Error())
			continue
		}

		// Call the handler
		t.handle(p)

		// Remove handler if it was not a callback
		if !p.Callback() {
			t.handler(p.UID(), p.FunctionID(), p.SequenceNum(), nil)
		}
	}
}

// handle searches for a matching hander for p and executes it
func (t *tinkerforge) handle(p *Packet) {
	t.handlersMutex.RLock()

	fmt.Println(handlerIdFromPacket(p))
	fmt.Println(t.handlers)

	if handler, ok := t.handlers[handlerIDFromPacket(p)]; ok {
		fmt.Println("Found handler exact")

		t.handlersMutex.RUnlock()
		handler.Handle(p)
	} else {
		// Maybe a wildcard?
		if handler, ok = t.handlers[handlerIDFromParam(0, p.FunctionID(), p.SequenceNum())]; ok {
			fmt.Println("Found handler wildcard")

			t.handlersMutex.RUnlock()
			handler.Handle(p)
		}
	}
}

// handlerIdFromParam creates a new handler ID from the params
func handlerIDFromParam(uid uint32, funcID, seqNum uint8) handlerID {
	return handlerID{
		uid:    uid,
		funcID: funcID,
		seqNum: seqNum,
	}
}

// handlerIdFromPacket creates a new handler ID from packet p
func handlerIDFromPacket(p *Packet) handlerID {
	return handlerID{
		uid:    p.UID(),
		funcID: p.FunctionID(),
		seqNum: p.SequenceNum(),
	}
}
<commit_msg>Fix typo in handlerIDFromPacket<commit_after>// Package tinkerforge implements the tinkerforge protocol
// Author: Tim Scheuermann (https://github.com/noxer)
package tinkerforge

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"net"
	"sync"
	"time"
)

// Handler to get callbacks
type Handler interface {
	Handle(packet *Packet)
}

// respHandler for getting responses back
type respHandler struct {
	c chan *Packet
	t time.Duration
}

// Handles responses
func (r respHandler) Handle(p *Packet) {
	// No timeout provided (don't!)
	if r.t == 0 {
		r.c <- p
		return
	}

	timer := time.NewTimer(r.t)

	select {
	case r.c <- p:
		timer.Stop()
	case <-timer.C:
		close(r.c)
	}
}

// Tinkerforge interface
type Tinkerforge interface {
	io.Closer
	Handler(uid uint32, funcID uint8, handler Handler)
	Send(packet *Packet) (*Packet, error)
}

// Tinkerforge structure
type tinkerforge struct {
	conn          io.ReadWriteCloser
	seqNum        chan byte
	handlers      map[handlerID]Handler
	handlersMutex sync.RWMutex

	sendQueue chan func()

	done chan struct{}
	wait sync.WaitGroup

	Timeout time.Duration
}

type handlerID struct {
	uid    uint32
	funcID uint8
	seqNum uint8
}

var (
	// ErrTimeout represents a timeout while waiting for a callback
	ErrTimeout = errors.New("Timeout while waiting for callback")
)

// New creates a new tinkerforge client
func New(host string) (Tinkerforge, error) {

	fmt.Println("New.")

	// Set standard host
	if host == "" {
		host = "localhost:4223"
	}

	// Resolve service address
	addr, err := net.ResolveTCPAddr("tcp", host)
	if err != nil {
		return nil, err
	}

	// Connect to service
	conn, err := net.DialTCP("tcp", nil, addr)
	if err != nil {
		return nil, err
	}

	// Build up structure
	tf := &tinkerforge{
		conn:      conn,
		seqNum:    make(chan byte, 8),
		handlers:  make(map[handlerID]Handler),
		sendQueue: make(chan func(), 8),
		done:      make(chan struct{}),
		Timeout:   10 * time.Second,
	}

	// Start the go routines
	tf.wait.Add(3)
	go tf.seqNumGenerator() // Sequence number generator
	go tf.sender()          // Sender (queue for functions to send packets)
	go tf.receiver()        // Receiver

	// Okay
	return tf, nil
}

// Close closes the connection to the tinkerforge service
func (t *tinkerforge) Close() error {

	fmt.Println("Close.")

	// Close the channels
	close(t.done)
	close(t.sendQueue)

	// Close the tcp connection
	if err := t.conn.Close(); err != nil {
		return err
	}

	// Wait for termination
	t.wait.Wait()
	return nil
}

// Send sends a new packet to the service and returns the answer (if an answer is expected)
func (t *tinkerforge) Send(p *Packet) (*Packet, error) {

	fmt.Println("Send.")

	var packets chan *Packet

	errors := make(chan error, 1)

	// Create response channel in case we need it
	if p.ResponseExpected() {
		packets = make(chan *Packet, 1)
	}

	f := func() {

		fmt.Println("Send f.")

		// Generate sequence number
		seqNum := <-t.seqNum

		// Register callback for expected response (if any)
		if p.ResponseExpected() {
			t.handler(p.UID(), p.FunctionID(), seqNum, respHandler{c: packets, t: t.Timeout})
		}

		// Send packet
		if err := p.Serialize(t.conn, seqNum); err != nil {
			errors <- err
			return
		}

		// Close the error channel
		close(errors)
	}

	// Dispatch f
	t.sendQueue <- f

	// An error occurred
	if err := <-errors; err != nil {
		return nil, err
	}

	// Return depeneding of the expected response
	if p.ResponseExpected() {
		result, ok := <-packets
		if ok {
			return result, nil
		}
		// Timeout
		return nil, ErrTimeout
	}

	return nil, nil
}

// Handler registers a new handler for a packet
func (t *tinkerforge) Handler(uid uint32, funcID uint8, h Handler) {
	t.handler(uid, funcID, 0, h)
}

// handler registers any handler (internal)
func (t *tinkerforge) handler(uid uint32, funcID, seqNum uint8, h Handler) {

	fmt.Println("handler.")

	t.handlersMutex.Lock()
	defer t.handlersMutex.Unlock()

	// Make the handler removable
	if h == nil {
		delete(t.handlers, handlerIDFromParam(uid, funcID, seqNum))
		return
	}

	// Add handler
	t.handlers[handlerIDFromParam(uid, funcID, seqNum)] = h
}

// Sequence number generator
func (t *tinkerforge) seqNumGenerator() {
	defer t.wait.Done()

	num := byte(1)
	for {
		select {
		case t.seqNum <- num:
		case <-t.done:
			return
		}
		num++
		if num == 16 {
			num = 1
		}
	}
}

// Sender executes the funktions in sendQueue
func (t *tinkerforge) sender() {
	defer t.wait.Done()

	// Execute all functions
	for f := range t.sendQueue {
		f()
	}
}

// Receiver listens on the TCP connection and exeecutes the handlers accordingly
func (t *tinkerforge) receiver() {
	defer t.wait.Done()

	// Set up scanner
	scanner := bufio.NewScanner(t.conn)
	scanner.Split(scanPacket)

	// Scan for packets
	for scanner.Scan() {
		// Parse the packet
		p, err := readPacket(scanner.Bytes())
		if err != nil {
			fmt.Println(err.Error())
			continue
		}

		// Call the handler
		t.handle(p)

		// Remove handler if it was not a callback
		if !p.Callback() {
			t.handler(p.UID(), p.FunctionID(), p.SequenceNum(), nil)
		}
	}
}

// handle searches for a matching hander for p and executes it
func (t *tinkerforge) handle(p *Packet) {
	t.handlersMutex.RLock()

	fmt.Println(handlerIDFromPacket(p))
	fmt.Println(t.handlers)

	if handler, ok := t.handlers[handlerIDFromPacket(p)]; ok {
		fmt.Println("Found handler exact")

		t.handlersMutex.RUnlock()
		handler.Handle(p)
	} else {
		// Maybe a wildcard?
		if handler, ok = t.handlers[handlerIDFromParam(0, p.FunctionID(), p.SequenceNum())]; ok {
			fmt.Println("Found handler wildcard")

			t.handlersMutex.RUnlock()
			handler.Handle(p)
		}
	}
}

// handlerIdFromParam creates a new handler ID from the params
func handlerIDFromParam(uid uint32, funcID, seqNum uint8) handlerID {
	return handlerID{
		uid:    uid,
		funcID: funcID,
		seqNum: seqNum,
	}
}

// handlerIdFromPacket creates a new handler ID from packet p
func handlerIDFromPacket(p *Packet) handlerID {
	return handlerID{
		uid:    p.UID(),
		funcID: p.FunctionID(),
		seqNum: p.SequenceNum(),
	}
}
<|endoftext|>


<commit_before>package negotiator

import (
	"errors"
	"net/http/httptest"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestShouldProcessJSONAcceptHeader(t *testing.T) {
	var acceptTests = []struct {
		acceptheader string
	}{
		{"application/json"},
		{"application/json-"},
		{"text/json"},
		{"+json"},
	}

	jsonProcessor := &jsonProcessor{}

	for _, tt := range acceptTests {
		result := jsonProcessor.CanProcess(tt.acceptheader)
		assert.True(t, result, "Should process "+tt.acceptheader)
	}
}

func TestShouldSetContentTypeHeader(t *testing.T) {
	recorder := httptest.NewRecorder()

	model := struct {
		Name string
	}{
		"Joe Bloggs",
	}

	jsonProcessor := &jsonProcessor{}

	jsonProcessor.Process(recorder, model)

	assert.Equal(t, "application/json", recorder.HeaderMap.Get("Content-Type"))
}

func TestShouldSetResponseBody(t *testing.T) {
	recorder := httptest.NewRecorder()

	model := struct {
		Name string
	}{
		"Joe Bloggs",
	}

	jsonProcessor := &jsonProcessor{}

	jsonProcessor.Process(recorder, model)

	assert.Equal(t, "{\"Name\":\"Joe Bloggs\"}", recorder.Body.String())
}

func TestShouldSet500StatusCodeOnError(t *testing.T) {
	recorder := httptest.NewRecorder()

	model := &User{

		"Joe Bloggs",
	}

	jsonProcessor := &jsonProcessor{}

	jsonProcessor.Process(recorder, model)

	assert.Equal(t, 500, recorder.Code)
}

type User struct {
	Name string
}

func (u *User) MarshalJSON() ([]byte, error) {
	return nil, errors.New("oops")
}
<commit_msg>some comments about CanProcess tests<commit_after>package negotiator

import (
	"errors"
	"net/http/httptest"
	"testing"

	"github.com/stretchr/testify/assert"
)

//I think this will be obsolete when we add a library to handle the the Accept header better as that will be unit tested
func TestShouldProcessJSONAcceptHeader(t *testing.T) {
	var acceptTests = []struct {
		acceptheader string
	}{
		{"application/json"},
		{"application/json-"},
		{"text/json"},
		{"+json"},
	}

	jsonProcessor := &jsonProcessor{}

	for _, tt := range acceptTests {
		result := jsonProcessor.CanProcess(tt.acceptheader)
		assert.True(t, result, "Should process "+tt.acceptheader)
	}
}

func TestShouldSetContentTypeHeader(t *testing.T) {
	recorder := httptest.NewRecorder()

	model := struct {
		Name string
	}{
		"Joe Bloggs",
	}

	jsonProcessor := &jsonProcessor{}

	jsonProcessor.Process(recorder, model)

	assert.Equal(t, "application/json", recorder.HeaderMap.Get("Content-Type"))
}

func TestShouldSetResponseBody(t *testing.T) {
	recorder := httptest.NewRecorder()

	model := struct {
		Name string
	}{
		"Joe Bloggs",
	}

	jsonProcessor := &jsonProcessor{}

	jsonProcessor.Process(recorder, model)

	assert.Equal(t, "{\"Name\":\"Joe Bloggs\"}", recorder.Body.String())
}

func TestShouldSet500StatusCodeOnError(t *testing.T) {
	recorder := httptest.NewRecorder()

	model := &User{
		"Joe Bloggs",
	}

	jsonProcessor := &jsonProcessor{}

	jsonProcessor.Process(recorder, model)

	assert.Equal(t, 500, recorder.Code)
}

type User struct {
	Name string
}

func (u *User) MarshalJSON() ([]byte, error) {
	return nil, errors.New("oops")
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"os"
	"regexp"
	"strings"
)

func checkErr(err error, txn *sql.Tx) {
	if err != nil {
		fmt.Println("Error 33:", err)
		fmt.Println("Attempting Roll back!")
		txn.Rollback()
		os.Exit(1)
	}
}

func doExec(stmt string, txn *sql.Tx) {
	fmt.Println("  " + stmt)
	_, err := txn.Exec(stmt)
	if err != nil {
		fmt.Println("Error:", err)
		fmt.Println("Attempting Roll back!")
		txn.Rollback()
		os.Exit(1)
	}
}

// Up is executed when this migration is applied
func Up_20170205101432(txn *sql.Tx) {

	algorithmStr := map[int64]string{
		0: "consistent_hash",
		1: "false",
		2: "strict",
		3: "true",
		4: "latched",
	}

	fmt.Println("  Starting migration 20130106222315...")
	doExec("CREATE TYPE profile_type AS ENUM ("+
		"'ATS_PROFILE', 'TR_PROFILE', 'TM_PROFILE', 'TS_PROFILE', 'TP_PROFILE', 'INFLUXDB_PROFILE',"+
		"'RIAK_PROFILE', 'SPLUNK_PROFILE', 'DS_PROFILE', 'ORG_PROFILE', 'KAFKA_PROFILE', 'LOGSTASH_PROFILE',"+
		"'ES_PROFILE', 'UNK_PROFILE')", txn)

	doExec("CREATE OR REPLACE VIEW \"profile_type_values\" AS SELECT unnest(enum_range(NULL::profile_type )) AS value ORDER BY value", txn)

	doExec("ALTER TABLE public.profile ADD COLUMN type profile_type", txn)
	doExec("UPDATE public.profile SET type='UNK_PROFILE'", txn) // So we don't get any NULL, these should be checked.
	doExec("UPDATE public.profile SET type='TR_PROFILE' WHERE name like 'CCR_%' OR name like 'TR_%'", txn)
	doExec("UPDATE public.profile SET type='TM_PROFILE' WHERE name like 'RASCAL_%' OR name like 'TM_%'", txn)
	doExec("UPDATE public.profile SET type='TS_PROFILE' WHERE name like 'TRAFFIC_STATS%'", txn)
	doExec("UPDATE public.profile SET type='TP_PROFILE' WHERE name like 'TRAFFIC_PORTAL%'", txn)
	doExec("UPDATE public.profile SET type='INFLUXDB_PROFILE' WHERE name like 'INFLUXDB%'", txn)
	doExec("UPDATE public.profile SET type='RIAK_PROFILE' WHERE name like 'RIAK%'", txn)
	doExec("UPDATE public.profile SET type='SPLUNK_PROFILE' WHERE name like 'SPLUNK%'", txn)
	doExec("UPDATE public.profile SET type='ORG_PROFILE' WHERE name like '%ORG%' or name like 'MSO%' or name like '%ORIGIN%'", txn)
	doExec("UPDATE public.profile SET type='KAFKA_PROFILE' WHERE name like 'KAFKA%'", txn)
	doExec("UPDATE public.profile SET type='LOGSTASH_PROFILE' WHERE name like 'LOGSTASH_%'", txn)
	doExec("UPDATE public.profile SET type='ES_PROFILE' WHERE name like 'ELASTICSEARCH%'", txn)
	doExec("UPDATE public.profile SET type='ATS_PROFILE' WHERE name like 'EDGE%' or name like 'MID%'", txn)

	doExec("ALTER TABLE public.profile ALTER type SET NOT NULL", txn)

	doExec("ALTER TABLE public.cdn ADD COLUMN domain_name text", txn)

	doExec("UPDATE cdn SET domain_name=domainlist.value "+
		"FROM (SELECT distinct cdn_id,value FROM server,parameter WHERE type=(SELECT id FROM type WHERE name='EDGE') "+
		"AND parameter.id in (select parameter from profile_parameter WHERE profile_parameter.profile=server.profile) "+
		"AND parameter.name='domain_name' "+
		"AND config_file='CRConfig.json') AS domainlist "+
		"WHERE id = domainlist.cdn_id", txn)

	doExec("UPDATE public.cdn SET domain_name='-' WHERE name='ALL'", txn)

	doExec("ALTER TABLE public.cdn ALTER COLUMN domain_name SET NOT NULL", txn)

	doExec("ALTER TABLE public.profile ADD COLUMN cdn bigint", txn)

	doExec("ALTER TABLE public.profile "+
		"ADD CONSTRAINT fk_cdn1 FOREIGN KEY (cdn) "+
		"REFERENCES public.cdn (id) MATCH SIMPLE "+
		"ON UPDATE RESTRICT ON DELETE RESTRICT", txn)

	doExec("CREATE INDEX idx_181818_fk_cdn1 "+
		"ON public.profile "+
		"USING btree "+
		"(cdn)", txn)

	doExec("UPDATE profile set cdn=domainlist.cdn_id "+
		"FROM (SELECT distinct profile.id AS profile_id, value AS profile_domain_name, cdn.id cdn_id "+
		"FROM profile, parameter, cdn, profile_parameter "+
		"WHERE parameter.name='domain_name' "+
		"AND parameter.config_file='CRConfig.json' "+
		"AND parameter.value = cdn.domain_name "+
		"AND parameter.id in (select parameter from profile_parameter where profile=profile.id)) as domainlist "+
		"WHERE id = domainlist.profile_id", txn)

	doExec("ALTER TABLE deliveryservice ALTER profile DROP NOT NULL", txn)

	doExec("UPDATE deliveryservice SET profile=NULL", txn)

	doExec("INSERT INTO parameter (name, config_file, value) VALUES ('mso.parent_retry', 'parent.config', 'simple_retry');", txn)
	doExec("INSERT INTO parameter (name, config_file, value) VALUES ('mso.parent_retry', 'parent.config', 'unavailable_server_retry');", txn)
	doExec("INSERT INTO parameter (name, config_file, value) VALUES ('mso.parent_retry', 'parent.config', 'both');", txn)

	type Profile struct {
		Id                 int64
		Name               string
		Desc               string
		Type               string
		Cdn                int64
		MidHeaderRewrite   string
		MultiSiteOriginAlg int64
		XMLId              string
	}
	// move data
	pmap := make(map[string]Profile)
	rows, err := txn.Query("SELECT id,xml_id,mid_header_rewrite,multi_site_origin,multi_site_origin_algorithm,cdn_id FROM deliveryservice WHERE multi_site_origin=true")
	if err != nil {
		fmt.Println("Error:", err)
		fmt.Println("Attempting Roll back!")
		txn.Rollback()
		os.Exit(1)
	}
	existingParam := make(map[string]int64)
	for rows.Next() {
		var (
			id                          int64
			xml_id                      string
			mid_header_rewrite          sql.NullString
			multi_site_origin           sql.NullBool
			multi_site_origin_algorithm sql.NullInt64
			cdn_id                      int64
		)
		err := rows.Scan(&id, &xml_id, &mid_header_rewrite, &multi_site_origin, &multi_site_origin_algorithm, &cdn_id)
		checkErr(err, txn)

		pName := "DS_" + xml_id
		pDesc := "Deliveryservice profile for " + xml_id
		pType := "DS_PROFILE"
		mhrString := mid_header_rewrite.String
		pmap[pName] = Profile{
			Id:                 -1,
			Name:               pName,
			Desc:               pDesc,
			Type:               pType,
			Cdn:                cdn_id,
			MidHeaderRewrite:   mhrString,
			MultiSiteOriginAlg: multi_site_origin_algorithm.Int64,
			XMLId:              xml_id,
		}
	}
	err = rows.Err()
	checkErr(err, txn)
	rows.Close()

	for _, prof := range pmap {
		fmt.Println("--\nINSERT INTO PROFILE (name, description, type, cdn) VALUES($1, $2, $3, $4) RETURNING id", prof.Name, prof.Desc, prof.Type, prof.Cdn)
		newRow := txn.QueryRow("INSERT INTO PROFILE (name, description, type, cdn) VALUES($1, $2, $3, $4) RETURNING id", prof.Name, prof.Desc, prof.Type, prof.Cdn)
		var newProfileId int64
		err := newRow.Scan(&newProfileId)
		checkErr(err, txn)

		remainingString := ""
		parent_retry := ""
		var regexpOne = regexp.MustCompile(`^\s*set-config\s+proxy.config.http.parent_origin.(\S+)\s+(.*)$`)
		if prof.MidHeaderRewrite != "" {
			remapParts := strings.Split(prof.MidHeaderRewrite, "__RETURN__")
			for _, line := range remapParts {
				fmt.Println(line)
				match := regexpOne.FindSubmatch([]byte(line))
				if len(match) != 0 {
					var newId int64
					var ok bool
					newId, ok = existingParam[string(match[1])+string(match[2])]
					//	fmt.Printf("%s -> %v %v\n", string(match[1])+string(match[2]), newId, ok)
					if !ok {
						if string(match[1]) == "simple_retry_enabled" || string(match[1]) == "dead_server_retry_enabled" {
							if parent_retry == "" {
								parent_retry = string(match[1])
							} else {
								parent_retry = "both"
							}
						} else {
							// unavailable_server_retry_responses is the only one that survives?
							if string(match[1]) == "dead_server_retry_response_codes" {
								pName := "mso.unavailable_server_retry_responses"
								fmt.Println("INSERT INTO PARAMETER (name, config_file, value) VALUES ($1, $2, $3) RETURNING id",
									pName, "parent.config", string(match[2]))
								newRow := txn.QueryRow("INSERT INTO PARAMETER (name, config_file, value) VALUES ($1, $2, $3) RETURNING id",
									pName, "parent.config", string(match[2]))
								err := newRow.Scan(&newId)
								checkErr(err, txn)
								existingParam[string(match[1])+string(match[2])] = newId
							}
						}
					} else {
						newId = existingParam[string(match[1])+string(match[2])]
					}
					if newId != 0 {
						fmt.Println("INSERT INTO PROFILE_PARAMETER (parameter, profile) VALUES ($1, $2)", newId, newProfileId)
						_, err = txn.Exec("INSERT INTO PROFILE_PARAMETER (parameter, profile) VALUES ($1, $2)", newId, newProfileId)
						checkErr(err, txn)
					}
				} else {
					if !strings.HasSuffix(line, "__RETURN__") {
						remainingString = remainingString + line + "__RETURN__"
					}
				}
			}
		}
		/* Not doing the removal of the header_rewrite stuff  at this time, so we can support both 5.x and 6.x - 6.x will have a warning if these are
		   not removed from the DS, but should run just fine.

		fmt.Printf("MHRW was: %s, \nMHRW now: %s\n", prof.MidHeaderRewrite, remainingString)

		_, err = txn.Exec("UPDATE deliveryservice set mid_header_rewrite=$1 where xml_id=$2", remainingString, prof.XMLId)
		checkErr(err, txn)

		*/

		var newId int64
		var ok bool
		newId, ok = existingParam["mso.algorithm"+"parent.config"+algorithmStr[prof.MultiSiteOriginAlg]]
		if !ok {
			fmt.Println("INSERT INTO PARAMETER (name, config_file, value) VALUES ($1, $2, $3) RETURNING id",
				"mso.algorithm", "parent.config", algorithmStr[prof.MultiSiteOriginAlg])
			newRow = txn.QueryRow("INSERT INTO PARAMETER (name, config_file, value) VALUES ($1, $2, $3) RETURNING id",
				"mso.algorithm", "parent.config", algorithmStr[prof.MultiSiteOriginAlg])
			err = newRow.Scan(&newId)
			checkErr(err, txn)
			existingParam["mso.algorithm"+"parent.config"+algorithmStr[prof.MultiSiteOriginAlg]] = newId
		} else {
			newId = existingParam["mso.algorithm"+"parent.config"+algorithmStr[prof.MultiSiteOriginAlg]]
		}

		fmt.Println("INSERT INTO PROFILE_PARAMETER (parameter, profile) VALUES ($1, $2)", newId, newProfileId)
		_, err = txn.Exec("INSERT INTO PROFILE_PARAMETER (parameter, profile) VALUES ($1, $2)", newId, newProfileId)
		checkErr(err, txn)

		fmt.Println("INSERT INTO profile_parameter (profile, parameter) SELECT $1, id FROM parameter "+
			"WHERE config_file='parent.config' AND name='mso.parent_retry' AND value=$2", newProfileId, parent_retry)
		_, err = txn.Exec("INSERT INTO profile_parameter (profile, parameter) SELECT $1, id FROM parameter "+
			"WHERE config_file='parent.config' AND name='mso.parent_retry' AND value=$2", newProfileId, parent_retry)
		checkErr(err, txn)

		_, err = txn.Exec("UPDATE deliveryservice SET profile=$1 WHERE xml_id=$2", newProfileId, prof.XMLId)
		checkErr(err, txn)
	}
}

// Down is executed when this migration is rolled back
func Down_20170205101432(txn *sql.Tx) {
	//
}
<commit_msg>Add license to migration<commit_after>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

// http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This file was initially generated by gen_to_start.go (add link), as a start
// of the Traffic Ops golang data model

package main

import (
	"database/sql"
	"fmt"
	"os"
	"regexp"
	"strings"
)

func checkErr(err error, txn *sql.Tx) {
	if err != nil {
		fmt.Println("Error 33:", err)
		fmt.Println("Attempting Roll back!")
		txn.Rollback()
		os.Exit(1)
	}
}

func doExec(stmt string, txn *sql.Tx) {
	fmt.Println("  " + stmt)
	_, err := txn.Exec(stmt)
	if err != nil {
		fmt.Println("Error:", err)
		fmt.Println("Attempting Roll back!")
		txn.Rollback()
		os.Exit(1)
	}
}

// Up is executed when this migration is applied
func Up_20170205101432(txn *sql.Tx) {

	algorithmStr := map[int64]string{
		0: "consistent_hash",
		1: "false",
		2: "strict",
		3: "true",
		4: "latched",
	}

	fmt.Println("  Starting migration 20130106222315...")
	doExec("CREATE TYPE profile_type AS ENUM ("+
		"'ATS_PROFILE', 'TR_PROFILE', 'TM_PROFILE', 'TS_PROFILE', 'TP_PROFILE', 'INFLUXDB_PROFILE',"+
		"'RIAK_PROFILE', 'SPLUNK_PROFILE', 'DS_PROFILE', 'ORG_PROFILE', 'KAFKA_PROFILE', 'LOGSTASH_PROFILE',"+
		"'ES_PROFILE', 'UNK_PROFILE')", txn)

	doExec("CREATE OR REPLACE VIEW \"profile_type_values\" AS SELECT unnest(enum_range(NULL::profile_type )) AS value ORDER BY value", txn)

	doExec("ALTER TABLE public.profile ADD COLUMN type profile_type", txn)
	doExec("UPDATE public.profile SET type='UNK_PROFILE'", txn) // So we don't get any NULL, these should be checked.
	doExec("UPDATE public.profile SET type='TR_PROFILE' WHERE name like 'CCR_%' OR name like 'TR_%'", txn)
	doExec("UPDATE public.profile SET type='TM_PROFILE' WHERE name like 'RASCAL_%' OR name like 'TM_%'", txn)
	doExec("UPDATE public.profile SET type='TS_PROFILE' WHERE name like 'TRAFFIC_STATS%'", txn)
	doExec("UPDATE public.profile SET type='TP_PROFILE' WHERE name like 'TRAFFIC_PORTAL%'", txn)
	doExec("UPDATE public.profile SET type='INFLUXDB_PROFILE' WHERE name like 'INFLUXDB%'", txn)
	doExec("UPDATE public.profile SET type='RIAK_PROFILE' WHERE name like 'RIAK%'", txn)
	doExec("UPDATE public.profile SET type='SPLUNK_PROFILE' WHERE name like 'SPLUNK%'", txn)
	doExec("UPDATE public.profile SET type='ORG_PROFILE' WHERE name like '%ORG%' or name like 'MSO%' or name like '%ORIGIN%'", txn)
	doExec("UPDATE public.profile SET type='KAFKA_PROFILE' WHERE name like 'KAFKA%'", txn)
	doExec("UPDATE public.profile SET type='LOGSTASH_PROFILE' WHERE name like 'LOGSTASH_%'", txn)
	doExec("UPDATE public.profile SET type='ES_PROFILE' WHERE name like 'ELASTICSEARCH%'", txn)
	doExec("UPDATE public.profile SET type='ATS_PROFILE' WHERE name like 'EDGE%' or name like 'MID%'", txn)

	doExec("ALTER TABLE public.profile ALTER type SET NOT NULL", txn)

	doExec("ALTER TABLE public.cdn ADD COLUMN domain_name text", txn)

	doExec("UPDATE cdn SET domain_name=domainlist.value "+
		"FROM (SELECT distinct cdn_id,value FROM server,parameter WHERE type=(SELECT id FROM type WHERE name='EDGE') "+
		"AND parameter.id in (select parameter from profile_parameter WHERE profile_parameter.profile=server.profile) "+
		"AND parameter.name='domain_name' "+
		"AND config_file='CRConfig.json') AS domainlist "+
		"WHERE id = domainlist.cdn_id", txn)

	doExec("UPDATE public.cdn SET domain_name='-' WHERE name='ALL'", txn)

	doExec("ALTER TABLE public.cdn ALTER COLUMN domain_name SET NOT NULL", txn)

	doExec("ALTER TABLE public.profile ADD COLUMN cdn bigint", txn)

	doExec("ALTER TABLE public.profile "+
		"ADD CONSTRAINT fk_cdn1 FOREIGN KEY (cdn) "+
		"REFERENCES public.cdn (id) MATCH SIMPLE "+
		"ON UPDATE RESTRICT ON DELETE RESTRICT", txn)

	doExec("CREATE INDEX idx_181818_fk_cdn1 "+
		"ON public.profile "+
		"USING btree "+
		"(cdn)", txn)

	doExec("UPDATE profile set cdn=domainlist.cdn_id "+
		"FROM (SELECT distinct profile.id AS profile_id, value AS profile_domain_name, cdn.id cdn_id "+
		"FROM profile, parameter, cdn, profile_parameter "+
		"WHERE parameter.name='domain_name' "+
		"AND parameter.config_file='CRConfig.json' "+
		"AND parameter.value = cdn.domain_name "+
		"AND parameter.id in (select parameter from profile_parameter where profile=profile.id)) as domainlist "+
		"WHERE id = domainlist.profile_id", txn)

	doExec("ALTER TABLE deliveryservice ALTER profile DROP NOT NULL", txn)

	doExec("UPDATE deliveryservice SET profile=NULL", txn)

	doExec("INSERT INTO parameter (name, config_file, value) VALUES ('mso.parent_retry', 'parent.config', 'simple_retry');", txn)
	doExec("INSERT INTO parameter (name, config_file, value) VALUES ('mso.parent_retry', 'parent.config', 'unavailable_server_retry');", txn)
	doExec("INSERT INTO parameter (name, config_file, value) VALUES ('mso.parent_retry', 'parent.config', 'both');", txn)

	type Profile struct {
		Id                 int64
		Name               string
		Desc               string
		Type               string
		Cdn                int64
		MidHeaderRewrite   string
		MultiSiteOriginAlg int64
		XMLId              string
	}
	// move data
	pmap := make(map[string]Profile)
	rows, err := txn.Query("SELECT id,xml_id,mid_header_rewrite,multi_site_origin,multi_site_origin_algorithm,cdn_id FROM deliveryservice WHERE multi_site_origin=true")
	if err != nil {
		fmt.Println("Error:", err)
		fmt.Println("Attempting Roll back!")
		txn.Rollback()
		os.Exit(1)
	}
	existingParam := make(map[string]int64)
	for rows.Next() {
		var (
			id                          int64
			xml_id                      string
			mid_header_rewrite          sql.NullString
			multi_site_origin           sql.NullBool
			multi_site_origin_algorithm sql.NullInt64
			cdn_id                      int64
		)
		err := rows.Scan(&id, &xml_id, &mid_header_rewrite, &multi_site_origin, &multi_site_origin_algorithm, &cdn_id)
		checkErr(err, txn)

		pName := "DS_" + xml_id
		pDesc := "Deliveryservice profile for " + xml_id
		pType := "DS_PROFILE"
		mhrString := mid_header_rewrite.String
		pmap[pName] = Profile{
			Id:                 -1,
			Name:               pName,
			Desc:               pDesc,
			Type:               pType,
			Cdn:                cdn_id,
			MidHeaderRewrite:   mhrString,
			MultiSiteOriginAlg: multi_site_origin_algorithm.Int64,
			XMLId:              xml_id,
		}
	}
	err = rows.Err()
	checkErr(err, txn)
	rows.Close()

	for _, prof := range pmap {
		fmt.Println("--\nINSERT INTO PROFILE (name, description, type, cdn) VALUES($1, $2, $3, $4) RETURNING id", prof.Name, prof.Desc, prof.Type, prof.Cdn)
		newRow := txn.QueryRow("INSERT INTO PROFILE (name, description, type, cdn) VALUES($1, $2, $3, $4) RETURNING id", prof.Name, prof.Desc, prof.Type, prof.Cdn)
		var newProfileId int64
		err := newRow.Scan(&newProfileId)
		checkErr(err, txn)

		remainingString := ""
		parent_retry := ""
		var regexpOne = regexp.MustCompile(`^\s*set-config\s+proxy.config.http.parent_origin.(\S+)\s+(.*)$`)
		if prof.MidHeaderRewrite != "" {
			remapParts := strings.Split(prof.MidHeaderRewrite, "__RETURN__")
			for _, line := range remapParts {
				fmt.Println(line)
				match := regexpOne.FindSubmatch([]byte(line))
				if len(match) != 0 {
					var newId int64
					var ok bool
					newId, ok = existingParam[string(match[1])+string(match[2])]
					//	fmt.Printf("%s -> %v %v\n", string(match[1])+string(match[2]), newId, ok)
					if !ok {
						if string(match[1]) == "simple_retry_enabled" || string(match[1]) == "dead_server_retry_enabled" {
							if parent_retry == "" {
								parent_retry = string(match[1])
							} else {
								parent_retry = "both"
							}
						} else {
							// unavailable_server_retry_responses is the only one that survives?
							if string(match[1]) == "dead_server_retry_response_codes" {
								pName := "mso.unavailable_server_retry_responses"
								fmt.Println("INSERT INTO PARAMETER (name, config_file, value) VALUES ($1, $2, $3) RETURNING id",
									pName, "parent.config", string(match[2]))
								newRow := txn.QueryRow("INSERT INTO PARAMETER (name, config_file, value) VALUES ($1, $2, $3) RETURNING id",
									pName, "parent.config", string(match[2]))
								err := newRow.Scan(&newId)
								checkErr(err, txn)
								existingParam[string(match[1])+string(match[2])] = newId
							}
						}
					} else {
						newId = existingParam[string(match[1])+string(match[2])]
					}
					if newId != 0 {
						fmt.Println("INSERT INTO PROFILE_PARAMETER (parameter, profile) VALUES ($1, $2)", newId, newProfileId)
						_, err = txn.Exec("INSERT INTO PROFILE_PARAMETER (parameter, profile) VALUES ($1, $2)", newId, newProfileId)
						checkErr(err, txn)
					}
				} else {
					if !strings.HasSuffix(line, "__RETURN__") {
						remainingString = remainingString + line + "__RETURN__"
					}
				}
			}
		}
		/* Not doing the removal of the header_rewrite stuff  at this time, so we can support both 5.x and 6.x - 6.x will have a warning if these are
		   not removed from the DS, but should run just fine.

		fmt.Printf("MHRW was: %s, \nMHRW now: %s\n", prof.MidHeaderRewrite, remainingString)

		_, err = txn.Exec("UPDATE deliveryservice set mid_header_rewrite=$1 where xml_id=$2", remainingString, prof.XMLId)
		checkErr(err, txn)

		*/

		var newId int64
		var ok bool
		newId, ok = existingParam["mso.algorithm"+"parent.config"+algorithmStr[prof.MultiSiteOriginAlg]]
		if !ok {
			fmt.Println("INSERT INTO PARAMETER (name, config_file, value) VALUES ($1, $2, $3) RETURNING id",
				"mso.algorithm", "parent.config", algorithmStr[prof.MultiSiteOriginAlg])
			newRow = txn.QueryRow("INSERT INTO PARAMETER (name, config_file, value) VALUES ($1, $2, $3) RETURNING id",
				"mso.algorithm", "parent.config", algorithmStr[prof.MultiSiteOriginAlg])
			err = newRow.Scan(&newId)
			checkErr(err, txn)
			existingParam["mso.algorithm"+"parent.config"+algorithmStr[prof.MultiSiteOriginAlg]] = newId
		} else {
			newId = existingParam["mso.algorithm"+"parent.config"+algorithmStr[prof.MultiSiteOriginAlg]]
		}

		fmt.Println("INSERT INTO PROFILE_PARAMETER (parameter, profile) VALUES ($1, $2)", newId, newProfileId)
		_, err = txn.Exec("INSERT INTO PROFILE_PARAMETER (parameter, profile) VALUES ($1, $2)", newId, newProfileId)
		checkErr(err, txn)

		fmt.Println("INSERT INTO profile_parameter (profile, parameter) SELECT $1, id FROM parameter "+
			"WHERE config_file='parent.config' AND name='mso.parent_retry' AND value=$2", newProfileId, parent_retry)
		_, err = txn.Exec("INSERT INTO profile_parameter (profile, parameter) SELECT $1, id FROM parameter "+
			"WHERE config_file='parent.config' AND name='mso.parent_retry' AND value=$2", newProfileId, parent_retry)
		checkErr(err, txn)

		_, err = txn.Exec("UPDATE deliveryservice SET profile=$1 WHERE xml_id=$2", newProfileId, prof.XMLId)
		checkErr(err, txn)
	}
}

// Down is executed when this migration is rolled back
func Down_20170205101432(txn *sql.Tx) {
	//
}
<|endoftext|>


<commit_before>package models

import "time"

type UnifiedMessage struct {
	Id           int64
	Body         string
	CreatedAt    time.Time
	UpdatedAt    time.Time
	DeletedAt    time.Time
	LikeCount    int64
	CommentCount int64
}
<commit_msg>SocialApi: add message type into struct<commit_after>package models

import "time"

type UnifiedMessage struct {
	Id           int64
	MessageType  string
	Body         string
	CreatedAt    time.Time
	UpdatedAt    time.Time
	DeletedAt    time.Time
	LikeCount    int64
	CommentCount int64
}
<|endoftext|>


<commit_before>package octokit

import (
	"fmt"
	"time"
)

type Status struct {
	CreatedAt   time.Time `json:"created_at"`
	UpdatedAt   time.Time `json:"updated_at"`
	State       string    `json:"state"`
	TargetUrl   string    `json:"target_url"`
	Description string    `json:"description"`
	Id          int       `json:"id"`
	Url         string    `json:"url"`
}

type StatusCreator struct {
	Login      string `json:"login"`
	Id         int    `json:"id"`
	AvatarUrl  string `json:"avatar_url"`
	GravatarId string `json:"gravatar_id"`
	Url        string `json:"url"`
}

func (c *Client) Statuses(repo Repository, sha string) ([]Status, error) {
	path := fmt.Sprintf("repos/%s/statuses/%s", repo, sha)
	var statuses []Status
	err := c.jsonGet(path, nil, &statuses)
	if err != nil {
		return nil, err
	}

	return statuses, nil
}
<commit_msg>Add Creator to Status API<commit_after>package octokit

import (
	"fmt"
	"time"
)

type Status struct {
	CreatedAt   time.Time     `json:"created_at"`
	UpdatedAt   time.Time     `json:"updated_at"`
	State       string        `json:"state"`
	TargetUrl   string        `json:"target_url"`
	Description string        `json:"description"`
	Id          int           `json:"id"`
	Url         string        `json:"url"`
	Creator     StatusCreator `json:"creator"`
}

type StatusCreator struct {
	Login      string `json:"login"`
	Id         int    `json:"id"`
	AvatarUrl  string `json:"avatar_url"`
	GravatarId string `json:"gravatar_id"`
	Url        string `json:"url"`
}

func (c *Client) Statuses(repo Repository, sha string) ([]Status, error) {
	path := fmt.Sprintf("repos/%s/statuses/%s", repo, sha)
	var statuses []Status
	err := c.jsonGet(path, nil, &statuses)
	if err != nil {
		return nil, err
	}

	return statuses, nil
}
<|endoftext|>