<commit_before>package controllers

import (
	"bytes"
	"github.com/airdispatch/dpl"
	"github.com/robfig/revel"
	"html/template"
	"melange/app/models"
	"melange/app/routes"
	"melange/mailserver"
	"net/http"
	"time"
)

type Dispatch struct {
	GorpController
}

func (c Dispatch) Init() revel.Result {
	if c.Session["user"] == "" {
		return c.Redirect(routes.App.Login())
	}

	// Function to load user apps
	apps, err := models.GetUserApps(c.Txn, c.Session)
	if err != nil {
		panic(err)
	}

	c.RenderArgs["apps"] = apps

	mailserver.InitRouter()

	c.RenderArgs["news"] = func(m dpl.Message) template.HTML {
		return "hello"
	}

	return nil
}

func (d Dispatch) Dashboard() revel.Result {
	// Download all recents from subscribed, download all recents from alerts, sort them chronologically
	u, err := d.Txn.Get(&models.User{}, GetUserId(d.Session))
	if err != nil {
		panic(err)
	}

	id, err := models.UserIdentities(u.(*models.User), d.Txn)
	if err != nil {
		panic(err)
	}
	if len(id) == 0 {
		panic("Not enough IDs")
	}

	msgs, err := mailserver.Messages(mailserver.LookupRouter,
		d.Txn,
		id[0],
		u.(*models.User),
		true, true, true,
		time.Now().Add(-7*24*time.Hour).Unix())
	if err != nil {
		panic(err)
	}

	var apps []*models.UserApp
	_, err = d.Txn.Select(&apps, "select * from dispatch_app where userid = $1", GetUserId(d.Session))
	var loadedApps []*dpl.PluginInstance = make([]*dpl.PluginInstance, len(apps))
	for i, v := range apps {
		resp, err := http.Get(v.AppURL)
		if err != nil {
			// Couldn't load that application
			resp.Body.Close()
			continue
		}
		o, err := dpl.ParseDPLStream(resp.Body)
		if err != nil {
			// Couldn't parse that application
			resp.Body.Close()
			continue
		}

		plugin := o.CreateInstance(&PluginHost{
			User: u.(*models.User),
			Txn:  d.Txn,
			R:    mailserver.LookupRouter,
		}, nil)
		loadedApps[i] = plugin
		resp.Body.Close()
	}

	type DisplayMessage struct {
		mailserver.MelangeMessage
		Display template.HTML
	}
	recents := make([]DisplayMessage, 0)
	for _, v := range msgs {
		rendered := false
		for _, a := range loadedApps {
			for _, t := range a.Tag {
				if t.FeedAction != "" {
					match := true
					for _, f := range t.Fields {
						if !v.Has(f.Name) && !f.Optional {
							match = false
						}
					}
					if match {
						// Do Something
					}
				}
			}
		}
		if !rendered {
			t, _ := template.New("").Parse(`{{ range .Components }}<p><strong>{{ .Key }}</strong> {{ .String }}</p>{{ end }}`)
			var b bytes.Buffer
			t.Execute(&b, v)
			recents = append(recents, DisplayMessage{v, template.HTML(b.String())})
		}
	}

	return d.Render(recents)
}

func (d Dispatch) Profile() revel.Result {
	user, err := d.Txn.Get(&models.User{}, GetUserId(d.Session))
	if err != nil {
		panic(err)
	}

	id, err := models.UserIdentities(user.(*models.User), d.Txn)
	if err != nil {
		panic(err)
	}

	messages, err := mailserver.Messages(mailserver.LookupRouter,
		d.Txn,
		id[0],
		user.(*models.User),
		false, false, true,
		time.Now().Add(-7*24*time.Hour).Unix())
	if err != nil {
		panic(err)
	}

	return d.Render(messages, user)
}

func (d Dispatch) All() revel.Result {
	u, err := d.Txn.Get(&models.User{}, GetUserId(d.Session))
	if err != nil {
		panic(err)
	}

	id, err := models.UserIdentities(u.(*models.User), d.Txn)
	if err != nil {
		panic(err)
	}
	if len(id) == 0 {
		panic("Not enough IDs")
	}

	recents, err := mailserver.Messages(mailserver.LookupRouter,
		d.Txn,
		id[0],
		u.(*models.User),
		true, true, true,
		0)
	if err != nil {
		panic(err)
	}

	return d.Render(recents)
}

func (d Dispatch) Applications() revel.Result {
	return d.Render()
}

func (d Dispatch) AddApplication(url string) revel.Result {
	resp, err := http.Get(url)
	if err != nil {
		d.Flash.Error("Unable to download application via supplied url.")
		return d.Redirect(routes.Dispatch.Applications())
	}
	defer resp.Body.Close()

	plugin, err := dpl.ParseDPLStream(resp.Body)
	if err != nil {
		d.Flash.Error("Plugin is not valid.")
		return d.Redirect(routes.Dispatch.Applications())
	}

	in := &models.UserApp{
		UserId: GetUserId(d.Session),
		AppURL: url,
		Name:   plugin.Name,
		Path:   plugin.Path,
	}
	err = d.Txn.Insert(in)
	if err != nil {
		panic(err)
	}

	return d.Redirect(routes.Loader.LoadAppDefault(plugin.Name))
}

func (d Dispatch) UninstallApplication(app string) revel.Result {
	u := GetUserId(d.Session)

	var apps []*models.UserApp
	_, err := d.Txn.Select(&apps, "select * from dispatch_app where userid = $1 and UPPER(name) = UPPER($2)", u, app)
	if err != nil {
		panic(err)
	}
	if len(apps) != 1 {
		panic(len(apps))
	}

	toDelete := apps[0]
	count, err := d.Txn.Delete(toDelete)
	if err != nil || count != 1 {
		panic(err)
	}

	return d.Redirect(routes.Dispatch.Applications())
}

func (c Dispatch) Account() revel.Result {
	if c.Session["user"] == "" {
		return c.Redirect(routes.App.Login())
	}
	var users []*models.User

	_, err := c.Txn.Select(&users, "select * from dispatch_user where userid = $1", GetUserId(c.Session))
	if err != nil {
		panic(err)
	}
	user := users[0]

	var subscriptions []*models.UserSubscription
	_, err = c.Txn.Select(&subscriptions, "select * from dispatch_subscription where userid = $1", user.UserId)
	if err != nil {
		panic(err)
	}

	var identities []*models.Identity
	_, err = c.Txn.Select(&identities, "select * from dispatch_identity where userid = $1", user.UserId)
	if err != nil {
		panic(err)
	}

	return c.Render(user, subscriptions, identities)
}

func (c Dispatch) AddSubscription(address string) revel.Result {
	// TODO: Verify the Address Somehow...
	mailserver.InitRouter()
	_, err := mailserver.LookupRouter.LookupAlias(address)
	if err != nil {
		c.Flash.Error("Unable to find that address. It has not been added.")
		return c.Redirect(routes.Dispatch.Account())
	}

	user := GetUserId(c.Session)
	newSub := &models.UserSubscription{
		UserId:  user,
		Address: address,
	}

	err = c.Txn.Insert(newSub)
	if err != nil {
		panic(err)
	}

	return c.Redirect(routes.Dispatch.Account())
}

func (d Dispatch) RemoveSubscription(id int) revel.Result {
	u := GetUserId(d.Session)

	var apps []*models.UserSubscription
	_, err := d.Txn.Select(&apps, "select * from dispatch_subscription where subscriptionid = $1 and userid = $2", id, u)
	if err != nil {
		panic(err)
	}
	if len(apps) != 1 {
		panic(len(apps))
	}

	toDelete := apps[0]
	count, err := d.Txn.Delete(toDelete)
	if err != nil || count != 1 {
		panic(err)
	}

	return d.Redirect(routes.Dispatch.Account())
}

func (d Dispatch) RegisterIdentity(id int) revel.Result {
	u := GetUserId(d.Session)

	var ids []*models.Identity
	_, err := d.Txn.Select(&ids, "select * from dispatch_identity where identityid = $1 and userid = $2", id, u)
	if err != nil {
		panic(err)
	}
	if len(ids) != 1 {
		panic(len(ids))
	}

	toRegister := ids[0]

	mailserver.InitRouter()
	did, err := toRegister.ToDispatch()
	if err != nil {
		panic(err)
	}
	did.SetLocation(revel.Config.StringDefault("server.location", ""))

	err = mailserver.RegistrationRouter.Register(did, toRegister.Alias)
	if err != nil {
		panic(err)
	}

	return d.Redirect(routes.Dispatch.Account())
}

func (d Dispatch) ProcessAccount(name string, username string, password1 string, password2 string, password string) revel.Result {
	u, err := d.Txn.Get(&models.User{}, GetUserId(d.Session))
	if err != nil {
		panic(err)
	}

	user := u.(*models.User)
	user.Name = name

	if user.VerifyPassword(password) {
		if password1 == password2 {
			user.UpdatePassword(password1)
		} else {
			d.Flash.Error("New passwords do not match.")
		}
	} else {
		if password != "" {
			d.Flash.Error("Current Password is not correct. Did not update username or password.")
		}
	}

	user.Save(d.Txn)
	return d.Redirect(routes.Dispatch.Account())
}
<commit_msg>Fixed Error with Response Object<commit_after>package controllers

import (
	"bytes"
	"github.com/airdispatch/dpl"
	"github.com/robfig/revel"
	"html/template"
	"melange/app/models"
	"melange/app/routes"
	"melange/mailserver"
	"net/http"
	"time"
)

type Dispatch struct {
	GorpController
}

func (c Dispatch) Init() revel.Result {
	if c.Session["user"] == "" {
		return c.Redirect(routes.App.Login())
	}

	// Function to load user apps
	apps, err := models.GetUserApps(c.Txn, c.Session)
	if err != nil {
		panic(err)
	}

	c.RenderArgs["apps"] = apps

	mailserver.InitRouter()

	c.RenderArgs["news"] = func(m dpl.Message) template.HTML {
		return "hello"
	}

	return nil
}

func (d Dispatch) Dashboard() revel.Result {
	// Download all recents from subscribed, download all recents from alerts, sort them chronologically
	u, err := d.Txn.Get(&models.User{}, GetUserId(d.Session))
	if err != nil {
		panic(err)
	}

	id, err := models.UserIdentities(u.(*models.User), d.Txn)
	if err != nil {
		panic(err)
	}
	if len(id) == 0 {
		panic("Not enough IDs")
	}

	msgs, err := mailserver.Messages(mailserver.LookupRouter,
		d.Txn,
		id[0],
		u.(*models.User),
		true, true, true,
		time.Now().Add(-7*24*time.Hour).Unix())
	if err != nil {
		panic(err)
	}

	var apps []*models.UserApp
	_, err = d.Txn.Select(&apps, "select * from dispatch_app where userid = $1", GetUserId(d.Session))
	var loadedApps []*dpl.PluginInstance = make([]*dpl.PluginInstance, len(apps))
	for i, v := range apps {
		resp, err := http.Get(v.AppURL)
		if err != nil {
			// Couldn't load that application
			continue
		}
		o, err := dpl.ParseDPLStream(resp.Body)
		if err != nil {
			// Couldn't parse that application
			resp.Body.Close()
			continue
		}

		plugin := o.CreateInstance(&PluginHost{
			User: u.(*models.User),
			Txn:  d.Txn,
			R:    mailserver.LookupRouter,
		}, nil)
		loadedApps[i] = plugin
		resp.Body.Close()
	}

	type DisplayMessage struct {
		mailserver.MelangeMessage
		Display template.HTML
	}
	recents := make([]DisplayMessage, 0)
	for _, v := range msgs {
		rendered := false
		for _, a := range loadedApps {
			for _, t := range a.Tag {
				if t.FeedAction != "" {
					match := true
					for _, f := range t.Fields {
						if !v.Has(f.Name) && !f.Optional {
							match = false
						}
					}
					if match {
						// Do Something
					}
				}
			}
		}
		if !rendered {
			t, _ := template.New("").Parse(`{{ range .Components }}<p><strong>{{ .Key }}</strong> {{ .String }}</p>{{ end }}`)
			var b bytes.Buffer
			t.Execute(&b, v)
			recents = append(recents, DisplayMessage{v, template.HTML(b.String())})
		}
	}

	return d.Render(recents)
}

func (d Dispatch) Profile() revel.Result {
	user, err := d.Txn.Get(&models.User{}, GetUserId(d.Session))
	if err != nil {
		panic(err)
	}

	id, err := models.UserIdentities(user.(*models.User), d.Txn)
	if err != nil {
		panic(err)
	}

	messages, err := mailserver.Messages(mailserver.LookupRouter,
		d.Txn,
		id[0],
		user.(*models.User),
		false, false, true,
		time.Now().Add(-7*24*time.Hour).Unix())
	if err != nil {
		panic(err)
	}

	return d.Render(messages, user)
}

func (d Dispatch) All() revel.Result {
	u, err := d.Txn.Get(&models.User{}, GetUserId(d.Session))
	if err != nil {
		panic(err)
	}

	id, err := models.UserIdentities(u.(*models.User), d.Txn)
	if err != nil {
		panic(err)
	}
	if len(id) == 0 {
		panic("Not enough IDs")
	}

	recents, err := mailserver.Messages(mailserver.LookupRouter,
		d.Txn,
		id[0],
		u.(*models.User),
		true, true, true,
		0)
	if err != nil {
		panic(err)
	}

	return d.Render(recents)
}

func (d Dispatch) Applications() revel.Result {
	return d.Render()
}

func (d Dispatch) AddApplication(url string) revel.Result {
	resp, err := http.Get(url)
	if err != nil {
		d.Flash.Error("Unable to download application via supplied url.")
		return d.Redirect(routes.Dispatch.Applications())
	}
	defer resp.Body.Close()

	plugin, err := dpl.ParseDPLStream(resp.Body)
	if err != nil {
		d.Flash.Error("Plugin is not valid.")
		return d.Redirect(routes.Dispatch.Applications())
	}

	in := &models.UserApp{
		UserId: GetUserId(d.Session),
		AppURL: url,
		Name:   plugin.Name,
		Path:   plugin.Path,
	}
	err = d.Txn.Insert(in)
	if err != nil {
		panic(err)
	}

	return d.Redirect(routes.Loader.LoadAppDefault(plugin.Name))
}

func (d Dispatch) UninstallApplication(app string) revel.Result {
	u := GetUserId(d.Session)

	var apps []*models.UserApp
	_, err := d.Txn.Select(&apps, "select * from dispatch_app where userid = $1 and UPPER(name) = UPPER($2)", u, app)
	if err != nil {
		panic(err)
	}
	if len(apps) != 1 {
		panic(len(apps))
	}

	toDelete := apps[0]
	count, err := d.Txn.Delete(toDelete)
	if err != nil || count != 1 {
		panic(err)
	}

	return d.Redirect(routes.Dispatch.Applications())
}

func (c Dispatch) Account() revel.Result {
	if c.Session["user"] == "" {
		return c.Redirect(routes.App.Login())
	}
	var users []*models.User

	_, err := c.Txn.Select(&users, "select * from dispatch_user where userid = $1", GetUserId(c.Session))
	if err != nil {
		panic(err)
	}
	user := users[0]

	var subscriptions []*models.UserSubscription
	_, err = c.Txn.Select(&subscriptions, "select * from dispatch_subscription where userid = $1", user.UserId)
	if err != nil {
		panic(err)
	}

	var identities []*models.Identity
	_, err = c.Txn.Select(&identities, "select * from dispatch_identity where userid = $1", user.UserId)
	if err != nil {
		panic(err)
	}

	return c.Render(user, subscriptions, identities)
}

func (c Dispatch) AddSubscription(address string) revel.Result {
	// TODO: Verify the Address Somehow...
	mailserver.InitRouter()
	_, err := mailserver.LookupRouter.LookupAlias(address)
	if err != nil {
		c.Flash.Error("Unable to find that address. It has not been added.")
		return c.Redirect(routes.Dispatch.Account())
	}

	user := GetUserId(c.Session)
	newSub := &models.UserSubscription{
		UserId:  user,
		Address: address,
	}

	err = c.Txn.Insert(newSub)
	if err != nil {
		panic(err)
	}

	return c.Redirect(routes.Dispatch.Account())
}

func (d Dispatch) RemoveSubscription(id int) revel.Result {
	u := GetUserId(d.Session)

	var apps []*models.UserSubscription
	_, err := d.Txn.Select(&apps, "select * from dispatch_subscription where subscriptionid = $1 and userid = $2", id, u)
	if err != nil {
		panic(err)
	}
	if len(apps) != 1 {
		panic(len(apps))
	}

	toDelete := apps[0]
	count, err := d.Txn.Delete(toDelete)
	if err != nil || count != 1 {
		panic(err)
	}

	return d.Redirect(routes.Dispatch.Account())
}

func (d Dispatch) RegisterIdentity(id int) revel.Result {
	u := GetUserId(d.Session)

	var ids []*models.Identity
	_, err := d.Txn.Select(&ids, "select * from dispatch_identity where identityid = $1 and userid = $2", id, u)
	if err != nil {
		panic(err)
	}
	if len(ids) != 1 {
		panic(len(ids))
	}

	toRegister := ids[0]

	mailserver.InitRouter()
	did, err := toRegister.ToDispatch()
	if err != nil {
		panic(err)
	}
	did.SetLocation(revel.Config.StringDefault("server.location", ""))

	err = mailserver.RegistrationRouter.Register(did, toRegister.Alias)
	if err != nil {
		panic(err)
	}

	return d.Redirect(routes.Dispatch.Account())
}

func (d Dispatch) ProcessAccount(name string, username string, password1 string, password2 string, password string) revel.Result {
	u, err := d.Txn.Get(&models.User{}, GetUserId(d.Session))
	if err != nil {
		panic(err)
	}

	user := u.(*models.User)
	user.Name = name

	if user.VerifyPassword(password) {
		if password1 == password2 {
			user.UpdatePassword(password1)
		} else {
			d.Flash.Error("New passwords do not match.")
		}
	} else {
		if password != "" {
			d.Flash.Error("Current Password is not correct. Did not update username or password.")
		}
	}

	user.Save(d.Txn)
	return d.Redirect(routes.Dispatch.Account())
}
<|endoftext|>


<commit_before>// Copyright 2014 <chaishushan{AT}gmail.com>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package webp

import (
	"image"
	"image/color"
)

var (
	_ color.Color = (*RGBColor)(nil)
	_ image.Image = (*RGBImage)(nil)
)

var RGBModel color.Model = color.ModelFunc(rgbModel)

type RGBColor struct {
	R, G, B uint8
}

func (c RGBColor) RGBA() (r, g, b, a uint32) {
	r = uint32(c.R)
	r |= r << 8
	g = uint32(c.G)
	g |= g << 8
	b = uint32(c.B)
	b |= b << 8
	a = 0xffff
	return
}

func rgbModel(c color.Color) color.Color {
	if _, ok := c.(RGBColor); ok {
		return c
	}
	r, g, b, _ := c.RGBA()
	return RGBColor{R: uint8(r >> 8), G: uint8(g >> 8), B: uint8(b >> 8)}
}

type RGBImage struct {
	Pix    []uint8
	Stride int
	Rect   image.Rectangle
}

func (p *RGBImage) ColorModel() color.Model { return color.RGBAModel }

func (p *RGBImage) Bounds() image.Rectangle { return p.Rect }

func (p *RGBImage) At(x, y int) color.Color {
	return p.RGBAt(x, y)
}

func (p *RGBImage) RGBAt(x, y int) RGBColor {
	if !(image.Point{x, y}.In(p.Rect)) {
		return RGBColor{}
	}
	i := p.PixOffset(x, y)
	return RGBColor{
		R: p.Pix[i+0],
		G: p.Pix[i+1],
		B: p.Pix[i+2],
	}
}

// PixOffset returns the index of the first element of Pix that corresponds to
// the pixel at (x, y).
func (p *RGBImage) PixOffset(x, y int) int {
	return (y-p.Rect.Min.Y)*p.Stride + (x-p.Rect.Min.X)*3
}

func (p *RGBImage) Set(x, y int, c color.Color) {
	if !(image.Point{x, y}.In(p.Rect)) {
		return
	}
	i := p.PixOffset(x, y)
	c1 := RGBModel.Convert(c).(RGBColor)
	p.Pix[i+0] = c1.R
	p.Pix[i+1] = c1.G
	p.Pix[i+2] = c1.B
	return
}

func (p *RGBImage) SetRGB(x, y int, c RGBColor) {
	if !(image.Point{x, y}.In(p.Rect)) {
		return
	}
	i := p.PixOffset(x, y)
	p.Pix[i+0] = c.R
	p.Pix[i+1] = c.G
	p.Pix[i+2] = c.B
	return
}

// SubImage returns an image representing the portion of the image p visible
// through r. The returned value shares pixels with the original image.
func (p *RGBImage) SubImage(r image.Rectangle) image.Image {
	r = r.Intersect(p.Rect)
	// If r1 and r2 are Rectangles, r1.Intersect(r2) is not guaranteed to be inside
	// either r1 or r2 if the intersection is empty. Without explicitly checking for
	// this, the Pix[i:] expression below can panic.
	if r.Empty() {
		return &RGBImage{}
	}
	i := p.PixOffset(r.Min.X, r.Min.Y)
	return &RGBImage{
		Pix:    p.Pix[i:],
		Stride: p.Stride,
		Rect:   r,
	}
}

// Opaque scans the entire image and reports whether it is fully opaque.
func (p *RGBImage) Opaque() bool {
	return true
}

// NewRGBImage returns a new RGBImage with the given bounds.
func NewRGBImage(r image.Rectangle) *RGBImage {
	w, h := r.Dx(), r.Dy()
	pix := make([]uint8, 3*w*h)
	return &RGBImage{
		Pix:    pix,
		Stride: 3 * w,
		Rect:   r,
	}
}

func NewRGBImageFrom(m image.Image) *RGBImage {
	if m, ok := m.(*RGBImage); ok {
		return m
	}

	// convert to RGBImage
	b := m.Bounds()
	rgb := NewRGBImage(b)
	for y := b.Min.Y; y < b.Max.Y; y++ {
		for x := b.Min.X; x < b.Max.X; x++ {
			pr, pg, pb, _ := m.At(x, y).RGBA()
			rgb.SetRGB(x, y, RGBColor{
				R: uint8(pr >> 8),
				G: uint8(pg >> 8),
				B: uint8(pb >> 8),
			})
		}
	}
	return rgb
}
<commit_msg>sync RGBImage type, see https://github.com/chai2010/image<commit_after>// Copyright 2014 <chaishushan{AT}gmail.com>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package webp

import (
	"image"
	"image/color"
)

var (
	_ RGBImager   = (*RGBImage)(nil)
	_ image.Image = (*RGBImage)(nil)
)

type RGBImager interface {
	image.Image
	RGBAt(x, y int) [3]uint8
	SetRGB(x, y int, c [3]uint8)
}

type RGBImage struct {
	Pix    []uint8
	Stride int
	Rect   image.Rectangle
}

func (p *RGBImage) ColorModel() color.Model { return color.RGBAModel }

func (p *RGBImage) Bounds() image.Rectangle { return p.Rect }

func (p *RGBImage) At(x, y int) color.Color {
	if !(image.Point{x, y}.In(p.Rect)) {
		return color.RGBA{}
	}
	i := p.PixOffset(x, y)
	return color.RGBA{
		R: p.Pix[i+0],
		G: p.Pix[i+1],
		B: p.Pix[i+2],
		A: 0xff,
	}
}

func (p *RGBImage) RGBAt(x, y int) [3]uint8 {
	if !(image.Point{x, y}.In(p.Rect)) {
		return [3]uint8{}
	}
	i := p.PixOffset(x, y)
	return [3]uint8{
		p.Pix[i+0],
		p.Pix[i+1],
		p.Pix[i+2],
	}
}

// PixOffset returns the index of the first element of Pix that corresponds to
// the pixel at (x, y).
func (p *RGBImage) PixOffset(x, y int) int {
	return (y-p.Rect.Min.Y)*p.Stride + (x-p.Rect.Min.X)*3
}

func (p *RGBImage) Set(x, y int, c color.Color) {
	if !(image.Point{x, y}.In(p.Rect)) {
		return
	}
	i := p.PixOffset(x, y)
	c1 := color.RGBAModel.Convert(c).(color.RGBA)
	p.Pix[i+0] = c1.R
	p.Pix[i+1] = c1.G
	p.Pix[i+2] = c1.B
	return
}

func (p *RGBImage) SetRGB(x, y int, c [3]uint8) {
	if !(image.Point{x, y}.In(p.Rect)) {
		return
	}
	i := p.PixOffset(x, y)
	p.Pix[i+0] = c[0]
	p.Pix[i+1] = c[1]
	p.Pix[i+2] = c[2]
	return
}

// SubImage returns an image representing the portion of the image p visible
// through r. The returned value shares pixels with the original image.
func (p *RGBImage) SubImage(r image.Rectangle) image.Image {
	r = r.Intersect(p.Rect)
	// If r1 and r2 are Rectangles, r1.Intersect(r2) is not guaranteed to be inside
	// either r1 or r2 if the intersection is empty. Without explicitly checking for
	// this, the Pix[i:] expression below can panic.
	if r.Empty() {
		return &RGBImage{}
	}
	i := p.PixOffset(r.Min.X, r.Min.Y)
	return &RGBImage{
		Pix:    p.Pix[i:],
		Stride: p.Stride,
		Rect:   r,
	}
}

// Opaque scans the entire image and reports whether it is fully opaque.
func (p *RGBImage) Opaque() bool {
	return true
}

// NewRGBImage returns a new RGBImage with the given bounds.
func NewRGBImage(r image.Rectangle) *RGBImage {
	w, h := r.Dx(), r.Dy()
	pix := make([]uint8, 3*w*h)
	return &RGBImage{
		Pix:    pix,
		Stride: 3 * w,
		Rect:   r,
	}
}

func NewRGBImageFrom(m image.Image) *RGBImage {
	if m, ok := m.(*RGBImage); ok {
		return m
	}

	// convert to RGBImage
	b := m.Bounds()
	rgb := NewRGBImage(b)
	for y := b.Min.Y; y < b.Max.Y; y++ {
		for x := b.Min.X; x < b.Max.X; x++ {
			pr, pg, pb, _ := m.At(x, y).RGBA()
			rgb.SetRGB(x, y, [3]uint8{
				uint8(pr >> 8),
				uint8(pg >> 8),
				uint8(pb >> 8),
			})
		}
	}
	return rgb
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"text/template"
	"time"

	"github.com/juju/errors"
	"github.com/rancher/go-rancher-metadata/metadata"
	"gopkg.in/pipe.v2"
)

const (
	DEFAULT_TEMPLATE_DIR = "/etc/rancher-meta-template/templates"
)

//////////////////////////////////////////////////////////////////////////////
func createTemplateCtx(meta *metadata.Client) (interface{}, error) {

	services, err := meta.GetServices()
	if err != nil {
		return nil, errors.Annotate(err, "get services")
	}
	containers, err := meta.GetContainers()
	if err != nil {
		return nil, errors.Annotate(err, "get containers")
	}

	containersW := make([]ContainerWrap, 0)
	for _, container := range containers {
		cw := ContainerWrap{container}
		containersW = append(containersW, cw)
	}

	servicesW := make([]ServiceWrap, 0)
	for _, service := range services {
		sw := ServiceWrap{service}
		servicesW = append(servicesW, sw)
	}

	ctx := map[string]interface{}{
		"Services":   servicesW,
		"Containers": containersW,
	}

	return ctx, nil
}

//////////////////////////////////////////////////////////////////////////////
func computeMd5(filePath string) (string, error) {
	if _, err := os.Stat(filePath); err != nil {
		return "", nil
	}

	file, err := os.Open(filePath)
	if err != nil {
		return "", err
	}
	defer file.Close()

	hash := md5.New()
	if _, err := io.Copy(hash, file); err != nil {
		return "", err
	}

	return fmt.Sprintf("%x", hash.Sum(nil)), nil
}

//////////////////////////////////////////////////////////////////////////////////
func appendCommandPipe(cmd Command, pipes []pipe.Pipe) []pipe.Pipe {
	if cmd.Cmd != "" {
		if cmd.Args != nil {
			return append(pipes, pipe.Exec(cmd.Cmd, cmd.Args...))
		}
		return append(pipes, pipe.Exec(cmd.Cmd))
	}

	return pipes
}

//////////////////////////////////////////////////////////////////////////////////
func processTemplateSet(meta *metadata.Client, set TemplateSet) error {

	if _, err := os.Stat(set.TemplatePath); err != nil {
		printWarning("template path %q is not available: skip", set.TemplatePath)
		return nil
	}

	buf, err := ioutil.ReadFile(set.TemplatePath)
	if err != nil {
		return errors.Annotate(err, "read template file")
	}

	templ := template.New(set.Name).Funcs(newFuncMap())
	tmpl, err := templ.Parse(string(buf))
	if err != nil {
		return errors.Annotate(err, "parse template")
	}

	lastMd5 := ""
	if _, err = os.Stat(set.DestinationPath); err == nil {
		lmd5, err := computeMd5(set.DestinationPath)
		if err != nil {
			return errors.Annotate(err, "get last md5 hash")
		}
		lastMd5 = lmd5
	}

	ctx, err := createTemplateCtx(meta)
	if err != nil {
		return errors.Annotate(err, "create template context")
	}

	var newBuf bytes.Buffer
	wr := bufio.NewWriter(&newBuf)
	if err := tmpl.Execute(wr, ctx); err != nil {
		return errors.Annotate(err, "execute template")
	}

	if err := wr.Flush(); err != nil {
		return errors.Annotate(err, "flush tmpl writer")
	}

	hash := md5.New()
	hash.Write(newBuf.Bytes())
	currentMd5 := fmt.Sprintf("%x", hash.Sum(nil))

	if lastMd5 == currentMd5 {
		return nil
	}

	if lastMd5 == "" {
		printInfo("create output file")
	} else {
		printInfo("output file needs refresh")
		printInfo("last md5 sum is %q", lastMd5)
		printInfo("current md5 sum is %q", currentMd5)
	}

	f, err := os.Create(set.DestinationPath)
	if err != nil {
		return errors.Annotate(err, "create destination file")
	}
	defer f.Close()

	w := bufio.NewWriter(f)
	if _, err := w.Write(newBuf.Bytes()); err != nil {
		return errors.Annotate(err, "write to output")
	}

	if err := w.Flush(); err != nil {
		return errors.Annotate(err, "flush out writer")
	}

	printInfo("process check & run")

	pipes := make([]pipe.Pipe, 0)
	pipes = appendCommandPipe(set.Check, pipes)
	pipes = appendCommandPipe(set.Run, pipes)

	script := pipe.Script(pipes...)
	if output, err := pipe.CombinedOutput(script); err != nil {
		printInfo(string(output))
		return errors.Annotate(err, "check & run")
	}

	return nil
}

//////////////////////////////////////////////////////////////////////////////////
func processTemplates(cnf *Config) error {

	apiURL := fmt.Sprintf("%s%s", cnf.Host, cnf.Prefix)
	meta := metadata.NewClient(apiURL)

	printInfo("connect rancher metadata url: %q", apiURL)

	//expand template paths
	printDebug("expand template paths")
	for idx, set := range cnf.Sets {
		if !path.IsAbs(set.TemplatePath) {
			cnf.Sets[idx].TemplatePath = path.Join(DEFAULT_TEMPLATE_DIR, set.TemplatePath)
		}
	}

	version := "init"
	for {
		newVersion, err := meta.GetVersion()
		if err != nil {
			return errors.Annotate(err, "get version")
		}

		if newVersion == version {
			time.Sleep(5 * time.Second)
			continue
		}

		version = newVersion
		printInfo("metadata changed - refresh config")

		for _, set := range cnf.Sets {
			if err := processTemplateSet(meta, set); err != nil {
				return errors.Annotate(err, "process template set")
			}
		}

		time.Sleep(time.Duration(cnf.Repeat) * time.Second)
	}

	return nil
}
<commit_msg>autopush@1447082325<commit_after>package main

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"text/template"
	"time"

	"github.com/juju/errors"
	"github.com/rancher/go-rancher-metadata/metadata"
	"gopkg.in/pipe.v2"
)

const (
	DEFAULT_TEMPLATE_DIR = "/etc/rancher-meta-template/templates"
)

//////////////////////////////////////////////////////////////////////////////
func createTemplateCtx(meta *metadata.Client) (interface{}, error) {

	services, err := meta.GetServices()
	if err != nil {
		return nil, errors.Annotate(err, "get services")
	}
	containers, err := meta.GetContainers()
	if err != nil {
		return nil, errors.Annotate(err, "get containers")
	}

	containersW := make([]ContainerWrap, 0)
	for _, container := range containers {
		cw := ContainerWrap{container}
		containersW = append(containersW, cw)
	}

	servicesW := make([]ServiceWrap, 0)
	for _, service := range services {
		sw := ServiceWrap{service}
		servicesW = append(servicesW, sw)
	}

	ctx := map[string]interface{}{
		"Services":   servicesW,
		"Containers": containersW,
	}

	return ctx, nil
}

//////////////////////////////////////////////////////////////////////////////
func computeMd5(filePath string) (string, error) {
	if _, err := os.Stat(filePath); err != nil {
		return "", nil
	}

	file, err := os.Open(filePath)
	if err != nil {
		return "", err
	}
	defer file.Close()

	hash := md5.New()
	if _, err := io.Copy(hash, file); err != nil {
		return "", err
	}

	return fmt.Sprintf("%x", hash.Sum(nil)), nil
}

//////////////////////////////////////////////////////////////////////////////////
func appendCommandPipe(cmd Command, pipes []pipe.Pipe) []pipe.Pipe {
	if cmd.Cmd != "" {
		if cmd.Args != nil {
			return append(pipes, pipe.Exec(cmd.Cmd, cmd.Args...))
		}
		return append(pipes, pipe.Exec(cmd.Cmd))
	}

	return pipes
}

//////////////////////////////////////////////////////////////////////////////////
func processTemplateSet(meta *metadata.Client, set TemplateSet) error {

	if _, err := os.Stat(set.TemplatePath); err != nil {
		printWarning("template path %q is not available: skip", set.TemplatePath)
		return nil
	}

	buf, err := ioutil.ReadFile(set.TemplatePath)
	if err != nil {
		return errors.Annotate(err, "read template file")
	}

	templ := template.New(set.Name).Funcs(newFuncMap())
	tmpl, err := templ.Parse(string(buf))
	if err != nil {
		return errors.Annotate(err, "parse template")
	}

	lastMd5 := ""
	if _, err = os.Stat(set.DestinationPath); err == nil {
		lmd5, err := computeMd5(set.DestinationPath)
		if err != nil {
			return errors.Annotate(err, "get last md5 hash")
		}
		lastMd5 = lmd5
	}

	ctx, err := createTemplateCtx(meta)
	if err != nil {
		return errors.Annotate(err, "create template context")
	}

	var newBuf bytes.Buffer
	wr := bufio.NewWriter(&newBuf)
	if err := tmpl.Execute(wr, ctx); err != nil {
		return errors.Annotate(err, "execute template")
	}

	if err := wr.Flush(); err != nil {
		return errors.Annotate(err, "flush tmpl writer")
	}

	hash := md5.New()
	hash.Write(newBuf.Bytes())
	currentMd5 := fmt.Sprintf("%x", hash.Sum(nil))

	if lastMd5 == currentMd5 {
		return nil
	}

	if lastMd5 == "" {
		printInfo("create output file")
	} else {
		printInfo("last md5 sum is %q", lastMd5)
		printInfo("current md5 sum is %q", currentMd5)
		printInfo("output file needs refresh")
	}

	f, err := os.Create(set.DestinationPath)
	if err != nil {
		return errors.Annotate(err, "create destination file")
	}
	defer f.Close()

	w := bufio.NewWriter(f)
	if _, err := w.Write(newBuf.Bytes()); err != nil {
		return errors.Annotate(err, "write to output")
	}

	if err := w.Flush(); err != nil {
		return errors.Annotate(err, "flush out writer")
	}

	printInfo("process check & run")

	pipes := make([]pipe.Pipe, 0)
	pipes = appendCommandPipe(set.Check, pipes)
	pipes = appendCommandPipe(set.Run, pipes)

	script := pipe.Script(pipes...)
	if output, err := pipe.CombinedOutput(script); err != nil {
		printInfo(string(output))
		return errors.Annotate(err, "check & run")
	}

	return nil
}

//////////////////////////////////////////////////////////////////////////////////
func processTemplates(cnf *Config) error {

	apiURL := fmt.Sprintf("%s%s", cnf.Host, cnf.Prefix)
	meta := metadata.NewClient(apiURL)

	printInfo("connect rancher metadata url: %q", apiURL)

	//expand template paths
	printDebug("expand template paths")
	for idx, set := range cnf.Sets {
		if !path.IsAbs(set.TemplatePath) {
			cnf.Sets[idx].TemplatePath = path.Join(DEFAULT_TEMPLATE_DIR, set.TemplatePath)
		}
	}

	version := "init"
	for {
		newVersion, err := meta.GetVersion()
		if err != nil {
			return errors.Annotate(err, "get version")
		}

		if newVersion == version {
			time.Sleep(5 * time.Second)
			continue
		}

		version = newVersion
		printInfo("metadata changed - refresh config")

		for _, set := range cnf.Sets {
			if err := processTemplateSet(meta, set); err != nil {
				return errors.Annotate(err, "process template set")
			}
		}

		time.Sleep(time.Duration(cnf.Repeat) * time.Second)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/pprof"
	"os"
	"time"

	"cloud.google.com/go/pubsub"
	cloudtrace "cloud.google.com/go/trace"
	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
	"github.com/google/go-github/github"
	flags "github.com/jessevdk/go-flags"
	"github.com/pivotal-cf/paraphernalia/operate/admin"
	"github.com/pivotal-cf/paraphernalia/secure/tlsconfig"
	"github.com/pivotal-cf/paraphernalia/serve/grpcrunner"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/sigmon"
	"golang.org/x/net/trace"
	"golang.org/x/oauth2"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	"cred-alert/config"
	"cred-alert/crypto"
	"cred-alert/db"
	"cred-alert/db/migrations"
	"cred-alert/gitclient"
	"cred-alert/metrics"
	"cred-alert/notifications"
	"cred-alert/queue"
	"cred-alert/revok"
	"cred-alert/revok/stats"
	"cred-alert/revokpb"
	"cred-alert/search"
	"cred-alert/sniff"
	"rolodex/rolodexpb"
)

var info = admin.ServiceInfo{
	Name:        "revok",
	Description: "A service which fetches new Git commits and scans them for credentials.",
	Team:        "PCF Security Enablement",
}

func main() {
	var cfg *config.WorkerConfig
	var flagOpts config.WorkerOpts

	logger := lager.NewLogger("revok-worker")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))

	logger.Info("starting")

	_, err := flags.Parse(&flagOpts)
	if err != nil {
		os.Exit(1)
	}

	bs, err := ioutil.ReadFile(string(flagOpts.ConfigFile))
	if err != nil {
		logger.Error("failed-to-open-config-file", err)
		os.Exit(1)
	}

	cfg, err = config.LoadWorkerConfig(bs)
	if err != nil {
		logger.Error("failed-to-load-config-file", err)
		os.Exit(1)
	}

	errs := cfg.Validate()
	if errs != nil {
		for _, err := range errs {
			fmt.Println(err.Error())
		}
		os.Exit(1)
	}

	if cfg.Metrics.SentryDSN != "" {
		logger.RegisterSink(revok.NewSentrySink(cfg.Metrics.SentryDSN, cfg.Metrics.Environment))
	}

	workdir := cfg.WorkDir
	_, err = os.Lstat(workdir)
	if err != nil {
		log.Fatalf("workdir error: %s", err)
	}

	dbCertificate, dbCaCertPool := loadCerts(
		cfg.MySQL.CertificatePath,
		cfg.MySQL.PrivateKeyPath,
		cfg.MySQL.PrivateKeyPassphrase,
		cfg.MySQL.CACertificatePath,
	)

	dbURI := db.NewDSN(
		cfg.MySQL.Username,
		cfg.MySQL.Password,
		cfg.MySQL.DBName,
		cfg.MySQL.Hostname,
		int(cfg.MySQL.Port),
		cfg.MySQL.ServerName,
		dbCertificate,
		dbCaCertPool,
	)

	database, err := migrations.LockDBAndMigrate(logger, "mysql", dbURI)
	if err != nil {
		log.Fatalf("db error: %s", err)
	}

	database.LogMode(false)

	clk := clock.NewClock()

	cloneMsgCh := make(chan revok.CloneMsg)

	scanRepository := db.NewScanRepository(database, clk)
	repositoryRepository := db.NewRepositoryRepository(database)
	fetchRepository := db.NewFetchRepository(database)
	credentialRepository := db.NewCredentialRepository(database)
	branchRepository := db.NewBranchRepository(database)

	emitter := metrics.BuildEmitter(cfg.Metrics.DatadogAPIKey, cfg.Metrics.Environment)
	gitClient := gitclient.New(cfg.GitHub.PrivateKeyPath, cfg.GitHub.PublicKeyPath)
	repoWhitelist := notifications.BuildWhitelist(cfg.Whitelist...)
	formatter := notifications.NewSlackNotificationFormatter()

	slackHTTPClient := &http.Client{
		Timeout: 3 * time.Second,
	}
	notifier := notifications.NewSlackNotifier(clk, slackHTTPClient, formatter)

	certificate, caCertPool := loadCerts(
		cfg.Identity.CertificatePath,
		cfg.Identity.PrivateKeyPath,
		cfg.Identity.PrivateKeyPassphrase,
		cfg.Identity.CACertificatePath,
	)

	rolodexServerAddr := fmt.Sprintf("%s:%d", cfg.Rolodex.ServerAddress, cfg.Rolodex.ServerPort)

	tlsConfig := tlsconfig.Build(
		tlsconfig.WithPivotalDefaults(),
		tlsconfig.WithIdentity(certificate),
	)

	transportCreds := credentials.NewTLS(tlsConfig.Client(tlsconfig.WithAuthority(caCertPool)))

	traceClient, err := cloudtrace.NewClient(context.Background(), cfg.Trace.ProjectName)
	if err != nil {
		logger.Error("failed-to-create-trace-client", err)
	}

	conn, err := grpc.Dial(
		rolodexServerAddr,
		grpc.WithDialer(keepAliveDial),
		grpc.WithTransportCredentials(transportCreds),
		grpc.WithUnaryInterceptor(cloudtrace.GRPCClientInterceptor()),
	)

	rolodexClient := rolodexpb.NewRolodexClient(conn)

	teamURLs := notifications.NewTeamURLs(
		cfg.Slack.DefaultURL,
		cfg.Slack.DefaultChannel,
		cfg.Slack.TeamURLs,
	)

	addressBook := notifications.NewRolodex(
		rolodexClient,
		teamURLs,
	)

	router := notifications.NewRouter(
		notifier,
		addressBook,
		repoWhitelist,
	)

	sniffer := sniff.NewDefaultSniffer()
	scanner := revok.NewScanner(
		gitClient,
		repositoryRepository,
		scanRepository,
		credentialRepository,
		sniffer,
	)

	notificationComposer := revok.NewNotificationComposer(
		repositoryRepository,
		router,
		scanner,
	)

	changeFetcher := revok.NewChangeFetcher(
		logger,
		gitClient,
		notificationComposer,
		repositoryRepository,
		fetchRepository,
		emitter,
	)

	changeScheduleRunner := revok.NewScheduleRunner(logger)

	changeScheduler := revok.NewChangeScheduler(
		logger,
		repositoryRepository,
		changeScheduleRunner,
		changeFetcher,
	)

	cloner := revok.NewCloner(
		logger,
		workdir,
		cloneMsgCh,
		gitClient,
		repositoryRepository,
		notificationComposer,
		emitter,
		changeScheduler,
	)

	dirscanUpdater := revok.NewRescanner(
		logger,
		scanRepository,
		credentialRepository,
		scanner,
		router,
		emitter,
	)

	statsReporter := stats.NewReporter(
		logger,
		clk,
		60*time.Second,
		db.NewStatsRepository(database),
		emitter,
	)

	headCredentialCounter := revok.NewHeadCredentialCounter(
		logger,
		branchRepository,
		repositoryRepository,
		clk,
		cfg.CredentialCounterInterval,
		gitClient,
		sniffer,
	)

	debug := admin.Runner(
		"6060",
		admin.WithInfo(info),
		admin.WithUptime(),
	)

	members := []grouper.Member{
		{"cloner", cloner},
		{"dirscan-updater", dirscanUpdater},
		{"stats-reporter", statsReporter},
		{"head-credential-counter", headCredentialCounter},
		{"change-schedule-runner", changeScheduleRunner},
		{"debug", debug},
	}

	looper := gitclient.NewLooper()
	searcher := search.NewSearcher(repositoryRepository, looper)
	handler := revok.NewServer(logger, searcher, repositoryRepository, branchRepository)

	serverTls := tlsConfig.Server(tlsconfig.WithClientAuthentication(caCertPool))

	grpcServer := grpcrunner.New(
		logger,
		fmt.Sprintf("%s:%d", cfg.API.BindIP, cfg.API.BindPort),
		func(server *grpc.Server) {
			revokpb.RegisterRevokServer(server, handler)
		},
		grpc.Creds(credentials.NewTLS(serverTls)),
	)

	members = append(members, grouper.Member{
		Name:   "grpc-server",
		Runner: grpcServer,
	})

	pubSubClient, err := pubsub.NewClient(context.Background(), cfg.PubSub.ProjectName)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}

	subscription := pubSubClient.Subscription(cfg.PubSub.FetchHint.Subscription)

	publicKey, err := crypto.ReadRSAPublicKey(cfg.PubSub.PublicKeyPath)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}
	pushEventProcessor := queue.NewPushEventProcessor(
		changeFetcher,
		crypto.NewRSAVerifier(publicKey),
		emitter,
		clk,
	)

	members = append(members, grouper.Member{
		Name:   "github-hint-handler",
		Runner: queue.NewPubSubSubscriber(logger, subscription, pushEventProcessor, emitter, traceClient),
	})

	if cfg.GitHub.AccessToken != "" {
		githubHTTPClient := &http.Client{
			Timeout: 30 * time.Second,
			Transport: &oauth2.Transport{
				Source: oauth2.StaticTokenSource(
					&oauth2.Token{AccessToken: cfg.GitHub.AccessToken},
				),
				Base: &http.Transport{
					DisableKeepAlives: true,
				},
			},
		}

		ghClient := revok.NewGitHubClient(github.NewClient(githubHTTPClient))

		repoDiscoverer := revok.NewRepoDiscoverer(
			logger,
			workdir,
			cloneMsgCh,
			ghClient,
			clk,
			cfg.RepositoryDiscovery.Interval,
			cfg.RepositoryDiscovery.Organizations,
			cfg.RepositoryDiscovery.Users,
			repositoryRepository,
		)

		members = append(members, grouper.Member{
			Name:   "repo-discoverer",
			Runner: repoDiscoverer,
		})
	}

	startupTasks := []grouper.Member{
		{
			Name:   "schedule-fetches",
			Runner: changeScheduler,
		},
	}

	system := []grouper.Member{
		{
			Name:   "servers",
			Runner: grouper.NewParallel(os.Interrupt, members),
		},
		{
			Name:   "startup-tasks",
			Runner: grouper.NewParallel(os.Interrupt, startupTasks),
		},
	}

	runner := sigmon.New(grouper.NewOrdered(os.Interrupt, system))

	err = <-ifrit.Invoke(runner).Wait()
	if err != nil {
		log.Fatalf("failed-to-start: %s", err)
	}
}

func loadCerts(certificatePath, privateKeyPath, privateKeyPassphrase, caCertificatePath string) (tls.Certificate, *x509.CertPool) {
	certificate, err := config.LoadCertificate(
		certificatePath,
		privateKeyPath,
		privateKeyPassphrase,
	)
	if err != nil {
		log.Fatalln(err)
	}

	caCertPool, err := config.LoadCertificatePool(caCertificatePath)
	if err != nil {
		log.Fatalln(err)
	}

	return certificate, caCertPool
}

func keepAliveDial(addr string, timeout time.Duration) (net.Conn, error) {
	d := net.Dialer{
		Timeout:   timeout,
		KeepAlive: 60 * time.Second,
	}
	return d.Dial("tcp", addr)
}

func debugHandler() http.Handler {
	debugRouter := http.NewServeMux()
	debugRouter.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	debugRouter.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
	debugRouter.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	debugRouter.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	debugRouter.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))

	debugRouter.HandleFunc("/debug/requests", func(w http.ResponseWriter, req *http.Request) {
		any, sensitive := trace.AuthRequest(req)
		if !any {
			http.Error(w, "not allowed", http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		trace.Render(w, req, sensitive)
	})

	debugRouter.HandleFunc("/debug/events", func(w http.ResponseWriter, req *http.Request) {
		any, sensitive := trace.AuthRequest(req)
		if !any {
			http.Error(w, "not allowed", http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		trace.RenderEvents(w, req, sensitive)
	})

	return debugRouter
}
<commit_msg>remove unused function<commit_after>package main

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"time"

	"cloud.google.com/go/pubsub"
	cloudtrace "cloud.google.com/go/trace"
	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
	"github.com/google/go-github/github"
	flags "github.com/jessevdk/go-flags"
	"github.com/pivotal-cf/paraphernalia/operate/admin"
	"github.com/pivotal-cf/paraphernalia/secure/tlsconfig"
	"github.com/pivotal-cf/paraphernalia/serve/grpcrunner"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/sigmon"
	"golang.org/x/oauth2"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	"cred-alert/config"
	"cred-alert/crypto"
	"cred-alert/db"
	"cred-alert/db/migrations"
	"cred-alert/gitclient"
	"cred-alert/metrics"
	"cred-alert/notifications"
	"cred-alert/queue"
	"cred-alert/revok"
	"cred-alert/revok/stats"
	"cred-alert/revokpb"
	"cred-alert/search"
	"cred-alert/sniff"
	"rolodex/rolodexpb"
)

var info = admin.ServiceInfo{
	Name:        "revok",
	Description: "A service which fetches new Git commits and scans them for credentials.",
	Team:        "PCF Security Enablement",
}

func main() {
	var cfg *config.WorkerConfig
	var flagOpts config.WorkerOpts

	logger := lager.NewLogger("revok-worker")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))

	logger.Info("starting")

	_, err := flags.Parse(&flagOpts)
	if err != nil {
		os.Exit(1)
	}

	bs, err := ioutil.ReadFile(string(flagOpts.ConfigFile))
	if err != nil {
		logger.Error("failed-to-open-config-file", err)
		os.Exit(1)
	}

	cfg, err = config.LoadWorkerConfig(bs)
	if err != nil {
		logger.Error("failed-to-load-config-file", err)
		os.Exit(1)
	}

	errs := cfg.Validate()
	if errs != nil {
		for _, err := range errs {
			fmt.Println(err.Error())
		}
		os.Exit(1)
	}

	if cfg.Metrics.SentryDSN != "" {
		logger.RegisterSink(revok.NewSentrySink(cfg.Metrics.SentryDSN, cfg.Metrics.Environment))
	}

	workdir := cfg.WorkDir
	_, err = os.Lstat(workdir)
	if err != nil {
		log.Fatalf("workdir error: %s", err)
	}

	dbCertificate, dbCaCertPool := loadCerts(
		cfg.MySQL.CertificatePath,
		cfg.MySQL.PrivateKeyPath,
		cfg.MySQL.PrivateKeyPassphrase,
		cfg.MySQL.CACertificatePath,
	)

	dbURI := db.NewDSN(
		cfg.MySQL.Username,
		cfg.MySQL.Password,
		cfg.MySQL.DBName,
		cfg.MySQL.Hostname,
		int(cfg.MySQL.Port),
		cfg.MySQL.ServerName,
		dbCertificate,
		dbCaCertPool,
	)

	database, err := migrations.LockDBAndMigrate(logger, "mysql", dbURI)
	if err != nil {
		log.Fatalf("db error: %s", err)
	}

	database.LogMode(false)

	clk := clock.NewClock()

	cloneMsgCh := make(chan revok.CloneMsg)

	scanRepository := db.NewScanRepository(database, clk)
	repositoryRepository := db.NewRepositoryRepository(database)
	fetchRepository := db.NewFetchRepository(database)
	credentialRepository := db.NewCredentialRepository(database)
	branchRepository := db.NewBranchRepository(database)

	emitter := metrics.BuildEmitter(cfg.Metrics.DatadogAPIKey, cfg.Metrics.Environment)
	gitClient := gitclient.New(cfg.GitHub.PrivateKeyPath, cfg.GitHub.PublicKeyPath)
	repoWhitelist := notifications.BuildWhitelist(cfg.Whitelist...)
	formatter := notifications.NewSlackNotificationFormatter()

	slackHTTPClient := &http.Client{
		Timeout: 3 * time.Second,
	}
	notifier := notifications.NewSlackNotifier(clk, slackHTTPClient, formatter)

	certificate, caCertPool := loadCerts(
		cfg.Identity.CertificatePath,
		cfg.Identity.PrivateKeyPath,
		cfg.Identity.PrivateKeyPassphrase,
		cfg.Identity.CACertificatePath,
	)

	rolodexServerAddr := fmt.Sprintf("%s:%d", cfg.Rolodex.ServerAddress, cfg.Rolodex.ServerPort)

	tlsConfig := tlsconfig.Build(
		tlsconfig.WithPivotalDefaults(),
		tlsconfig.WithIdentity(certificate),
	)

	transportCreds := credentials.NewTLS(tlsConfig.Client(tlsconfig.WithAuthority(caCertPool)))

	traceClient, err := cloudtrace.NewClient(context.Background(), cfg.Trace.ProjectName)
	if err != nil {
		logger.Error("failed-to-create-trace-client", err)
	}

	conn, err := grpc.Dial(
		rolodexServerAddr,
		grpc.WithDialer(keepAliveDial),
		grpc.WithTransportCredentials(transportCreds),
		grpc.WithUnaryInterceptor(cloudtrace.GRPCClientInterceptor()),
	)

	rolodexClient := rolodexpb.NewRolodexClient(conn)

	teamURLs := notifications.NewTeamURLs(
		cfg.Slack.DefaultURL,
		cfg.Slack.DefaultChannel,
		cfg.Slack.TeamURLs,
	)

	addressBook := notifications.NewRolodex(
		rolodexClient,
		teamURLs,
	)

	router := notifications.NewRouter(
		notifier,
		addressBook,
		repoWhitelist,
	)

	sniffer := sniff.NewDefaultSniffer()
	scanner := revok.NewScanner(
		gitClient,
		repositoryRepository,
		scanRepository,
		credentialRepository,
		sniffer,
	)

	notificationComposer := revok.NewNotificationComposer(
		repositoryRepository,
		router,
		scanner,
	)

	changeFetcher := revok.NewChangeFetcher(
		logger,
		gitClient,
		notificationComposer,
		repositoryRepository,
		fetchRepository,
		emitter,
	)

	changeScheduleRunner := revok.NewScheduleRunner(logger)

	changeScheduler := revok.NewChangeScheduler(
		logger,
		repositoryRepository,
		changeScheduleRunner,
		changeFetcher,
	)

	cloner := revok.NewCloner(
		logger,
		workdir,
		cloneMsgCh,
		gitClient,
		repositoryRepository,
		notificationComposer,
		emitter,
		changeScheduler,
	)

	dirscanUpdater := revok.NewRescanner(
		logger,
		scanRepository,
		credentialRepository,
		scanner,
		router,
		emitter,
	)

	statsReporter := stats.NewReporter(
		logger,
		clk,
		60*time.Second,
		db.NewStatsRepository(database),
		emitter,
	)

	headCredentialCounter := revok.NewHeadCredentialCounter(
		logger,
		branchRepository,
		repositoryRepository,
		clk,
		cfg.CredentialCounterInterval,
		gitClient,
		sniffer,
	)

	debug := admin.Runner(
		"6060",
		admin.WithInfo(info),
		admin.WithUptime(),
	)

	members := []grouper.Member{
		{"cloner", cloner},
		{"dirscan-updater", dirscanUpdater},
		{"stats-reporter", statsReporter},
		{"head-credential-counter", headCredentialCounter},
		{"change-schedule-runner", changeScheduleRunner},
		{"debug", debug},
	}

	looper := gitclient.NewLooper()
	searcher := search.NewSearcher(repositoryRepository, looper)
	handler := revok.NewServer(logger, searcher, repositoryRepository, branchRepository)

	serverTls := tlsConfig.Server(tlsconfig.WithClientAuthentication(caCertPool))

	grpcServer := grpcrunner.New(
		logger,
		fmt.Sprintf("%s:%d", cfg.API.BindIP, cfg.API.BindPort),
		func(server *grpc.Server) {
			revokpb.RegisterRevokServer(server, handler)
		},
		grpc.Creds(credentials.NewTLS(serverTls)),
	)

	members = append(members, grouper.Member{
		Name:   "grpc-server",
		Runner: grpcServer,
	})

	pubSubClient, err := pubsub.NewClient(context.Background(), cfg.PubSub.ProjectName)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}

	subscription := pubSubClient.Subscription(cfg.PubSub.FetchHint.Subscription)

	publicKey, err := crypto.ReadRSAPublicKey(cfg.PubSub.PublicKeyPath)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}
	pushEventProcessor := queue.NewPushEventProcessor(
		changeFetcher,
		crypto.NewRSAVerifier(publicKey),
		emitter,
		clk,
	)

	members = append(members, grouper.Member{
		Name:   "github-hint-handler",
		Runner: queue.NewPubSubSubscriber(logger, subscription, pushEventProcessor, emitter, traceClient),
	})

	if cfg.GitHub.AccessToken != "" {
		githubHTTPClient := &http.Client{
			Timeout: 30 * time.Second,
			Transport: &oauth2.Transport{
				Source: oauth2.StaticTokenSource(
					&oauth2.Token{AccessToken: cfg.GitHub.AccessToken},
				),
				Base: &http.Transport{
					DisableKeepAlives: true,
				},
			},
		}

		ghClient := revok.NewGitHubClient(github.NewClient(githubHTTPClient))

		repoDiscoverer := revok.NewRepoDiscoverer(
			logger,
			workdir,
			cloneMsgCh,
			ghClient,
			clk,
			cfg.RepositoryDiscovery.Interval,
			cfg.RepositoryDiscovery.Organizations,
			cfg.RepositoryDiscovery.Users,
			repositoryRepository,
		)

		members = append(members, grouper.Member{
			Name:   "repo-discoverer",
			Runner: repoDiscoverer,
		})
	}

	startupTasks := []grouper.Member{
		{
			Name:   "schedule-fetches",
			Runner: changeScheduler,
		},
	}

	system := []grouper.Member{
		{
			Name:   "servers",
			Runner: grouper.NewParallel(os.Interrupt, members),
		},
		{
			Name:   "startup-tasks",
			Runner: grouper.NewParallel(os.Interrupt, startupTasks),
		},
	}

	runner := sigmon.New(grouper.NewOrdered(os.Interrupt, system))

	err = <-ifrit.Invoke(runner).Wait()
	if err != nil {
		log.Fatalf("failed-to-start: %s", err)
	}
}

func loadCerts(certificatePath, privateKeyPath, privateKeyPassphrase, caCertificatePath string) (tls.Certificate, *x509.CertPool) {
	certificate, err := config.LoadCertificate(
		certificatePath,
		privateKeyPath,
		privateKeyPassphrase,
	)
	if err != nil {
		log.Fatalln(err)
	}

	caCertPool, err := config.LoadCertificatePool(caCertificatePath)
	if err != nil {
		log.Fatalln(err)
	}

	return certificate, caCertPool
}

func keepAliveDial(addr string, timeout time.Duration) (net.Conn, error) {
	d := net.Dialer{
		Timeout:   timeout,
		KeepAlive: 60 * time.Second,
	}
	return d.Dial("tcp", addr)
}
<|endoftext|>


<commit_before>package test

import (
	_ "github.com/boredomist/gob/parse"
	_ "os"
	"testing"
)

var tests = []string{"convert.b", "copy.b", "lower.b", "snide.b"}

func TestExampleDummy(t *testing.T) {
}

// TODO: parser is not quite ready for this, and I'd rather not have every test fail
// func TestExamples(t *testing.T) {
// 	for _, test := range tests {
// 		if file, err := os.Open(test); err != nil {
// 			t.Errorf("failed to open test: %s", err)
// 		} else {
// 			p := parse.NewParser(test, file)
// 			if _, err := p.Parse(); err != nil {
// 				t.Errorf("%s failed: %v", test, err)
// 			}
// 		}
// 	}
// }
<commit_msg>Uncomment (currently failing) tests<commit_after>package test

import (
	"github.com/boredomist/gob/parse"
	"os"
	"testing"
)

var tests = []string{"convert.b", "copy.b", "lower.b", "snide.b"}

func TestExampleDummy(t *testing.T) {
}

// TODO: parser is not quite ready for this, and I'd rather not have every test fail
func TestExamples(t *testing.T) {
	for _, test := range tests {
		if file, err := os.Open(test); err != nil {
			t.Errorf("failed to open test: %s", err)
		} else {
			p := parse.NewParser(test, file)
			if _, err := p.Parse(); err != nil {
				t.Errorf("%s failed: %v", test, err)
			}
		}
	}
}
<|endoftext|>


<commit_before>package disk

import (
	"bytes"
	"io/ioutil"
	"os"
	"testing"

	"github.com/xephonhq/xephon-k/pkg/storage/disk"
	//"encoding/binary"
	"fmt"
	"encoding/binary"
)

const (
	magicnumber uint64 = 0x786570686F6E2D6B
)

// writing series to disk without any compression and then read it out
type fileHeader struct {
	version          uint8
	timeCompression  uint8
	valueCompression uint8
}

// NOTE: must pass a pointer of buffer
func (header *fileHeader) write(buf *bytes.Buffer) {
	b := make([]byte, 8)
	binary.BigEndian.PutUint64(b, magicnumber)
	buf.Write(b)
	buf.WriteByte(header.version)
	buf.WriteByte(header.timeCompression)
	buf.WriteByte(header.valueCompression)
}

func TestMagicNumber(t *testing.T) {
	var str = "xephon-k"
	t.Log(len([]byte(str))) // 8 byte, uint64
	t.Log([]byte(str))
	// [120 101 112 104 111 110 45 107]
	// 78 65 70 68 6F 6E 2D 6B
	t.Logf("% X", []byte(str))
	t.Logf("%X", []byte(str))
	t.Log([]byte(str)[0])

	// convert the magic number into binary
	t.Log(magicnumber)
	b := make([]byte, 10)
	// FIXME: it takes 9 byte instead of 8 byte to write a uint64 http://stackoverflow.com/questions/17289898/does-a-uint64-take-8-bytes-storage
	t.Log(binary.PutUvarint(b, magicnumber)) // 9 instead of 8
	t.Log(b)
	v, err := binary.ReadUvarint(bytes.NewReader(b))
	if err != nil {
		t.Fatal(err)
	}
	t.Log(v)
	// this use 8 byte
	binary.BigEndian.PutUint64(b, magicnumber)
	t.Log(b)
	t.Log(binary.BigEndian.Uint64(b))

	// Uvarint would use less than 8 byte for small value
	t.Log(binary.PutUvarint(b, 1)) // 1
	t.Log(binary.PutUvarint(b, 256)) // 2
}

func TestNoCompress_Header(t *testing.T) {
	header := fileHeader{version: 1, timeCompression: disk.CompressionNone, valueCompression: disk.CompressionNone}
	//header := fileHeader{version: 1, timeCompression: disk.CompressionGzip, valueCompression: disk.CompressionZlib}
	tmpfile, err := ioutil.TempFile("", "xephon-no-compress")
	if err != nil {
		t.Fatal(err)
	}

	//defer os.Remove(tmpfile.Name())

	var buf bytes.Buffer
	// TODO: Endianness problem https://github.com/xephonhq/xephon-k/issues/34
	// but it seems for single uint8, this is not a problem
	//binary.Write(&buf, binary.LittleEndian, header.version)
	//binary.Write(&buf, binary.LittleEndian, header.timeCompression)
	//binary.Write(&buf, binary.LittleEndian, header.valueCompression)

	header.write(&buf)

	n, err := tmpfile.Write(buf.Bytes())
	t.Logf("written %d bytes\n", n)
	if err != nil {
		t.Fatal(err)
	}
	tmpfile.Close()

	// read stuff back
	f, err := os.Open(tmpfile.Name())
	readBuf := make([]byte, 11)
	n, err = f.Read(readBuf)
	t.Logf("read %d bytes\n", n)
	if err != nil {
		t.Fatal(err)
	}
	f.Close()
	// convert to header
	newHeader := fileHeader{}
	if binary.BigEndian.Uint64(readBuf[:8]) != magicnumber {
		t.Fatal("magic number does not match!")
	} else {
		t.Log("magic number match")
	}
	newHeader.version = uint8(readBuf[8])
	newHeader.timeCompression = uint8(readBuf[9])
	newHeader.valueCompression = uint8(readBuf[10])
	fmt.Printf("version %d, time compression %d, value compression %d\n",
		newHeader.version, newHeader.timeCompression, newHeader.valueCompression)
}

func TestNoCompress_Block(t *testing.T) {

}
<commit_msg>[play][disk] Init block writer #32<commit_after>package disk

import (
	"bytes"
	"io/ioutil"
	"os"
	"testing"

	"github.com/xephonhq/xephon-k/pkg/storage/disk"
	//"encoding/binary"
	"fmt"
	"encoding/binary"
	"io"
	"bufio"
	"github.com/xephonhq/xephon-k/pkg/common"
	"github.com/pkg/errors"
)

const (
	magicnumber uint64 = 0x786570686F6E2D6B
)

// writing series to disk without any compression and then read it out
type fileHeader struct {
	version          uint8
	timeCompression  uint8
	valueCompression uint8
}

// NOTE: must pass a pointer of buffer
func (header *fileHeader) write(buf *bytes.Buffer) {
	b := make([]byte, 8)
	binary.BigEndian.PutUint64(b, magicnumber)
	buf.Write(b)
	buf.WriteByte(header.version)
	buf.WriteByte(header.timeCompression)
	buf.WriteByte(header.valueCompression)
}

func (header *fileHeader) Bytes() []byte {
	var buf bytes.Buffer
	b := make([]byte, 8)
	binary.BigEndian.PutUint64(b, magicnumber)
	buf.Write(b)
	buf.WriteByte(header.version)
	buf.WriteByte(header.timeCompression)
	buf.WriteByte(header.valueCompression)
	return buf.Bytes()
}

/*

 file

 | magic | version | time compression | value compression | blocks | indexes | footer | magic |

 footer

 | offset of indexes |

 blocks
 | b1 | b2 | b3|

 block
 | t1, t2, ... | v1, v2, .... |

 indexes
 | num indexes | i1 | i2 ... |

 index
 | len | tags | num blocks |b1 offset | b1 size | b1 count | b2 .... |
 */
type blockWriter struct {
	header         fileHeader
	originalWriter io.WriteCloser
	w              *bufio.Writer
	n              int64
}

const intBlock byte = 1
const doubleBlock byte = 2

type indexEntry struct {
	blockType byte
	offset    int64
	size      int64
}

func NewBlockWriter(w io.WriteCloser) *blockWriter {
	return &blockWriter{
		originalWriter: w,
		w:              bufio.NewWriter(w),
		n:              0,
	}
}

func (w *blockWriter) WriteIntSeries(series *common.IntSeries) error {
	n := 0
	// write header if it does not exists
	if w.n == 0 {
		hbits, err := w.w.Write(w.header.Bytes())
		if err != nil {
			return err
		}
		n += hbits
	}
	// write timestamps and values separately
	var tBuf bytes.Buffer
	var vBuf bytes.Buffer
	b := make([]byte, 10)
	for i := 0; i < len(series.Points); i++ {
		written := binary.PutVarint(b, series.Points[i].T)
		tBuf.Write(b[:written])
		written = binary.PutVarint(b, series.Points[i].V)
		vBuf.Write(b[:written])
	}
	tbits, err := w.w.Write(tBuf.Bytes())
	if err != nil {
		return errors.Wrap(err, "fail writing time ")
	}
	n += tbits
	vbits, err := w.w.Write(vBuf.Bytes())
	if err != nil {
		return errors.Wrap(err, "fail writing value")
	}
	n += vbits
	// TODO: add the index

	w.n += int64(n)
	return nil
}

func (w *blockWriter) WriteIndex() error {
	// TODO: implementation
	return nil
}

func (w *blockWriter) Flush() error {
	if err := w.w.Flush(); err != nil {
		return err
	}

	if f, ok := w.originalWriter.(*os.File); ok {
		if err := f.Sync(); err != nil {
			return err
		}
	}
	return nil
}

func (w *blockWriter) Close() error {
	if err := w.Flush(); err != nil {
		return err
	}
	return w.originalWriter.Close()
}

func TestMagicNumber(t *testing.T) {
	var str = "xephon-k"
	t.Log(len([]byte(str))) // 8 byte, uint64
	t.Log([]byte(str))
	// [120 101 112 104 111 110 45 107]
	// 78 65 70 68 6F 6E 2D 6B
	t.Logf("% X", []byte(str))
	t.Logf("%X", []byte(str))
	t.Log([]byte(str)[0])

	// convert the magic number into binary
	t.Log(magicnumber)
	b := make([]byte, 10)
	// FIXME: it takes 9 byte instead of 8 byte to write a uint64 http://stackoverflow.com/questions/17289898/does-a-uint64-take-8-bytes-storage
	t.Log(binary.PutUvarint(b, magicnumber)) // 9 instead of 8
	t.Log(b)
	v, err := binary.ReadUvarint(bytes.NewReader(b))
	if err != nil {
		t.Fatal(err)
	}
	t.Log(v)
	// this use 8 byte
	binary.BigEndian.PutUint64(b, magicnumber)
	t.Log(b)
	t.Log(binary.BigEndian.Uint64(b))

	// Uvarint would use less than 8 byte for small value
	t.Log(binary.PutUvarint(b, 1))   // 1
	t.Log(binary.PutUvarint(b, 256)) // 2
}

func TestNoCompress_Header(t *testing.T) {
	header := fileHeader{version: 1, timeCompression: disk.CompressionNone, valueCompression: disk.CompressionNone}
	//header := fileHeader{version: 1, timeCompression: disk.CompressionGzip, valueCompression: disk.CompressionZlib}
	tmpfile, err := ioutil.TempFile("", "xephon-no-compress")
	if err != nil {
		t.Fatal(err)
	}

	defer os.Remove(tmpfile.Name())

	var buf bytes.Buffer
	// TODO: Endianness problem https://github.com/xephonhq/xephon-k/issues/34
	// but it seems for single uint8, this is not a problem
	//binary.Write(&buf, binary.LittleEndian, header.version)
	//binary.Write(&buf, binary.LittleEndian, header.timeCompression)
	//binary.Write(&buf, binary.LittleEndian, header.valueCompression)

	header.write(&buf)

	n, err := tmpfile.Write(buf.Bytes())
	t.Logf("written %d bytes\n", n)
	if err != nil {
		t.Fatal(err)
	}
	tmpfile.Close()

	// read stuff back
	f, err := os.Open(tmpfile.Name())
	readBuf := make([]byte, 11)
	n, err = f.Read(readBuf)
	t.Logf("read %d bytes\n", n)
	if err != nil {
		t.Fatal(err)
	}
	f.Close()
	// convert to header
	newHeader := fileHeader{}
	if binary.BigEndian.Uint64(readBuf[:8]) != magicnumber {
		t.Fatal("magic number does not match!")
	} else {
		t.Log("magic number match")
	}
	newHeader.version = uint8(readBuf[8])
	newHeader.timeCompression = uint8(readBuf[9])
	newHeader.valueCompression = uint8(readBuf[10])
	fmt.Printf("version %d, time compression %d, value compression %d\n",
		newHeader.version, newHeader.timeCompression, newHeader.valueCompression)
}

func TestNoCompress_Block(t *testing.T) {
	tmpfile, err := ioutil.TempFile("", "xephon-no-compress")
	if err != nil {
		t.Fatal(err)
	}

	defer os.Remove(tmpfile.Name())

	w := NewBlockWriter(tmpfile)
	s := common.NewIntSeries("s")
	s.Points = []common.IntPoint{{T: 1359788400000, V: 1}, {T: 1359788500000, V: 2}}
	w.WriteIntSeries(s)
	t.Log(w.n)
	w.Close()
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package atomic

import "unsafe"

//go:nosplit
//go:noinline
func Load(ptr *uint32) uint32 {
	return *ptr
}

//go:nosplit
//go:noinline
func Loadp(ptr unsafe.Pointer) unsafe.Pointer {
	return *(*unsafe.Pointer)(ptr)
}

//go:nosplit
//go:noinline
func Load64(ptr *uint64) uint64 {
	return *ptr
}

//go:nosplit
//go:noinline
func LoadAcq(ptr *uint32) uint32 {
	return *ptr
}

//go:noinline
//go:nosplit
func Store(ptr *uint32, val uint32) {
	*ptr = val
}

//go:noinline
//go:nosplit
func Store64(ptr *uint64, val uint64) {
	*ptr = val
}

// NO go:noescape annotation; see atomic_pointer.go.
//go:noinline
//go:nosplit
func StorepNoWB(ptr unsafe.Pointer, val unsafe.Pointer) {
	*(*uintptr)(ptr) = uintptr(val)
}

//go:noinline
//go:nosplit
func StoreRel(ptr *uint32, val uint32) {
	*ptr = val
}

//go:noescape
func And8(ptr *uint8, val uint8)

//go:noescape
func Or8(ptr *uint8, val uint8)

// NOTE: Do not add atomicxor8 (XOR is not idempotent).

//go:noescape
func Xadd(ptr *uint32, delta int32) uint32

//go:noescape
func Xadd64(ptr *uint64, delta int64) uint64

//go:noescape
func Xadduintptr(ptr *uintptr, delta uintptr) uintptr

//go:noescape
func Xchg(ptr *uint32, new uint32) uint32

//go:noescape
func Xchg64(ptr *uint64, new uint64) uint64

//go:noescape
func Xchguintptr(ptr *uintptr, new uintptr) uintptr

//go:noescape
func Cas64(ptr *uint64, old, new uint64) bool

//go:noescape
func CasRel(ptr *uint32, old, new uint32) bool
<commit_msg>runtime/internal/atomic: fix s390x's StorepNoWB implementation<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package atomic

import "unsafe"

//go:nosplit
//go:noinline
func Load(ptr *uint32) uint32 {
	return *ptr
}

//go:nosplit
//go:noinline
func Loadp(ptr unsafe.Pointer) unsafe.Pointer {
	return *(*unsafe.Pointer)(ptr)
}

//go:nosplit
//go:noinline
func Load64(ptr *uint64) uint64 {
	return *ptr
}

//go:nosplit
//go:noinline
func LoadAcq(ptr *uint32) uint32 {
	return *ptr
}

//go:noinline
//go:nosplit
func Store(ptr *uint32, val uint32) {
	*ptr = val
}

//go:noinline
//go:nosplit
func Store64(ptr *uint64, val uint64) {
	*ptr = val
}

//go:notinheap
type noWB struct{}

// NO go:noescape annotation; see atomic_pointer.go.
//go:noinline
//go:nosplit
func StorepNoWB(ptr unsafe.Pointer, val unsafe.Pointer) {
	*(**noWB)(ptr) = (*noWB)(val)
}

//go:noinline
//go:nosplit
func StoreRel(ptr *uint32, val uint32) {
	*ptr = val
}

//go:noescape
func And8(ptr *uint8, val uint8)

//go:noescape
func Or8(ptr *uint8, val uint8)

// NOTE: Do not add atomicxor8 (XOR is not idempotent).

//go:noescape
func Xadd(ptr *uint32, delta int32) uint32

//go:noescape
func Xadd64(ptr *uint64, delta int64) uint64

//go:noescape
func Xadduintptr(ptr *uintptr, delta uintptr) uintptr

//go:noescape
func Xchg(ptr *uint32, new uint32) uint32

//go:noescape
func Xchg64(ptr *uint64, new uint64) uint64

//go:noescape
func Xchguintptr(ptr *uintptr, new uintptr) uintptr

//go:noescape
func Cas64(ptr *uint64, old, new uint64) bool

//go:noescape
func CasRel(ptr *uint32, old, new uint32) bool
<|endoftext|>


<commit_before><commit_msg>Create the lock file, if we need to<commit_after><|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"text/template"
	"time"
)

type work struct {
	src, dst string
	t        *template.Template
	info     os.FileInfo
}

type build struct {
	Package  string
	Distros  []string
	Versions []version
}

type version struct {
	Version, Revision string
	Stable            bool
}

type cfg struct {
	version
	DistroName, Arch, DebArch, Package string
}

var (
	architectures = []string{"amd64", "arm", "arm64"}
	serverDistros = []string{"xenial"}
	allDistros    = []string{"xenial", "jessie", "precise", "sid", "stretch", "trusty",
		"utopic", "vivid", "wheezy", "wily", "yakkety"}

	builtins = map[string]interface{}{
		"date": func() string {
			return time.Now().Format(time.RFC1123Z)
		},
	}

	keepTmp = flag.Bool("keep_tmp", false, "keep tmp dir after build")
)

func runCommand(pwd string, command string, cmdArgs ...string) error {
	cmd := exec.Command(command, cmdArgs...)
	if len(pwd) != 0 {
		cmd.Dir = pwd
	}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}
	return nil
}

func (c cfg) run() error {
	log.Printf("!!!!!!!!! doing: %#v", c)
	var w []work

	srcdir := filepath.Join(c.DistroName, c.Package)
	dstdir, err := ioutil.TempDir(os.TempDir(), "debs")
	if err != nil {
		return err
	}
	if !*keepTmp {
		defer os.RemoveAll(dstdir)
	}

	// allow base package dir to by a symlink so we can reuse packages
	// that don't change between distros
	realSrcdir, err := filepath.EvalSymlinks(srcdir)
	if err != nil {
		return err
	}

	if err := filepath.Walk(realSrcdir, func(srcfile string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		dstfile := filepath.Join(dstdir, srcfile[len(realSrcdir):])
		if dstfile == dstdir {
			return nil
		}
		if f.IsDir() {
			log.Printf(dstfile)
			return os.Mkdir(dstfile, f.Mode())
		}
		t, err := template.
			New("").
			Funcs(builtins).
			Option("missingkey=error").
			ParseFiles(srcfile)
		if err != nil {
			return err
		}
		w = append(w, work{
			src:  srcfile,
			dst:  dstfile,
			t:    t.Templates()[0],
			info: f,
		})

		return nil
	}); err != nil {
		return err
	}

	for _, w := range w {
		log.Printf("w: %#v", w)
		if err := func() error {
			f, err := os.OpenFile(w.dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0)
			if err != nil {
				return err
			}
			defer f.Close()

			if err := w.t.Execute(f, c); err != nil {
				return err
			}
			if err := os.Chmod(w.dst, w.info.Mode()); err != nil {
				return err
			}
			return nil
		}(); err != nil {
			return err
		}
	}

	err = runCommand(dstdir, "dpkg-buildpackage", "-us", "-uc", "-b", "-a"+c.DebArch)
	if err != nil {
		return err
	}

	dstParts := []string{"bin"}
	if c.Stable {
		dstParts = append(dstParts, "stable")
	} else {
		dstParts = append(dstParts, "unstable")
	}
	dstParts = append(dstParts, c.DistroName)

	dstPath := filepath.Join(dstParts...)
	os.MkdirAll(dstPath, 0777)

	fileName := fmt.Sprintf("%s_%s-%s_%s.deb", c.Package, c.Version, c.Revision, c.DebArch)
	err = runCommand("", "mv", filepath.Join("/tmp", fileName), dstPath)
	if err != nil {
		return err
	}

	return nil
}

func walkBuilds(builds []build, f func(pkg, distro, arch string, v version) error) error {
	for _, a := range architectures {
		for _, b := range builds {
			for _, d := range b.Distros {
				for _, v := range b.Versions {
					if err := f(b.Package, d, a, v); err != nil {
						return err
					}
				}
			}
		}
	}
	return nil
}

func main() {
	flag.Parse()

	builds := []build{
		{
			Package: "kubectl",
			Distros: allDistros,
			Versions: []version{
				{
					Version:  "1.4.0",
					Revision: "00",
					Stable:   true,
				},
				{
					Version:  "1.4.0",
					Revision: "00",
					Stable:   false,
				},
			},
		},
		{
			Package: "kubelet",
			Distros: serverDistros,
			Versions: []version{
				{
					Version:  "1.4.0",
					Revision: "00",
					Stable:   true,
				},
				{
					Version:  "1.4.0",
					Revision: "00",
					Stable:   false,
				},
			},
		},
		{
			Package: "kubernetes-cni",
			Distros: serverDistros,
			Versions: []version{
				{
					Version:  "0.3.0.1-07a8a2",
					Revision: "00",
					Stable:   true,
				},
				{
					Version:  "0.3.0.1-07a8a2",
					Revision: "00",
					Stable:   true,
				},
			},
		},
		{
			Package: "kubeadm",
			Distros: serverDistros,
			Versions: []version{
				{
					Version:  "1.5.0-alpha.0-1403-gc19e08e",
					Revision: "00",
					Stable:   true,
				},
				{
					Version:  "1.5.0-alpha.0-1403-gc19e08e",
					Revision: "00",
					Stable:   false,
				},
			},
		},
	}

	if err := walkBuilds(builds, func(pkg, distro, arch string, v version) error {
		c := cfg{
			Package:    pkg,
			version:    v,
			DistroName: distro,
			Arch:       arch,
		}
		if c.Arch == "arm" {
			c.DebArch = "armhf"
		} else {
			c.DebArch = c.Arch
		}
		return c.run()
	}); err != nil {
		log.Fatalf("err: %v", err)
	}
}
<commit_msg>Bump kubeadm for the version that has arm and arm64 builds<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"text/template"
	"time"
)

type work struct {
	src, dst string
	t        *template.Template
	info     os.FileInfo
}

type build struct {
	Package  string
	Distros  []string
	Versions []version
}

type version struct {
	Version, Revision string
	Stable            bool
}

type cfg struct {
	version
	DistroName, Arch, DebArch, Package string
}

var (
	architectures = []string{"amd64", "arm", "arm64"}
	serverDistros = []string{"xenial"}
	allDistros    = []string{"xenial", "jessie", "precise", "sid", "stretch", "trusty",
		"utopic", "vivid", "wheezy", "wily", "yakkety"}

	builtins = map[string]interface{}{
		"date": func() string {
			return time.Now().Format(time.RFC1123Z)
		},
	}

	keepTmp = flag.Bool("keep_tmp", false, "keep tmp dir after build")
)

func runCommand(pwd string, command string, cmdArgs ...string) error {
	cmd := exec.Command(command, cmdArgs...)
	if len(pwd) != 0 {
		cmd.Dir = pwd
	}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}
	return nil
}

func (c cfg) run() error {
	log.Printf("!!!!!!!!! doing: %#v", c)
	var w []work

	srcdir := filepath.Join(c.DistroName, c.Package)
	dstdir, err := ioutil.TempDir(os.TempDir(), "debs")
	if err != nil {
		return err
	}
	if !*keepTmp {
		defer os.RemoveAll(dstdir)
	}

	// allow base package dir to by a symlink so we can reuse packages
	// that don't change between distros
	realSrcdir, err := filepath.EvalSymlinks(srcdir)
	if err != nil {
		return err
	}

	if err := filepath.Walk(realSrcdir, func(srcfile string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		dstfile := filepath.Join(dstdir, srcfile[len(realSrcdir):])
		if dstfile == dstdir {
			return nil
		}
		if f.IsDir() {
			log.Printf(dstfile)
			return os.Mkdir(dstfile, f.Mode())
		}
		t, err := template.
			New("").
			Funcs(builtins).
			Option("missingkey=error").
			ParseFiles(srcfile)
		if err != nil {
			return err
		}
		w = append(w, work{
			src:  srcfile,
			dst:  dstfile,
			t:    t.Templates()[0],
			info: f,
		})

		return nil
	}); err != nil {
		return err
	}

	for _, w := range w {
		log.Printf("w: %#v", w)
		if err := func() error {
			f, err := os.OpenFile(w.dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0)
			if err != nil {
				return err
			}
			defer f.Close()

			if err := w.t.Execute(f, c); err != nil {
				return err
			}
			if err := os.Chmod(w.dst, w.info.Mode()); err != nil {
				return err
			}
			return nil
		}(); err != nil {
			return err
		}
	}

	err = runCommand(dstdir, "dpkg-buildpackage", "-us", "-uc", "-b", "-a"+c.DebArch)
	if err != nil {
		return err
	}

	dstParts := []string{"bin"}
	if c.Stable {
		dstParts = append(dstParts, "stable")
	} else {
		dstParts = append(dstParts, "unstable")
	}
	dstParts = append(dstParts, c.DistroName)

	dstPath := filepath.Join(dstParts...)
	os.MkdirAll(dstPath, 0777)

	fileName := fmt.Sprintf("%s_%s-%s_%s.deb", c.Package, c.Version, c.Revision, c.DebArch)
	err = runCommand("", "mv", filepath.Join("/tmp", fileName), dstPath)
	if err != nil {
		return err
	}

	return nil
}

func walkBuilds(builds []build, f func(pkg, distro, arch string, v version) error) error {
	for _, a := range architectures {
		for _, b := range builds {
			for _, d := range b.Distros {
				for _, v := range b.Versions {
					if err := f(b.Package, d, a, v); err != nil {
						return err
					}
				}
			}
		}
	}
	return nil
}

func main() {
	flag.Parse()

	builds := []build{
		{
			Package: "kubectl",
			Distros: allDistros,
			Versions: []version{
				{
					Version:  "1.4.0",
					Revision: "00",
					Stable:   true,
				},
				{
					Version:  "1.4.0",
					Revision: "00",
					Stable:   false,
				},
			},
		},
		{
			Package: "kubelet",
			Distros: serverDistros,
			Versions: []version{
				{
					Version:  "1.4.0",
					Revision: "00",
					Stable:   true,
				},
				{
					Version:  "1.4.0",
					Revision: "00",
					Stable:   false,
				},
			},
		},
		{
			Package: "kubernetes-cni",
			Distros: serverDistros,
			Versions: []version{
				{
					Version:  "0.3.0.1-07a8a2",
					Revision: "00",
					Stable:   true,
				},
				{
					Version:  "0.3.0.1-07a8a2",
					Revision: "00",
					Stable:   true,
				},
			},
		},
		{
			Package: "kubeadm",
			Distros: serverDistros,
			Versions: []version{
				{
					Version:  "1.5.0-alpha.0-1495-g1e7fa1f",
					Revision: "00",
					Stable:   true,
				},
				{
					Version:  "1.5.0-alpha.0-1495-g1e7fa1f",
					Revision: "00",
					Stable:   false,
				},
			},
		},
	}

	if err := walkBuilds(builds, func(pkg, distro, arch string, v version) error {
		c := cfg{
			Package:    pkg,
			version:    v,
			DistroName: distro,
			Arch:       arch,
		}
		if c.Arch == "arm" {
			c.DebArch = "armhf"
		} else {
			c.DebArch = c.Arch
		}
		return c.run()
	}); err != nil {
		log.Fatalf("err: %v", err)
	}
}
<|endoftext|>


<commit_before>package apd

import (
	"fmt"
	"math/big"
	"testing"
)

func (d *Decimal) GoString() string {
	return fmt.Sprintf(`{Coeff: %s, Exponent: %d, MaxExponent: %d, MinExponent: %d, Precision: %d}`, d.Coeff.String(), d.Exponent, d.MaxExponent, d.MinExponent, d.Precision)
}

func TestNewFromString(t *testing.T) {
	tests := []struct {
		s   string
		out string
	}{
		{s: "0"},
		{s: "0.0"},
		{s: "00.0", out: "0.0"},
		{s: "0.00"},
		{s: "00.00", out: "0.00"},
		{s: "1"},
		{s: "1.0"},
		{s: "0.1"},
		{s: ".1", out: "0.1"},
		{s: "01.10", out: "1.10"},
		{s: "123456.789"},
		{s: "-123"},
		{s: "1e1", out: "10"},
		{s: "1e-1", out: "0.1"},
		{s: "0.1e1", out: "1"},
		{s: "0.10e1", out: "1.0"},
		{s: "0.1e-1", out: "0.01"},
		{s: "1e10", out: "10000000000"},
		{s: "1e-10", out: "0.0000000001"},
		{s: "0.1e10", out: "1000000000"},
		{s: "0.1e-10", out: "0.00000000001"},
	}
	for _, tc := range tests {
		t.Run(tc.s, func(t *testing.T) {
			d := newDecimal(t, tc.s)
			expect := tc.out
			if expect == "" {
				expect = tc.s
			}
			s := d.String()
			if s != expect {
				t.Errorf("expected: %s, got %s", expect, s)
			}
		})
	}
}

func newDecimal(t *testing.T, s string) *Decimal {
	d, err := NewFromString(s)
	if err != nil {
		t.Fatal(err)
	}
	return d
}

func TestUpscale(t *testing.T) {
	tests := []struct {
		x, y *Decimal
		a, b *big.Int
		s    int32
	}{
		{x: New(1, 0), y: New(100, -1), a: big.NewInt(10), b: big.NewInt(100), s: -1},
		{x: New(1, 0), y: New(10, -1), a: big.NewInt(10), b: big.NewInt(10), s: -1},
		{x: New(1, 0), y: New(10, 0), a: big.NewInt(1), b: big.NewInt(10), s: 0},
		{x: New(1, 1), y: New(1, 0), a: big.NewInt(10), b: big.NewInt(1), s: 0},
		{x: New(10, -2), y: New(1, -1), a: big.NewInt(10), b: big.NewInt(10), s: -2},
		{x: New(1, -2), y: New(100, 1), a: big.NewInt(1), b: big.NewInt(100000), s: -2},
	}
	for _, tc := range tests {
		t.Run(fmt.Sprintf("%s, %s", tc.x, tc.y), func(t *testing.T) {
			a, b, s, err := upscale(tc.x, tc.y)
			if err != nil {
				t.Fatal(err)
			}
			if a.Cmp(tc.a) != 0 {
				t.Errorf("a: expected %s, got %s", tc.a, a)
			}
			if b.Cmp(tc.b) != 0 {
				t.Errorf("b: expected %s, got %s", tc.b, b)
			}
			if s != tc.s {
				t.Errorf("s: expected %d, got %d", tc.s, s)
			}
		})
	}
}

func TestAdd(t *testing.T) {
	tests := []struct {
		x, y string
		r    string
	}{
		{x: "1", y: "10", r: "11"},
		{x: "1", y: "1e1", r: "11"},
		{x: "1e1", y: "1", r: "11"},
		{x: ".1e1", y: "100e-1", r: "11.0"},
	}
	for _, tc := range tests {
		t.Run(fmt.Sprintf("%s, %s", tc.x, tc.y), func(t *testing.T) {
			x := newDecimal(t, tc.x)
			y := newDecimal(t, tc.y)
			d := new(Decimal)
			err := d.Add(x, y)
			if err != nil {
				t.Fatal(err)
			}
			s := d.String()
			if s != tc.r {
				t.Fatalf("expected: %s, got: %s", tc.r, s)
			}
		})
	}
}

func TestCmp(t *testing.T) {
	tests := []struct {
		x, y string
		c    int
	}{
		{x: "1", y: "10", c: -1},
		{x: "1", y: "1e1", c: -1},
		{x: "1e1", y: "1", c: 1},
		{x: ".1e1", y: "100e-1", c: -1},

		{x: ".1e1", y: "100e-2", c: 0},
		{x: "1", y: ".1e1", c: 0},
		{x: "1", y: "1", c: 0},
	}
	for _, tc := range tests {
		t.Run(fmt.Sprintf("%s, %s", tc.x, tc.y), func(t *testing.T) {
			x := newDecimal(t, tc.x)
			y := newDecimal(t, tc.y)
			c, err := x.Cmp(y)
			if err != nil {
				t.Fatal(err)
			}
			if c != tc.c {
				t.Fatalf("expected: %d, got: %d", tc.c, c)
			}
		})
	}
}

func TestModf(t *testing.T) {
	tests := []struct {
		x string
		i string
		f string
	}{
		{x: "1", i: "1", f: "0"},
		{x: "1.0", i: "1", f: "0.0"},
		{x: "1.0e1", i: "10", f: "0"},
		{x: "1.0e2", i: "100", f: "0"},
		{x: "1.0e-1", i: "0", f: "0.10"},
		{x: "1.0e-2", i: "0", f: "0.010"},
		{x: "1234.56", i: "1234", f: "0.56"},
		{x: "1234.56e2", i: "123456", f: "0"},
		{x: "1234.56e4", i: "12345600", f: "0"},
		{x: "1234.56e-2", i: "12", f: "0.3456"},
		{x: "1234.56e-4", i: "0", f: "0.123456"},
		{x: "1234.56e-6", i: "0", f: "0.00123456"},
		{x: "123456e-8", i: "0", f: "0.00123456"},
		{x: ".123456e8", i: "12345600", f: "0"},

		{x: "-1", i: "-1", f: "0"},
		{x: "-1.0", i: "-1", f: "0.0"},
		{x: "-1.0e1", i: "-10", f: "0"},
		{x: "-1.0e2", i: "-100", f: "0"},
		{x: "-1.0e-1", i: "0", f: "-0.10"},
		{x: "-1.0e-2", i: "0", f: "-0.010"},
		{x: "-1234.56", i: "-1234", f: "-0.56"},
		{x: "-1234.56e2", i: "-123456", f: "0"},
		{x: "-1234.56e4", i: "-12345600", f: "0"},
		{x: "-1234.56e-2", i: "-12", f: "-0.3456"},
		{x: "-1234.56e-4", i: "0", f: "-0.123456"},
		{x: "-1234.56e-6", i: "0", f: "-0.00123456"},
		{x: "-123456e-8", i: "0", f: "-0.00123456"},
		{x: "-.123456e8", i: "-12345600", f: "0"},
	}
	for _, tc := range tests {
		t.Run(tc.x, func(t *testing.T) {
			x := newDecimal(t, tc.x)
			integ, frac := new(Decimal), new(Decimal)
			x.Modf(integ, frac)
			if tc.i != integ.String() {
				t.Fatalf("integ: expected: %s, got: %s", tc.i, integ)
			}
			if tc.f != frac.String() {
				t.Fatalf("frac: expected: %s, got: %s", tc.f, frac)
			}
			a := new(Decimal)
			if err := a.Add(integ, frac); err != nil {
				t.Fatal(err)
			}
			if c, err := a.Cmp(x); err != nil {
				t.Fatal(err)
			} else if c != 0 {
				t.Fatalf("%s != %s", a, x)
			}
			if integ.Exponent < 0 {
				t.Fatal(integ.Exponent)
			}
			if frac.Exponent > 0 {
				t.Fatal(frac.Exponent)
			}
		})
	}
}
<commit_msg>add todo<commit_after>package apd

import (
	"fmt"
	"math/big"
	"testing"
)

// TODO(mjibson): audit functions to guarantee they still function correctly
// if the receiver and args are the same variable.

func (d *Decimal) GoString() string {
	return fmt.Sprintf(`{Coeff: %s, Exponent: %d, MaxExponent: %d, MinExponent: %d, Precision: %d}`, d.Coeff.String(), d.Exponent, d.MaxExponent, d.MinExponent, d.Precision)
}

func TestNewFromString(t *testing.T) {
	tests := []struct {
		s   string
		out string
	}{
		{s: "0"},
		{s: "0.0"},
		{s: "00.0", out: "0.0"},
		{s: "0.00"},
		{s: "00.00", out: "0.00"},
		{s: "1"},
		{s: "1.0"},
		{s: "0.1"},
		{s: ".1", out: "0.1"},
		{s: "01.10", out: "1.10"},
		{s: "123456.789"},
		{s: "-123"},
		{s: "1e1", out: "10"},
		{s: "1e-1", out: "0.1"},
		{s: "0.1e1", out: "1"},
		{s: "0.10e1", out: "1.0"},
		{s: "0.1e-1", out: "0.01"},
		{s: "1e10", out: "10000000000"},
		{s: "1e-10", out: "0.0000000001"},
		{s: "0.1e10", out: "1000000000"},
		{s: "0.1e-10", out: "0.00000000001"},
	}
	for _, tc := range tests {
		t.Run(tc.s, func(t *testing.T) {
			d := newDecimal(t, tc.s)
			expect := tc.out
			if expect == "" {
				expect = tc.s
			}
			s := d.String()
			if s != expect {
				t.Errorf("expected: %s, got %s", expect, s)
			}
		})
	}
}

func newDecimal(t *testing.T, s string) *Decimal {
	d, err := NewFromString(s)
	if err != nil {
		t.Fatal(err)
	}
	return d
}

func TestUpscale(t *testing.T) {
	tests := []struct {
		x, y *Decimal
		a, b *big.Int
		s    int32
	}{
		{x: New(1, 0), y: New(100, -1), a: big.NewInt(10), b: big.NewInt(100), s: -1},
		{x: New(1, 0), y: New(10, -1), a: big.NewInt(10), b: big.NewInt(10), s: -1},
		{x: New(1, 0), y: New(10, 0), a: big.NewInt(1), b: big.NewInt(10), s: 0},
		{x: New(1, 1), y: New(1, 0), a: big.NewInt(10), b: big.NewInt(1), s: 0},
		{x: New(10, -2), y: New(1, -1), a: big.NewInt(10), b: big.NewInt(10), s: -2},
		{x: New(1, -2), y: New(100, 1), a: big.NewInt(1), b: big.NewInt(100000), s: -2},
	}
	for _, tc := range tests {
		t.Run(fmt.Sprintf("%s, %s", tc.x, tc.y), func(t *testing.T) {
			a, b, s, err := upscale(tc.x, tc.y)
			if err != nil {
				t.Fatal(err)
			}
			if a.Cmp(tc.a) != 0 {
				t.Errorf("a: expected %s, got %s", tc.a, a)
			}
			if b.Cmp(tc.b) != 0 {
				t.Errorf("b: expected %s, got %s", tc.b, b)
			}
			if s != tc.s {
				t.Errorf("s: expected %d, got %d", tc.s, s)
			}
		})
	}
}

func TestAdd(t *testing.T) {
	tests := []struct {
		x, y string
		r    string
	}{
		{x: "1", y: "10", r: "11"},
		{x: "1", y: "1e1", r: "11"},
		{x: "1e1", y: "1", r: "11"},
		{x: ".1e1", y: "100e-1", r: "11.0"},
	}
	for _, tc := range tests {
		t.Run(fmt.Sprintf("%s, %s", tc.x, tc.y), func(t *testing.T) {
			x := newDecimal(t, tc.x)
			y := newDecimal(t, tc.y)
			d := new(Decimal)
			err := d.Add(x, y)
			if err != nil {
				t.Fatal(err)
			}
			s := d.String()
			if s != tc.r {
				t.Fatalf("expected: %s, got: %s", tc.r, s)
			}
		})
	}
}

func TestCmp(t *testing.T) {
	tests := []struct {
		x, y string
		c    int
	}{
		{x: "1", y: "10", c: -1},
		{x: "1", y: "1e1", c: -1},
		{x: "1e1", y: "1", c: 1},
		{x: ".1e1", y: "100e-1", c: -1},

		{x: ".1e1", y: "100e-2", c: 0},
		{x: "1", y: ".1e1", c: 0},
		{x: "1", y: "1", c: 0},
	}
	for _, tc := range tests {
		t.Run(fmt.Sprintf("%s, %s", tc.x, tc.y), func(t *testing.T) {
			x := newDecimal(t, tc.x)
			y := newDecimal(t, tc.y)
			c, err := x.Cmp(y)
			if err != nil {
				t.Fatal(err)
			}
			if c != tc.c {
				t.Fatalf("expected: %d, got: %d", tc.c, c)
			}
		})
	}
}

func TestModf(t *testing.T) {
	tests := []struct {
		x string
		i string
		f string
	}{
		{x: "1", i: "1", f: "0"},
		{x: "1.0", i: "1", f: "0.0"},
		{x: "1.0e1", i: "10", f: "0"},
		{x: "1.0e2", i: "100", f: "0"},
		{x: "1.0e-1", i: "0", f: "0.10"},
		{x: "1.0e-2", i: "0", f: "0.010"},
		{x: "1234.56", i: "1234", f: "0.56"},
		{x: "1234.56e2", i: "123456", f: "0"},
		{x: "1234.56e4", i: "12345600", f: "0"},
		{x: "1234.56e-2", i: "12", f: "0.3456"},
		{x: "1234.56e-4", i: "0", f: "0.123456"},
		{x: "1234.56e-6", i: "0", f: "0.00123456"},
		{x: "123456e-8", i: "0", f: "0.00123456"},
		{x: ".123456e8", i: "12345600", f: "0"},

		{x: "-1", i: "-1", f: "0"},
		{x: "-1.0", i: "-1", f: "0.0"},
		{x: "-1.0e1", i: "-10", f: "0"},
		{x: "-1.0e2", i: "-100", f: "0"},
		{x: "-1.0e-1", i: "0", f: "-0.10"},
		{x: "-1.0e-2", i: "0", f: "-0.010"},
		{x: "-1234.56", i: "-1234", f: "-0.56"},
		{x: "-1234.56e2", i: "-123456", f: "0"},
		{x: "-1234.56e4", i: "-12345600", f: "0"},
		{x: "-1234.56e-2", i: "-12", f: "-0.3456"},
		{x: "-1234.56e-4", i: "0", f: "-0.123456"},
		{x: "-1234.56e-6", i: "0", f: "-0.00123456"},
		{x: "-123456e-8", i: "0", f: "-0.00123456"},
		{x: "-.123456e8", i: "-12345600", f: "0"},
	}
	for _, tc := range tests {
		t.Run(tc.x, func(t *testing.T) {
			x := newDecimal(t, tc.x)
			integ, frac := new(Decimal), new(Decimal)
			x.Modf(integ, frac)
			if tc.i != integ.String() {
				t.Fatalf("integ: expected: %s, got: %s", tc.i, integ)
			}
			if tc.f != frac.String() {
				t.Fatalf("frac: expected: %s, got: %s", tc.f, frac)
			}
			a := new(Decimal)
			if err := a.Add(integ, frac); err != nil {
				t.Fatal(err)
			}
			if c, err := a.Cmp(x); err != nil {
				t.Fatal(err)
			} else if c != 0 {
				t.Fatalf("%s != %s", a, x)
			}
			if integ.Exponent < 0 {
				t.Fatal(integ.Exponent)
			}
			if frac.Exponent > 0 {
				t.Fatal(frac.Exponent)
			}
		})
	}
}
<|endoftext|>


<commit_before>package define

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"github.com/containers/image/v5/manifest"
	"github.com/containers/storage/pkg/archive"
	"github.com/containers/storage/pkg/chrootarchive"
	"github.com/containers/storage/pkg/ioutils"
	v1 "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/opencontainers/runtime-spec/specs-go"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	// Package is the name of this package, used in help output and to
	// identify working containers.
	Package = "buildah"
	// Version for the Package.  Bump version in contrib/rpm/buildah.spec
	// too.
	Version = "1.20.1-dev"

	// DefaultRuntime if containers.conf fails.
	DefaultRuntime = "runc"

	DefaultCNIPluginPath = "/usr/libexec/cni:/opt/cni/bin"
	// DefaultCNIConfigDir is the default location of CNI configuration files.
	DefaultCNIConfigDir = "/etc/cni/net.d"

	// OCIv1ImageManifest is the MIME type of an OCIv1 image manifest,
	// suitable for specifying as a value of the PreferredManifestType
	// member of a CommitOptions structure.  It is also the default.
	OCIv1ImageManifest = v1.MediaTypeImageManifest
	// Dockerv2ImageManifest is the MIME type of a Docker v2s2 image
	// manifest, suitable for specifying as a value of the
	// PreferredManifestType member of a CommitOptions structure.
	Dockerv2ImageManifest = manifest.DockerV2Schema2MediaType

	// OCI used to define the "oci" image format
	OCI = "oci"
	// DOCKER used to define the "docker" image format
	DOCKER = "docker"
)

var (
	// DefaultCapabilities is the list of capabilities which we grant by
	// default to containers which are running under UID 0.
	DefaultCapabilities = []string{
		"CAP_AUDIT_WRITE",
		"CAP_CHOWN",
		"CAP_DAC_OVERRIDE",
		"CAP_FOWNER",
		"CAP_FSETID",
		"CAP_KILL",
		"CAP_MKNOD",
		"CAP_NET_BIND_SERVICE",
		"CAP_SETFCAP",
		"CAP_SETGID",
		"CAP_SETPCAP",
		"CAP_SETUID",
		"CAP_SYS_CHROOT",
	}
	// DefaultNetworkSysctl is the list of Kernel parameters which we
	// grant by default to containers which are running under UID 0.
	DefaultNetworkSysctl = map[string]string{
		"net.ipv4.ping_group_range": "0 0",
	}

	Gzip         = archive.Gzip
	Bzip2        = archive.Bzip2
	Xz           = archive.Xz
	Zstd         = archive.Zstd
	Uncompressed = archive.Uncompressed
)

// IDMappingOptions controls how we set up UID/GID mapping when we set up a
// user namespace.
type IDMappingOptions struct {
	HostUIDMapping bool
	HostGIDMapping bool
	UIDMap         []specs.LinuxIDMapping
	GIDMap         []specs.LinuxIDMapping
}

// TempDirForURL checks if the passed-in string looks like a URL or -.  If it is,
// TempDirForURL creates a temporary directory, arranges for its contents to be
// the contents of that URL, and returns the temporary directory's path, along
// with the name of a subdirectory which should be used as the build context
// (which may be empty or ".").  Removal of the temporary directory is the
// responsibility of the caller.  If the string doesn't look like a URL,
// TempDirForURL returns empty strings and a nil error code.
func TempDirForURL(dir, prefix, url string) (name string, subdir string, err error) {
	if !strings.HasPrefix(url, "http://") &&
		!strings.HasPrefix(url, "https://") &&
		!strings.HasPrefix(url, "git://") &&
		!strings.HasPrefix(url, "github.com/") &&
		url != "-" {
		return "", "", nil
	}
	name, err = ioutil.TempDir(dir, prefix)
	if err != nil {
		return "", "", errors.Wrapf(err, "error creating temporary directory for %q", url)
	}
	if strings.HasPrefix(url, "git://") || strings.HasSuffix(url, ".git") {
		err = cloneToDirectory(url, name)
		if err != nil {
			if err2 := os.RemoveAll(name); err2 != nil {
				logrus.Debugf("error removing temporary directory %q: %v", name, err2)
			}
			return "", "", err
		}
		return name, "", nil
	}
	if strings.HasPrefix(url, "github.com/") {
		ghurl := url
		url = fmt.Sprintf("https://%s/archive/master.tar.gz", ghurl)
		logrus.Debugf("resolving url %q to %q", ghurl, url)
		subdir = path.Base(ghurl) + "-master"
	}
	if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
		err = downloadToDirectory(url, name)
		if err != nil {
			if err2 := os.RemoveAll(name); err2 != nil {
				logrus.Debugf("error removing temporary directory %q: %v", name, err2)
			}
			return "", subdir, err
		}
		return name, subdir, nil
	}
	if url == "-" {
		err = stdinToDirectory(name)
		if err != nil {
			if err2 := os.RemoveAll(name); err2 != nil {
				logrus.Debugf("error removing temporary directory %q: %v", name, err2)
			}
			return "", subdir, err
		}
		logrus.Debugf("Build context is at %q", name)
		return name, subdir, nil
	}
	logrus.Debugf("don't know how to retrieve %q", url)
	if err2 := os.Remove(name); err2 != nil {
		logrus.Debugf("error removing temporary directory %q: %v", name, err2)
	}
	return "", "", errors.Errorf("unreachable code reached")
}

func cloneToDirectory(url, dir string) error {
	if !strings.HasPrefix(url, "git://") && !strings.HasSuffix(url, ".git") {
		url = "git://" + url
	}
	gitBranch := strings.Split(url, "#")
	var cmd *exec.Cmd
	if len(gitBranch) < 2 {
		logrus.Debugf("cloning %q to %q", url, dir)
		cmd = exec.Command("git", "clone", url, dir)
	} else {
		logrus.Debugf("cloning repo %q and branch %q to %q", gitBranch[0], gitBranch[1], dir)
		cmd = exec.Command("git", "clone", "-b", gitBranch[1], gitBranch[0], dir)
	}
	return cmd.Run()
}

func downloadToDirectory(url, dir string) error {
	logrus.Debugf("extracting %q to %q", url, dir)
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.ContentLength == 0 {
		return errors.Errorf("no contents in %q", url)
	}
	if err := chrootarchive.Untar(resp.Body, dir, nil); err != nil {
		resp1, err := http.Get(url)
		if err != nil {
			return err
		}
		defer resp1.Body.Close()
		body, err := ioutil.ReadAll(resp1.Body)
		if err != nil {
			return err
		}
		dockerfile := filepath.Join(dir, "Dockerfile")
		// Assume this is a Dockerfile
		if err := ioutils.AtomicWriteFile(dockerfile, body, 0600); err != nil {
			return errors.Wrapf(err, "Failed to write %q to %q", url, dockerfile)
		}
	}
	return nil
}

func stdinToDirectory(dir string) error {
	logrus.Debugf("extracting stdin to %q", dir)
	r := bufio.NewReader(os.Stdin)
	b, err := ioutil.ReadAll(r)
	if err != nil {
		return errors.Wrapf(err, "Failed to read from stdin")
	}
	reader := bytes.NewReader(b)
	if err := chrootarchive.Untar(reader, dir, nil); err != nil {
		dockerfile := filepath.Join(dir, "Dockerfile")
		// Assume this is a Dockerfile
		if err := ioutils.AtomicWriteFile(dockerfile, b, 0600); err != nil {
			return errors.Wrapf(err, "Failed to write bytes to %q", dockerfile)
		}
	}
	return nil
}
<commit_msg>[NO TESTS NEEDED] Use --recurse-modules when building git context<commit_after>package define

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"github.com/containers/image/v5/manifest"
	"github.com/containers/storage/pkg/archive"
	"github.com/containers/storage/pkg/chrootarchive"
	"github.com/containers/storage/pkg/ioutils"
	v1 "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/opencontainers/runtime-spec/specs-go"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	// Package is the name of this package, used in help output and to
	// identify working containers.
	Package = "buildah"
	// Version for the Package.  Bump version in contrib/rpm/buildah.spec
	// too.
	Version = "1.20.1-dev"

	// DefaultRuntime if containers.conf fails.
	DefaultRuntime = "runc"

	DefaultCNIPluginPath = "/usr/libexec/cni:/opt/cni/bin"
	// DefaultCNIConfigDir is the default location of CNI configuration files.
	DefaultCNIConfigDir = "/etc/cni/net.d"

	// OCIv1ImageManifest is the MIME type of an OCIv1 image manifest,
	// suitable for specifying as a value of the PreferredManifestType
	// member of a CommitOptions structure.  It is also the default.
	OCIv1ImageManifest = v1.MediaTypeImageManifest
	// Dockerv2ImageManifest is the MIME type of a Docker v2s2 image
	// manifest, suitable for specifying as a value of the
	// PreferredManifestType member of a CommitOptions structure.
	Dockerv2ImageManifest = manifest.DockerV2Schema2MediaType

	// OCI used to define the "oci" image format
	OCI = "oci"
	// DOCKER used to define the "docker" image format
	DOCKER = "docker"
)

var (
	// DefaultCapabilities is the list of capabilities which we grant by
	// default to containers which are running under UID 0.
	DefaultCapabilities = []string{
		"CAP_AUDIT_WRITE",
		"CAP_CHOWN",
		"CAP_DAC_OVERRIDE",
		"CAP_FOWNER",
		"CAP_FSETID",
		"CAP_KILL",
		"CAP_MKNOD",
		"CAP_NET_BIND_SERVICE",
		"CAP_SETFCAP",
		"CAP_SETGID",
		"CAP_SETPCAP",
		"CAP_SETUID",
		"CAP_SYS_CHROOT",
	}
	// DefaultNetworkSysctl is the list of Kernel parameters which we
	// grant by default to containers which are running under UID 0.
	DefaultNetworkSysctl = map[string]string{
		"net.ipv4.ping_group_range": "0 0",
	}

	Gzip         = archive.Gzip
	Bzip2        = archive.Bzip2
	Xz           = archive.Xz
	Zstd         = archive.Zstd
	Uncompressed = archive.Uncompressed
)

// IDMappingOptions controls how we set up UID/GID mapping when we set up a
// user namespace.
type IDMappingOptions struct {
	HostUIDMapping bool
	HostGIDMapping bool
	UIDMap         []specs.LinuxIDMapping
	GIDMap         []specs.LinuxIDMapping
}

// TempDirForURL checks if the passed-in string looks like a URL or -.  If it is,
// TempDirForURL creates a temporary directory, arranges for its contents to be
// the contents of that URL, and returns the temporary directory's path, along
// with the name of a subdirectory which should be used as the build context
// (which may be empty or ".").  Removal of the temporary directory is the
// responsibility of the caller.  If the string doesn't look like a URL,
// TempDirForURL returns empty strings and a nil error code.
func TempDirForURL(dir, prefix, url string) (name string, subdir string, err error) {
	if !strings.HasPrefix(url, "http://") &&
		!strings.HasPrefix(url, "https://") &&
		!strings.HasPrefix(url, "git://") &&
		!strings.HasPrefix(url, "github.com/") &&
		url != "-" {
		return "", "", nil
	}
	name, err = ioutil.TempDir(dir, prefix)
	if err != nil {
		return "", "", errors.Wrapf(err, "error creating temporary directory for %q", url)
	}
	if strings.HasPrefix(url, "git://") || strings.HasSuffix(url, ".git") {
		err = cloneToDirectory(url, name)
		if err != nil {
			if err2 := os.RemoveAll(name); err2 != nil {
				logrus.Debugf("error removing temporary directory %q: %v", name, err2)
			}
			return "", "", err
		}
		return name, "", nil
	}
	if strings.HasPrefix(url, "github.com/") {
		ghurl := url
		url = fmt.Sprintf("https://%s/archive/master.tar.gz", ghurl)
		logrus.Debugf("resolving url %q to %q", ghurl, url)
		subdir = path.Base(ghurl) + "-master"
	}
	if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
		err = downloadToDirectory(url, name)
		if err != nil {
			if err2 := os.RemoveAll(name); err2 != nil {
				logrus.Debugf("error removing temporary directory %q: %v", name, err2)
			}
			return "", subdir, err
		}
		return name, subdir, nil
	}
	if url == "-" {
		err = stdinToDirectory(name)
		if err != nil {
			if err2 := os.RemoveAll(name); err2 != nil {
				logrus.Debugf("error removing temporary directory %q: %v", name, err2)
			}
			return "", subdir, err
		}
		logrus.Debugf("Build context is at %q", name)
		return name, subdir, nil
	}
	logrus.Debugf("don't know how to retrieve %q", url)
	if err2 := os.Remove(name); err2 != nil {
		logrus.Debugf("error removing temporary directory %q: %v", name, err2)
	}
	return "", "", errors.Errorf("unreachable code reached")
}

func cloneToDirectory(url, dir string) error {
	if !strings.HasPrefix(url, "git://") && !strings.HasSuffix(url, ".git") {
		url = "git://" + url
	}
	gitBranch := strings.Split(url, "#")
	var cmd *exec.Cmd
	if len(gitBranch) < 2 {
		logrus.Debugf("cloning %q to %q", url, dir)
		cmd = exec.Command("git", "clone", url, dir)
	} else {
		logrus.Debugf("cloning repo %q and branch %q to %q", gitBranch[0], gitBranch[1], dir)
		cmd = exec.Command("git", "clone", "--recurse-submodules", "-b", gitBranch[1], gitBranch[0], dir)
	}
	return cmd.Run()
}

func downloadToDirectory(url, dir string) error {
	logrus.Debugf("extracting %q to %q", url, dir)
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.ContentLength == 0 {
		return errors.Errorf("no contents in %q", url)
	}
	if err := chrootarchive.Untar(resp.Body, dir, nil); err != nil {
		resp1, err := http.Get(url)
		if err != nil {
			return err
		}
		defer resp1.Body.Close()
		body, err := ioutil.ReadAll(resp1.Body)
		if err != nil {
			return err
		}
		dockerfile := filepath.Join(dir, "Dockerfile")
		// Assume this is a Dockerfile
		if err := ioutils.AtomicWriteFile(dockerfile, body, 0600); err != nil {
			return errors.Wrapf(err, "Failed to write %q to %q", url, dockerfile)
		}
	}
	return nil
}

func stdinToDirectory(dir string) error {
	logrus.Debugf("extracting stdin to %q", dir)
	r := bufio.NewReader(os.Stdin)
	b, err := ioutil.ReadAll(r)
	if err != nil {
		return errors.Wrapf(err, "Failed to read from stdin")
	}
	reader := bytes.NewReader(b)
	if err := chrootarchive.Untar(reader, dir, nil); err != nil {
		dockerfile := filepath.Join(dir, "Dockerfile")
		// Assume this is a Dockerfile
		if err := ioutils.AtomicWriteFile(dockerfile, b, 0600); err != nil {
			return errors.Wrapf(err, "Failed to write bytes to %q", dockerfile)
		}
	}
	return nil
}
<|endoftext|>