<commit_before>// Copyright 2014, Hǎiliàng Wáng. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"github.com/hailiang/color"
	"go-mupdf/fz"
	//	"sync"
	//	"time"
	"unsafe"
)

type RenderOption struct {
	Width      int
	Dark       bool
	Brightness float32
}

type Model struct {
	ctx *fz.Context
	doc *fz.Document

	// render cache
	RenderOption
	height  int
	heights []int
	cache   []Pixel
	cached  []bool
	stop    bool
	//	mu      sync.Mutex // cache lock
	//	wg      sync.WaitGroup
}

func newModel() (*Model, error) {
	ctx := fz.AllocDefault.NewContext(nil, fz.StoreDefault)
	return &Model{
		ctx: ctx,
		RenderOption: RenderOption{
			Dark:       true,
			Brightness: 0.5,
		},
	}, nil
}

func (m *Model) Open(file string) error {
	if m.doc != nil {
		m.doc.Close()
		m.doc = nil
	}
	m.doc = m.ctx.OpenDocument(file)

	return nil
}

func (m *Model) PageCount() int {
	return m.doc.CountPages()
}

func (m *Model) Height() int {
	return len(m.cache) / m.RenderOption.Width
}

func (m *Model) setWidth(width int) {
	if width != m.Width {
		m.Width = width

		m.heights = make([]int, m.PageCount())
		m.height = 0
		for i := 0; i < m.PageCount(); i++ {
			page := m.doc.LoadPage(i)
			var bounds fz.Rect
			m.doc.BoundPage(page, &bounds)
			zoom := float32(width) / bounds.X1
			var transform fz.Matrix
			transform.Scale(zoom, zoom)
			bounds.Transform(&transform)
			var bbox fz.Irect
			bbox.RoundRect(&bounds)
			m.height += int(bbox.Y1)
			m.heights[i] = int(bbox.Y1)
			m.doc.FreePage(page)
		}
		m.cache = make([]Pixel, m.Width*m.height)
		m.cached = make([]bool, m.PageCount())
	}
}

func (m *Model) FlipDarkMode() {
	m.Dark = !m.Dark
	m.cached = make([]bool, m.PageCount())
}

/*
func (m *Model) stopEagerLoad() {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.stop = true
	m.wg.Wait()
	m.stop = false
}

func (m *Model) eagerLoad() {
	m.stopEagerLoad()
	m.wg.Add(1)
	go func() {
		for i := 0; i < m.PageCount(); i++ {
			if m.stop {
				break
			}
			if m.cached[i] {
				continue
			}
			m.renderPage(i)
			time.Sleep(10 * time.Millisecond)
		}
		m.wg.Done()
	}()
}
*/

func (m *Model) renderPage(pageIndex int) {
	if m.cached[pageIndex] {
		return
	}
	pageStart := 0
	for i := 0; i < pageIndex; i++ {
		pageStart += m.heights[i]
	}
	renderPage(m.ctx, m.doc, pageIndex, &m.RenderOption, m.cache[m.Width*pageStart:])
	m.cached[pageIndex] = true
}

func (m *Model) Get(y, h, w int) []Pixel {
	m.setWidth(w)
	pageStart := 0
	for i := 0; i < m.PageCount(); i++ {
		pageEnd := pageStart + m.heights[i]
		if !m.cached[i] {
			if pageEnd >= y && pageStart <= y+h {
				m.renderPage(i)
			}
		}
		pageStart = pageEnd
	}

	srcStart := m.Width * y
	srcEnd := srcStart + m.Width*h
	if srcEnd > len(m.cache) {
		srcEnd = len(m.cache)
	}
	return m.cache[srcStart:srcEnd]

}

func renderPage(ctx *fz.Context, doc *fz.Document, pageIndex int, opt *RenderOption, buf []Pixel) {
	page := doc.LoadPage(pageIndex)
	defer doc.FreePage(page)

	var bounds fz.Rect
	doc.BoundPage(page, &bounds)
	zoom := float32(opt.Width) / bounds.X1
	var transform fz.Matrix
	transform.Scale(zoom, zoom)
	bounds.Transform(&transform)
	var bbox fz.Irect
	bbox.RoundRect(&bounds)

	pix := ctx.NewPixmapWithBboxAndData(ctx.DeviceRgb(), &bbox,
		(*byte)(unsafe.Pointer(&buf[0])))
	ctx.ClearPixmapWithValue(pix, 0xFF)

	dev := fz.NewDeviceHook(ctx.NewDrawDevice(pix))
	defer dev.Free()

	var imgRects Rects
	dev.ImageHook = func(image *fz.Image, ctm *fz.Matrix, alpha float32) {
		pt := fz.Point{0, 0}
		pt.Transform(ctm)
		x, y := int(pt.X), int(pt.Y)
		w := round(sqrtf(ctm.A*ctm.A+ctm.B*ctm.B) + 1)
		h := round(sqrtf(ctm.C*ctm.C+ctm.D*ctm.D) + 1)
		imgRects = append(imgRects, &Rect{x, y, w, h})
	}

	doc.RunPage(page, dev.ToDevice(), &transform, nil)

	if opt.Dark {
		pixels := buf
		off := 0
		l := opt.Brightness
		ll := opt.Brightness * opt.Brightness

		// shortcut for white
		rgbWhite := Pixel{0xFF, 0xFF, 0xFF, 0xFF}
		hsvWhite := color.RGBToHSVi(0xFF, 0xFF, 0xFF)
		hsvIn, hsvOut := hsvWhite, hsvWhite
		hsvOut.V = int16(l * float32(color.Fac-hsvOut.V))
		hsvIn.V = int16(ll * float32(hsvIn.V))

		for y := 0; y < int(pix.H); y++ {
			for x := 0; x < int(pix.W); x++ {
				pixel := pixels[off+x]
				if pixel == rgbWhite {
					if imgRects.In(x, y) {
						pixels[off+x].SetRGB(hsvIn.ToRGB())
					} else {
						pixels[off+x].SetRGB(hsvOut.ToRGB())
					}
				} else {
					hsv := color.RGBToHSVi(pixels[off+x].GetRGB())
					if !imgRects.In(x, y) {
						hsv.V = int16(l * float32(color.Fac-hsv.V))
					} else {
						hsv.V = int16(ll * float32(hsv.V))
					}
					pixels[off+x].SetRGB(hsv.ToRGB())
				}
			}
			off += int(pix.W)
		}
	}
}

func (m *Model) disposePageCache() {
	m.cached = nil
	m.cache = nil
}

func (m *Model) Dispose() {
	m.disposePageCache()
	if m.doc != nil {
		m.doc.Close()
		m.doc = nil
	}
	if m.ctx != nil {
		m.ctx.FreeContext()
		m.ctx = nil
	}
}

type Rect struct {
	X      int
	Y      int
	Width  int
	Height int
}

func (r *Rect) In(x, y int) bool {
	return x >= r.X && x < r.X+r.Width &&
		y >= r.Y && y < r.Y+r.Height
}

type Rects []*Rect

func (rs Rects) In(x, y int) bool {
	for _, r := range rs {
		if r.In(x, y) {
			return true
		}
	}
	return false
}
<commit_msg>improve dark mode.<commit_after>// Copyright 2014, Hǎiliàng Wáng. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"github.com/hailiang/color"
	"go-mupdf/fz"
	//	"sync"
	//	"time"
	"unsafe"
)

type RenderOption struct {
	Width      int
	Dark       bool
	Brightness float64
}

type Model struct {
	ctx *fz.Context
	doc *fz.Document

	// render cache
	RenderOption
	height  int
	heights []int
	cache   []Pixel
	cached  []bool
	stop    bool
	//	mu      sync.Mutex // cache lock
	//	wg      sync.WaitGroup
}

func newModel() (*Model, error) {
	ctx := fz.AllocDefault.NewContext(nil, fz.StoreDefault)
	return &Model{
		ctx: ctx,
		RenderOption: RenderOption{
			Dark:       true,
			Brightness: 0.45,
		},
	}, nil
}

func (m *Model) Open(file string) error {
	if m.doc != nil {
		m.doc.Close()
		m.doc = nil
	}
	m.doc = m.ctx.OpenDocument(file)

	return nil
}

func (m *Model) PageCount() int {
	return m.doc.CountPages()
}

func (m *Model) Height() int {
	return len(m.cache) / m.RenderOption.Width
}

func (m *Model) setWidth(width int) {
	if width != m.Width {
		m.Width = width

		m.heights = make([]int, m.PageCount())
		m.height = 0
		for i := 0; i < m.PageCount(); i++ {
			page := m.doc.LoadPage(i)
			var bounds fz.Rect
			m.doc.BoundPage(page, &bounds)
			zoom := float32(width) / bounds.X1
			var transform fz.Matrix
			transform.Scale(zoom, zoom)
			bounds.Transform(&transform)
			var bbox fz.Irect
			bbox.RoundRect(&bounds)
			m.height += int(bbox.Y1)
			m.heights[i] = int(bbox.Y1)
			m.doc.FreePage(page)
		}
		m.cache = make([]Pixel, m.Width*m.height)
		m.cached = make([]bool, m.PageCount())
	}
}

func (m *Model) FlipDarkMode() {
	m.Dark = !m.Dark
	m.cached = make([]bool, m.PageCount())
}

/*
func (m *Model) stopEagerLoad() {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.stop = true
	m.wg.Wait()
	m.stop = false
}

func (m *Model) eagerLoad() {
	m.stopEagerLoad()
	m.wg.Add(1)
	go func() {
		for i := 0; i < m.PageCount(); i++ {
			if m.stop {
				break
			}
			if m.cached[i] {
				continue
			}
			m.renderPage(i)
			time.Sleep(10 * time.Millisecond)
		}
		m.wg.Done()
	}()
}
*/

func (m *Model) renderPage(pageIndex int) {
	if m.cached[pageIndex] {
		return
	}
	pageStart := 0
	for i := 0; i < pageIndex; i++ {
		pageStart += m.heights[i]
	}
	renderPage(m.ctx, m.doc, pageIndex, &m.RenderOption, m.cache[m.Width*pageStart:])
	m.cached[pageIndex] = true
}

func (m *Model) Get(y, h, w int) []Pixel {
	m.setWidth(w)
	pageStart := 0
	for i := 0; i < m.PageCount(); i++ {
		pageEnd := pageStart + m.heights[i]
		if !m.cached[i] {
			if pageEnd >= y && pageStart <= y+h {
				m.renderPage(i)
			}
		}
		pageStart = pageEnd
	}

	srcStart := m.Width * y
	srcEnd := srcStart + m.Width*h
	if srcEnd > len(m.cache) {
		srcEnd = len(m.cache)
	}
	return m.cache[srcStart:srcEnd]

}

func renderPage(ctx *fz.Context, doc *fz.Document, pageIndex int, opt *RenderOption, buf []Pixel) {
	page := doc.LoadPage(pageIndex)
	defer doc.FreePage(page)

	var bounds fz.Rect
	doc.BoundPage(page, &bounds)
	zoom := float32(opt.Width) / bounds.X1
	var transform fz.Matrix
	transform.Scale(zoom, zoom)
	bounds.Transform(&transform)
	var bbox fz.Irect
	bbox.RoundRect(&bounds)

	pix := ctx.NewPixmapWithBboxAndData(ctx.DeviceRgb(), &bbox,
		(*byte)(unsafe.Pointer(&buf[0])))
	ctx.ClearPixmapWithValue(pix, 0xFF)

	dev := fz.NewDeviceHook(ctx.NewDrawDevice(pix))
	defer dev.Free()

	var imgRects Rects
	dev.ImageHook = func(image *fz.Image, ctm *fz.Matrix, alpha float32) {
		pt := fz.Point{0, 0}
		pt.Transform(ctm)
		x, y := int(pt.X), int(pt.Y)
		w := round(sqrtf(ctm.A*ctm.A+ctm.B*ctm.B) + 1)
		h := round(sqrtf(ctm.C*ctm.C+ctm.D*ctm.D) + 1)
		imgRects = append(imgRects, &Rect{x, y, w, h})
	}

	doc.RunPage(page, dev.ToDevice(), &transform, nil)

	if opt.Dark {
		pixels := buf
		off := 0
		b := opt.Brightness
		bb := b * b

		for y := 0; y < int(pix.H); y++ {
			for x := 0; x < int(pix.W); x++ {
				pixel := pixels[off+x]
				rgb := color.RGBFromBytes(pixel.GetRGB())
				if !imgRects.In(x, y) {
					hcl := rgb.ToHCL()
					hcl.L = 1 - hcl.L
					rgb = hcl.ToRGB()
					rgb.Times(b)
				} else {
					rgb.Times(bb)
				}
				pixels[off+x].SetRGB(rgb.ToBytes())
			}
			off += int(pix.W)
		}
	}
}

func (m *Model) disposePageCache() {
	m.cached = nil
	m.cache = nil
}

func (m *Model) Dispose() {
	m.disposePageCache()
	if m.doc != nil {
		m.doc.Close()
		m.doc = nil
	}
	if m.ctx != nil {
		m.ctx.FreeContext()
		m.ctx = nil
	}
}

type Rect struct {
	X      int
	Y      int
	Width  int
	Height int
}

func (r *Rect) In(x, y int) bool {
	return x >= r.X && x < r.X+r.Width &&
		y >= r.Y && y < r.Y+r.Height
}

type Rects []*Rect

func (rs Rects) In(x, y int) bool {
	for _, r := range rs {
		if r.In(x, y) {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package command

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
	"github.com/chrislusf/seaweedfs/weed/remote_storage"
	"github.com/chrislusf/seaweedfs/weed/replication/source"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc"
	"os"
	"strings"
	"time"
)

type RemoteSyncOptions struct {
	filerAddress       *string
	grpcDialOption     grpc.DialOption
	readChunkFromFiler *bool
	debug              *bool
	timeAgo            *time.Duration
	dir                *string
}

const (
	RemoteSyncKeyPrefix = "remote.sync."
)

var _ = filer_pb.FilerClient(&RemoteSyncOptions{})

func (option *RemoteSyncOptions) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
	return pb.WithFilerClient(*option.filerAddress, option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		return fn(client)
	})
}
func (option *RemoteSyncOptions) AdjustedUrl(location *filer_pb.Location) string {
	return location.Url
}

var (
	remoteSyncOptions RemoteSyncOptions
)

func init() {
	cmdFilerRemoteSynchronize.Run = runFilerRemoteSynchronize // break init cycle
	remoteSyncOptions.filerAddress = cmdFilerRemoteSynchronize.Flag.String("filer", "localhost:8888", "filer of the SeaweedFS cluster")
	remoteSyncOptions.dir = cmdFilerRemoteSynchronize.Flag.String("dir", "/", "a mounted directory on filer")
	remoteSyncOptions.readChunkFromFiler = cmdFilerRemoteSynchronize.Flag.Bool("filerProxy", false, "read file chunks from filer instead of volume servers")
	remoteSyncOptions.debug = cmdFilerRemoteSynchronize.Flag.Bool("debug", false, "debug mode to print out filer updated remote files")
	remoteSyncOptions.timeAgo = cmdFilerRemoteSynchronize.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
}

var cmdFilerRemoteSynchronize = &Command{
	UsageLine: "filer.remote.sync -filer=<filerHost>:<filerPort> -dir=/mount/s3_on_cloud",
	Short:     "resumable continuously write back updates to remote storage if the directory is mounted to the remote storage",
	Long: `resumable continuously write back updates to remote storage if the directory is mounted to the remote storage

	filer.remote.sync listens on filer update events.
	If any mounted remote file is updated, it will fetch the updated content,
	and write to the remote storage.
`,
}

func runFilerRemoteSynchronize(cmd *Command, args []string) bool {

	util.LoadConfiguration("security", false)
	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
	remoteSyncOptions.grpcDialOption = grpcDialOption

	dir := *remoteSyncOptions.dir
	filerAddress := *remoteSyncOptions.filerAddress

	filerSource := &source.FilerSource{}
	filerSource.DoInitialize(
		filerAddress,
		pb.ServerToGrpcAddress(filerAddress),
		"/", // does not matter
		*remoteSyncOptions.readChunkFromFiler,
	)

	fmt.Printf("synchronize %s to remote storage...\n", dir)
	util.RetryForever("filer.remote.sync "+dir, func() error {
		return followUpdatesAndUploadToRemote(&remoteSyncOptions, filerSource, dir)
	}, func(err error) bool {
		if err != nil {
			glog.Errorf("synchronize %s: %v", dir, err)
		}
		return true
	})

	return true
}

func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *source.FilerSource, mountedDir string) error {

	// read filer remote storage mount mappings
	_, _, remoteStorageMountLocation, remoteStorage, detectErr := filer.DetectMountInfo(option.grpcDialOption, *option.filerAddress, mountedDir)
	if detectErr != nil {
		return fmt.Errorf("read mount info: %v", detectErr)
	}

	// 1. specified by timeAgo
	// 2. last offset timestamp for this directory
	// 3. directory creation time
	var lastOffsetTs time.Time
	if *option.timeAgo == 0 {
		mountedDirEntry, err := filer_pb.GetEntry(option, util.FullPath(mountedDir))
		if err != nil {
			return fmt.Errorf("lookup %s: %v", mountedDir, err)
		}

		lastOffsetTsNs, err := remote_storage.GetSyncOffset(option.grpcDialOption, *option.filerAddress, mountedDir)
		if mountedDirEntry != nil {
			if err == nil && mountedDirEntry.Attributes.Crtime < lastOffsetTsNs/1000000 {
				lastOffsetTs = time.Unix(0, lastOffsetTsNs)
				glog.V(0).Infof("resume from %v", lastOffsetTs)
			} else {
				lastOffsetTs = time.Unix(mountedDirEntry.Attributes.Crtime, 0)
			}
		} else {
			lastOffsetTs = time.Now()
		}
	} else {
		lastOffsetTs = time.Now().Add(-*option.timeAgo)
	}

	client, err := remote_storage.GetRemoteStorage(remoteStorage)
	if err != nil {
		return err
	}

	handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error {
		message := resp.EventNotification
		if message.NewEntry == nil {
			return nil
		}
		if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
			mappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
			if readErr != nil {
				return fmt.Errorf("unmarshal mappings: %v", readErr)
			}
			if remoteLoc, found := mappings.Mappings[mountedDir]; found {
				if remoteStorageMountLocation.Bucket != remoteLoc.Bucket || remoteStorageMountLocation.Path != remoteLoc.Path {
					glog.Fatalf("Unexpected mount changes %+v => %+v", remoteStorageMountLocation, remoteLoc)
				}
			} else {
				glog.V(0).Infof("unmounted %s exiting ...", mountedDir)
				os.Exit(0)
			}
		}
		if message.NewEntry.Name == remoteStorage.Name+filer.REMOTE_STORAGE_CONF_SUFFIX {
			conf := &remote_pb.RemoteConf{}
			if err := proto.Unmarshal(message.NewEntry.Content, conf); err != nil {
				return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err)
			}
			remoteStorage = conf
			client, err = remote_storage.GetRemoteStorage(remoteStorage)
			return err
		}

		return nil
	}

	eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
		message := resp.EventNotification
		if strings.HasPrefix(resp.Directory, filer.DirectoryEtcRemote) {
			return handleEtcRemoteChanges(resp)
		}

		if message.OldEntry == nil && message.NewEntry == nil {
			return nil
		}
		if message.OldEntry == nil && message.NewEntry != nil {
			if !filer.HasData(message.NewEntry) {
				return nil
			}
			glog.V(2).Infof("create: %+v", resp)
			if !shouldSendToRemote(message.NewEntry) {
				glog.V(2).Infof("skipping creating: %+v", resp)
				return nil
			}
			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
			if message.NewEntry.IsDirectory {
				glog.V(0).Infof("mkdir  %s", remote_storage.FormatLocation(dest))
				return client.WriteDirectory(dest, message.NewEntry)
			}
			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
			reader := filer.NewFileReader(filerSource, message.NewEntry)
			remoteEntry, writeErr := client.WriteFile(dest, message.NewEntry, reader)
			if writeErr != nil {
				return writeErr
			}
			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
		}
		if message.OldEntry != nil && message.NewEntry == nil {
			glog.V(2).Infof("delete: %+v", resp)
			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
			if message.OldEntry.IsDirectory {
				glog.V(0).Infof("rmdir  %s", remote_storage.FormatLocation(dest))
				return client.RemoveDirectory(dest)
			}
			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(dest))
			return client.DeleteFile(dest)
		}
		if message.OldEntry != nil && message.NewEntry != nil {
			oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
			if !shouldSendToRemote(message.NewEntry) {
				glog.V(2).Infof("skipping updating: %+v", resp)
				return nil
			}
			if message.NewEntry.IsDirectory {
				return client.WriteDirectory(dest, message.NewEntry)
			}
			if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name {
				if filer.IsSameData(message.OldEntry, message.NewEntry) {
					glog.V(2).Infof("update meta: %+v", resp)
					return client.UpdateFileMetadata(dest, message.OldEntry, message.NewEntry)
				}
			}
			glog.V(2).Infof("update: %+v", resp)
			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest))
			if err := client.DeleteFile(oldDest); err != nil {
				return err
			}
			reader := filer.NewFileReader(filerSource, message.NewEntry)
			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
			remoteEntry, writeErr := client.WriteFile(dest, message.NewEntry, reader)
			if writeErr != nil {
				return writeErr
			}
			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
		}

		return nil
	}

	processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
		lastTime := time.Unix(0, lastTsNs)
		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
		return remote_storage.SetSyncOffset(option.grpcDialOption, *option.filerAddress, mountedDir, lastTsNs)
	})

	return pb.FollowMetadata(*option.filerAddress, option.grpcDialOption, "filer.remote.sync",
		mountedDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
}

func toRemoteStorageLocation(mountDir, sourcePath util.FullPath, remoteMountLocation *remote_pb.RemoteStorageLocation) *remote_pb.RemoteStorageLocation {
	source := string(sourcePath[len(mountDir):])
	dest := util.FullPath(remoteMountLocation.Path).Child(source)
	return &remote_pb.RemoteStorageLocation{
		Name:   remoteMountLocation.Name,
		Bucket: remoteMountLocation.Bucket,
		Path:   string(dest),
	}
}

func shouldSendToRemote(entry *filer_pb.Entry) bool {
	if entry.RemoteEntry == nil {
		return true
	}
	if entry.RemoteEntry.LastLocalSyncTsNs/1e9 < entry.Attributes.Mtime {
		return true
	}
	return false
}

func updateLocalEntry(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.Entry, remoteEntry *filer_pb.RemoteEntry) error {
	entry.RemoteEntry = remoteEntry
	return filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
		_, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
			Directory: dir,
			Entry:     entry,
		})
		return err
	})
}
<commit_msg>refactor<commit_after>package command

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
	"github.com/chrislusf/seaweedfs/weed/remote_storage"
	"github.com/chrislusf/seaweedfs/weed/replication/source"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc"
	"os"
	"strings"
	"time"
)

type RemoteSyncOptions struct {
	filerAddress       *string
	grpcDialOption     grpc.DialOption
	readChunkFromFiler *bool
	debug              *bool
	timeAgo            *time.Duration
	dir                *string
}

const (
	RemoteSyncKeyPrefix = "remote.sync."
)

var _ = filer_pb.FilerClient(&RemoteSyncOptions{})

func (option *RemoteSyncOptions) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
	return pb.WithFilerClient(*option.filerAddress, option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		return fn(client)
	})
}
func (option *RemoteSyncOptions) AdjustedUrl(location *filer_pb.Location) string {
	return location.Url
}

var (
	remoteSyncOptions RemoteSyncOptions
)

func init() {
	cmdFilerRemoteSynchronize.Run = runFilerRemoteSynchronize // break init cycle
	remoteSyncOptions.filerAddress = cmdFilerRemoteSynchronize.Flag.String("filer", "localhost:8888", "filer of the SeaweedFS cluster")
	remoteSyncOptions.dir = cmdFilerRemoteSynchronize.Flag.String("dir", "/", "a mounted directory on filer")
	remoteSyncOptions.readChunkFromFiler = cmdFilerRemoteSynchronize.Flag.Bool("filerProxy", false, "read file chunks from filer instead of volume servers")
	remoteSyncOptions.debug = cmdFilerRemoteSynchronize.Flag.Bool("debug", false, "debug mode to print out filer updated remote files")
	remoteSyncOptions.timeAgo = cmdFilerRemoteSynchronize.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
}

var cmdFilerRemoteSynchronize = &Command{
	UsageLine: "filer.remote.sync -filer=<filerHost>:<filerPort> -dir=/mount/s3_on_cloud",
	Short:     "resumable continuously write back updates to remote storage if the directory is mounted to the remote storage",
	Long: `resumable continuously write back updates to remote storage if the directory is mounted to the remote storage

	filer.remote.sync listens on filer update events.
	If any mounted remote file is updated, it will fetch the updated content,
	and write to the remote storage.
`,
}

func runFilerRemoteSynchronize(cmd *Command, args []string) bool {

	util.LoadConfiguration("security", false)
	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
	remoteSyncOptions.grpcDialOption = grpcDialOption

	dir := *remoteSyncOptions.dir
	filerAddress := *remoteSyncOptions.filerAddress

	filerSource := &source.FilerSource{}
	filerSource.DoInitialize(
		filerAddress,
		pb.ServerToGrpcAddress(filerAddress),
		"/", // does not matter
		*remoteSyncOptions.readChunkFromFiler,
	)

	fmt.Printf("synchronize %s to remote storage...\n", dir)
	util.RetryForever("filer.remote.sync "+dir, func() error {
		return followUpdatesAndUploadToRemote(&remoteSyncOptions, filerSource, dir)
	}, func(err error) bool {
		if err != nil {
			glog.Errorf("synchronize %s: %v", dir, err)
		}
		return true
	})

	return true
}

func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *source.FilerSource, mountedDir string) error {

	// read filer remote storage mount mappings
	_, _, remoteStorageMountLocation, remoteStorage, detectErr := filer.DetectMountInfo(option.grpcDialOption, *option.filerAddress, mountedDir)
	if detectErr != nil {
		return fmt.Errorf("read mount info: %v", detectErr)
	}

	lastOffsetTs := collectLastSyncOffset(option, mountedDir)

	client, err := remote_storage.GetRemoteStorage(remoteStorage)
	if err != nil {
		return err
	}

	handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error {
		message := resp.EventNotification
		if message.NewEntry == nil {
			return nil
		}
		if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
			mappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
			if readErr != nil {
				return fmt.Errorf("unmarshal mappings: %v", readErr)
			}
			if remoteLoc, found := mappings.Mappings[mountedDir]; found {
				if remoteStorageMountLocation.Bucket != remoteLoc.Bucket || remoteStorageMountLocation.Path != remoteLoc.Path {
					glog.Fatalf("Unexpected mount changes %+v => %+v", remoteStorageMountLocation, remoteLoc)
				}
			} else {
				glog.V(0).Infof("unmounted %s exiting ...", mountedDir)
				os.Exit(0)
			}
		}
		if message.NewEntry.Name == remoteStorage.Name+filer.REMOTE_STORAGE_CONF_SUFFIX {
			conf := &remote_pb.RemoteConf{}
			if err := proto.Unmarshal(message.NewEntry.Content, conf); err != nil {
				return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err)
			}
			remoteStorage = conf
			client, err = remote_storage.GetRemoteStorage(remoteStorage)
			return err
		}

		return nil
	}

	eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
		message := resp.EventNotification
		if strings.HasPrefix(resp.Directory, filer.DirectoryEtcRemote) {
			return handleEtcRemoteChanges(resp)
		}

		if message.OldEntry == nil && message.NewEntry == nil {
			return nil
		}
		if message.OldEntry == nil && message.NewEntry != nil {
			if !filer.HasData(message.NewEntry) {
				return nil
			}
			glog.V(2).Infof("create: %+v", resp)
			if !shouldSendToRemote(message.NewEntry) {
				glog.V(2).Infof("skipping creating: %+v", resp)
				return nil
			}
			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
			if message.NewEntry.IsDirectory {
				glog.V(0).Infof("mkdir  %s", remote_storage.FormatLocation(dest))
				return client.WriteDirectory(dest, message.NewEntry)
			}
			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
			reader := filer.NewFileReader(filerSource, message.NewEntry)
			remoteEntry, writeErr := client.WriteFile(dest, message.NewEntry, reader)
			if writeErr != nil {
				return writeErr
			}
			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
		}
		if message.OldEntry != nil && message.NewEntry == nil {
			glog.V(2).Infof("delete: %+v", resp)
			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
			if message.OldEntry.IsDirectory {
				glog.V(0).Infof("rmdir  %s", remote_storage.FormatLocation(dest))
				return client.RemoveDirectory(dest)
			}
			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(dest))
			return client.DeleteFile(dest)
		}
		if message.OldEntry != nil && message.NewEntry != nil {
			oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
			if !shouldSendToRemote(message.NewEntry) {
				glog.V(2).Infof("skipping updating: %+v", resp)
				return nil
			}
			if message.NewEntry.IsDirectory {
				return client.WriteDirectory(dest, message.NewEntry)
			}
			if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name {
				if filer.IsSameData(message.OldEntry, message.NewEntry) {
					glog.V(2).Infof("update meta: %+v", resp)
					return client.UpdateFileMetadata(dest, message.OldEntry, message.NewEntry)
				}
			}
			glog.V(2).Infof("update: %+v", resp)
			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest))
			if err := client.DeleteFile(oldDest); err != nil {
				return err
			}
			reader := filer.NewFileReader(filerSource, message.NewEntry)
			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
			remoteEntry, writeErr := client.WriteFile(dest, message.NewEntry, reader)
			if writeErr != nil {
				return writeErr
			}
			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
		}

		return nil
	}

	processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
		lastTime := time.Unix(0, lastTsNs)
		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
		return remote_storage.SetSyncOffset(option.grpcDialOption, *option.filerAddress, mountedDir, lastTsNs)
	})

	return pb.FollowMetadata(*option.filerAddress, option.grpcDialOption, "filer.remote.sync",
		mountedDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
}

func collectLastSyncOffset(option *RemoteSyncOptions, mountedDir string) (time.Time) {
	// 1. specified by timeAgo
	// 2. last offset timestamp for this directory
	// 3. directory creation time
	var lastOffsetTs time.Time
	if *option.timeAgo == 0 {
		mountedDirEntry, err := filer_pb.GetEntry(option, util.FullPath(mountedDir))
		if err != nil {
			glog.V(0).Infof("get mounted directory %s: %v", mountedDir, err)
			return time.Now()
		}

		lastOffsetTsNs, err := remote_storage.GetSyncOffset(option.grpcDialOption, *option.filerAddress, mountedDir)
		if mountedDirEntry != nil {
			if err == nil && mountedDirEntry.Attributes.Crtime < lastOffsetTsNs/1000000 {
				lastOffsetTs = time.Unix(0, lastOffsetTsNs)
				glog.V(0).Infof("resume from %v", lastOffsetTs)
			} else {
				lastOffsetTs = time.Unix(mountedDirEntry.Attributes.Crtime, 0)
			}
		} else {
			lastOffsetTs = time.Now()
		}
	} else {
		lastOffsetTs = time.Now().Add(-*option.timeAgo)
	}
	return lastOffsetTs
}

func toRemoteStorageLocation(mountDir, sourcePath util.FullPath, remoteMountLocation *remote_pb.RemoteStorageLocation) *remote_pb.RemoteStorageLocation {
	source := string(sourcePath[len(mountDir):])
	dest := util.FullPath(remoteMountLocation.Path).Child(source)
	return &remote_pb.RemoteStorageLocation{
		Name:   remoteMountLocation.Name,
		Bucket: remoteMountLocation.Bucket,
		Path:   string(dest),
	}
}

func shouldSendToRemote(entry *filer_pb.Entry) bool {
	if entry.RemoteEntry == nil {
		return true
	}
	if entry.RemoteEntry.LastLocalSyncTsNs/1e9 < entry.Attributes.Mtime {
		return true
	}
	return false
}

func updateLocalEntry(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.Entry, remoteEntry *filer_pb.RemoteEntry) error {
	entry.RemoteEntry = remoteEntry
	return filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
		_, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
			Directory: dir,
			Entry:     entry,
		})
		return err
	})
}
<|endoftext|>


<commit_before>package node

import (
	"bytes"
	"crypto/tls"
	"net"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/gansoi/gansoi/cluster"
	"github.com/hashicorp/raft"
)

func getStream(t *testing.T) *HTTPStream {
	core := cluster.NewCore(&cluster.Info{})
	core.Bootstrap()

	certificates, _ := core.Start()
	ca := core.CA()

	stream, err := NewHTTPStream("127.0.0.1:0", certificates, ca)
	if err != nil {
		t.Fatalf("NewHTTPStream() returned an error: %s", err.Error())
		return nil
	}

	return stream
}

func echoServer(stream *HTTPStream) {
	conn, err := stream.Accept()
	if err != nil {
		return
	}

	for {
		b := make([]byte, 1024)
		n, err := conn.Read(b)
		if err != nil {
			return
		}

		_, err = conn.Write(b[:n])
		if err != nil {
			return
		}
	}
}

func TestStreamListen(t *testing.T) {
	stream := getStream(t)
	defer stream.Close()

	server := httptest.NewUnstartedServer(stream)
	server.TLS = &tls.Config{Certificates: stream.certificates}
	server.StartTLS()
	defer server.Close()

	conn, err := stream.Dial(raft.ServerAddress(server.Listener.Addr().String()), time.Millisecond*100)
	if err != nil {
		t.Errorf("Dial() errored: %s\n", err.Error())
	}
	defer conn.Close()
}

func exchange(conn net.Conn, payload []byte) []byte {
	n, err := conn.Write(payload)
	if err != nil {
		panic(err.Error())
	}
	if n != len(payload) {
		panic("fail")
	}

	time.Sleep(time.Millisecond * 10)

	b := make([]byte, 1024)
	n, _ = conn.Read(b)

	return b[:n]
}

func TestStreamEcho(t *testing.T) {
	stream := getStream(t)
	defer stream.Close()

	server := httptest.NewUnstartedServer(stream)
	server.TLS = &tls.Config{Certificates: stream.certificates, ClientAuth: tls.RequestClientCert}
	server.StartTLS()
	defer server.Close()

	go echoServer(stream)

	conn, err := stream.Dial(raft.ServerAddress(server.Listener.Addr().String()), time.Millisecond*100)
	if err != nil {
		t.Fatalf("Dial() errored: %s\n", err.Error())
	}
	defer conn.Close()

	time.Sleep(time.Millisecond * 20) // Give echoServer some time to accept.

	payload := []byte("Hello 123")
	result := exchange(conn, payload)

	if bytes.Compare(payload, result) != 0 {
		t.Errorf("Got wrong reply, got '%s', expected '%s'", result, payload)
	}

	payload = []byte("Hello hello")
	result = exchange(conn, payload)

	if bytes.Compare(payload, result) != 0 {
		t.Errorf("Got wrong reply, got '%s', expected '%s'", result, payload)
	}
}

var _ raft.StreamLayer = (*HTTPStream)(nil)
<commit_msg>More time :).<commit_after>package node

import (
	"bytes"
	"crypto/tls"
	"net"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/gansoi/gansoi/cluster"
	"github.com/hashicorp/raft"
)

func getStream(t *testing.T) *HTTPStream {
	core := cluster.NewCore(&cluster.Info{})
	core.Bootstrap()

	certificates, _ := core.Start()
	ca := core.CA()

	stream, err := NewHTTPStream("127.0.0.1:0", certificates, ca)
	if err != nil {
		t.Fatalf("NewHTTPStream() returned an error: %s", err.Error())
		return nil
	}

	return stream
}

func echoServer(stream *HTTPStream) {
	conn, err := stream.Accept()
	if err != nil {
		return
	}

	for {
		b := make([]byte, 1024)
		n, err := conn.Read(b)
		if err != nil {
			return
		}

		_, err = conn.Write(b[:n])
		if err != nil {
			return
		}
	}
}

func TestStreamListen(t *testing.T) {
	stream := getStream(t)
	defer stream.Close()

	server := httptest.NewUnstartedServer(stream)
	server.TLS = &tls.Config{Certificates: stream.certificates}
	server.StartTLS()
	defer server.Close()

	conn, err := stream.Dial(raft.ServerAddress(server.Listener.Addr().String()), time.Millisecond*100)
	if err != nil {
		t.Errorf("Dial() errored: %s\n", err.Error())
	}
	defer conn.Close()
}

func exchange(conn net.Conn, payload []byte) []byte {
	n, err := conn.Write(payload)
	if err != nil {
		panic(err.Error())
	}
	if n != len(payload) {
		panic("fail")
	}

	time.Sleep(time.Millisecond * 20)

	b := make([]byte, 1024)
	n, _ = conn.Read(b)

	return b[:n]
}

func TestStreamEcho(t *testing.T) {
	stream := getStream(t)
	defer stream.Close()

	server := httptest.NewUnstartedServer(stream)
	server.TLS = &tls.Config{Certificates: stream.certificates, ClientAuth: tls.RequestClientCert}
	server.StartTLS()
	defer server.Close()

	go echoServer(stream)

	conn, err := stream.Dial(raft.ServerAddress(server.Listener.Addr().String()), time.Millisecond*100)
	if err != nil {
		t.Fatalf("Dial() errored: %s\n", err.Error())
	}
	defer conn.Close()

	time.Sleep(time.Millisecond * 20) // Give echoServer some time to accept.

	payload := []byte("Hello 123")
	result := exchange(conn, payload)

	if bytes.Compare(payload, result) != 0 {
		t.Errorf("Got wrong reply, got '%s', expected '%s'", result, payload)
	}

	payload = []byte("Hello hello")
	result = exchange(conn, payload)

	if bytes.Compare(payload, result) != 0 {
		t.Errorf("Got wrong reply, got '%s', expected '%s'", result, payload)
	}
}

var _ raft.StreamLayer = (*HTTPStream)(nil)
<|endoftext|>


<commit_before>// Copyright 2016 The go-github AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package github

import (
	"context"
	"fmt"
	"net/http"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
)

func TestRepositoriesService_ListTrafficReferrers(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/traffic/popular/referrers", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprintf(w, `[{
			"referrer": "Google",
			"count": 4,
			"uniques": 3
 		}]`)
	})
	ctx := context.Background()
	got, _, err := client.Repositories.ListTrafficReferrers(ctx, "o", "r")
	if err != nil {
		t.Errorf("Repositories.ListTrafficReferrers returned error: %+v", err)
	}

	want := []*TrafficReferrer{{
		Referrer: String("Google"),
		Count:    Int(4),
		Uniques:  Int(3),
	}}
	if !cmp.Equal(got, want) {
		t.Errorf("Repositories.ListTrafficReferrers returned %+v, want %+v", got, want)
	}

	const methodName = "ListTrafficReferrers"
	testBadOptions(t, methodName, func() (err error) {
		_, _, err = client.Repositories.ListTrafficReferrers(ctx, "\n", "\n")
		return err
	})

	testNewRequestAndDoFailure(t, methodName, client, func() (*Response, error) {
		got, resp, err := client.Repositories.ListTrafficReferrers(ctx, "o", "r")
		if got != nil {
			t.Errorf("testNewRequestAndDoFailure %v = %#v, want nil", methodName, got)
		}
		return resp, err
	})
}

func TestRepositoriesService_ListTrafficPaths(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/traffic/popular/paths", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprintf(w, `[{
			"path": "/github/hubot",
			"title": "github/hubot: A customizable life embetterment robot.",
			"count": 3542,
			"uniques": 2225
 		}]`)
	})
	ctx := context.Background()
	got, _, err := client.Repositories.ListTrafficPaths(ctx, "o", "r")
	if err != nil {
		t.Errorf("Repositories.ListTrafficPaths returned error: %+v", err)
	}

	want := []*TrafficPath{{
		Path:    String("/github/hubot"),
		Title:   String("github/hubot: A customizable life embetterment robot."),
		Count:   Int(3542),
		Uniques: Int(2225),
	}}
	if !cmp.Equal(got, want) {
		t.Errorf("Repositories.ListTrafficPaths returned %+v, want %+v", got, want)
	}

	const methodName = "ListTrafficPaths"
	testBadOptions(t, methodName, func() (err error) {
		_, _, err = client.Repositories.ListTrafficPaths(ctx, "\n", "\n")
		return err
	})

	testNewRequestAndDoFailure(t, methodName, client, func() (*Response, error) {
		got, resp, err := client.Repositories.ListTrafficPaths(ctx, "o", "r")
		if got != nil {
			t.Errorf("testNewRequestAndDoFailure %v = %#v, want nil", methodName, got)
		}
		return resp, err
	})
}

func TestRepositoriesService_ListTrafficViews(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/traffic/views", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprintf(w, `{"count": 7,
			"uniques": 6,
			"views": [{
				"timestamp": "2016-05-31T16:00:00.000Z",
				"count": 7,
				"uniques": 6
		}]}`)
	})

	ctx := context.Background()
	got, _, err := client.Repositories.ListTrafficViews(ctx, "o", "r", nil)
	if err != nil {
		t.Errorf("Repositories.ListTrafficViews returned error: %+v", err)
	}

	want := &TrafficViews{
		Views: []*TrafficData{{
			Timestamp: &Timestamp{time.Date(2016, time.May, 31, 16, 0, 0, 0, time.UTC)},
			Count:     Int(7),
			Uniques:   Int(6),
		}},
		Count:   Int(7),
		Uniques: Int(6),
	}

	if !cmp.Equal(got, want) {
		t.Errorf("Repositories.ListTrafficViews returned %+v, want %+v", got, want)
	}

	const methodName = "ListTrafficViews"
	testBadOptions(t, methodName, func() (err error) {
		_, _, err = client.Repositories.ListTrafficViews(ctx, "\n", "\n", &TrafficBreakdownOptions{})
		return err
	})

	testNewRequestAndDoFailure(t, methodName, client, func() (*Response, error) {
		got, resp, err := client.Repositories.ListTrafficViews(ctx, "o", "r", nil)
		if got != nil {
			t.Errorf("testNewRequestAndDoFailure %v = %#v, want nil", methodName, got)
		}
		return resp, err
	})
}

func TestRepositoriesService_ListTrafficClones(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/traffic/clones", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprintf(w, `{"count": 7,
			"uniques": 6,
			"clones": [{
				"timestamp": "2016-05-31T16:00:00.00Z",
				"count": 7,
				"uniques": 6
		}]}`)
	})

	ctx := context.Background()
	got, _, err := client.Repositories.ListTrafficClones(ctx, "o", "r", nil)
	if err != nil {
		t.Errorf("Repositories.ListTrafficClones returned error: %+v", err)
	}

	want := &TrafficClones{
		Clones: []*TrafficData{{
			Timestamp: &Timestamp{time.Date(2016, time.May, 31, 16, 0, 0, 0, time.UTC)},
			Count:     Int(7),
			Uniques:   Int(6),
		}},
		Count:   Int(7),
		Uniques: Int(6),
	}

	if !cmp.Equal(got, want) {
		t.Errorf("Repositories.ListTrafficClones returned %+v, want %+v", got, want)
	}

	const methodName = "ListTrafficClones"
	testBadOptions(t, methodName, func() (err error) {
		_, _, err = client.Repositories.ListTrafficClones(ctx, "\n", "\n", &TrafficBreakdownOptions{})
		return err
	})

	testNewRequestAndDoFailure(t, methodName, client, func() (*Response, error) {
		got, resp, err := client.Repositories.ListTrafficClones(ctx, "o", "r", nil)
		if got != nil {
			t.Errorf("testNewRequestAndDoFailure %v = %#v, want nil", methodName, got)
		}
		return resp, err
	})
}
<commit_msg>Add test cases for JSON resource marshaling (#2151)<commit_after>// Copyright 2016 The go-github AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package github

import (
	"context"
	"fmt"
	"net/http"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
)

func TestRepositoriesService_ListTrafficReferrers(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/traffic/popular/referrers", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprintf(w, `[{
			"referrer": "Google",
			"count": 4,
			"uniques": 3
 		}]`)
	})
	ctx := context.Background()
	got, _, err := client.Repositories.ListTrafficReferrers(ctx, "o", "r")
	if err != nil {
		t.Errorf("Repositories.ListTrafficReferrers returned error: %+v", err)
	}

	want := []*TrafficReferrer{{
		Referrer: String("Google"),
		Count:    Int(4),
		Uniques:  Int(3),
	}}
	if !cmp.Equal(got, want) {
		t.Errorf("Repositories.ListTrafficReferrers returned %+v, want %+v", got, want)
	}

	const methodName = "ListTrafficReferrers"
	testBadOptions(t, methodName, func() (err error) {
		_, _, err = client.Repositories.ListTrafficReferrers(ctx, "\n", "\n")
		return err
	})

	testNewRequestAndDoFailure(t, methodName, client, func() (*Response, error) {
		got, resp, err := client.Repositories.ListTrafficReferrers(ctx, "o", "r")
		if got != nil {
			t.Errorf("testNewRequestAndDoFailure %v = %#v, want nil", methodName, got)
		}
		return resp, err
	})
}

func TestRepositoriesService_ListTrafficPaths(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/traffic/popular/paths", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprintf(w, `[{
			"path": "/github/hubot",
			"title": "github/hubot: A customizable life embetterment robot.",
			"count": 3542,
			"uniques": 2225
 		}]`)
	})
	ctx := context.Background()
	got, _, err := client.Repositories.ListTrafficPaths(ctx, "o", "r")
	if err != nil {
		t.Errorf("Repositories.ListTrafficPaths returned error: %+v", err)
	}

	want := []*TrafficPath{{
		Path:    String("/github/hubot"),
		Title:   String("github/hubot: A customizable life embetterment robot."),
		Count:   Int(3542),
		Uniques: Int(2225),
	}}
	if !cmp.Equal(got, want) {
		t.Errorf("Repositories.ListTrafficPaths returned %+v, want %+v", got, want)
	}

	const methodName = "ListTrafficPaths"
	testBadOptions(t, methodName, func() (err error) {
		_, _, err = client.Repositories.ListTrafficPaths(ctx, "\n", "\n")
		return err
	})

	testNewRequestAndDoFailure(t, methodName, client, func() (*Response, error) {
		got, resp, err := client.Repositories.ListTrafficPaths(ctx, "o", "r")
		if got != nil {
			t.Errorf("testNewRequestAndDoFailure %v = %#v, want nil", methodName, got)
		}
		return resp, err
	})
}

func TestRepositoriesService_ListTrafficViews(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/traffic/views", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprintf(w, `{"count": 7,
			"uniques": 6,
			"views": [{
				"timestamp": "2016-05-31T16:00:00.000Z",
				"count": 7,
				"uniques": 6
		}]}`)
	})

	ctx := context.Background()
	got, _, err := client.Repositories.ListTrafficViews(ctx, "o", "r", nil)
	if err != nil {
		t.Errorf("Repositories.ListTrafficViews returned error: %+v", err)
	}

	want := &TrafficViews{
		Views: []*TrafficData{{
			Timestamp: &Timestamp{time.Date(2016, time.May, 31, 16, 0, 0, 0, time.UTC)},
			Count:     Int(7),
			Uniques:   Int(6),
		}},
		Count:   Int(7),
		Uniques: Int(6),
	}

	if !cmp.Equal(got, want) {
		t.Errorf("Repositories.ListTrafficViews returned %+v, want %+v", got, want)
	}

	const methodName = "ListTrafficViews"
	testBadOptions(t, methodName, func() (err error) {
		_, _, err = client.Repositories.ListTrafficViews(ctx, "\n", "\n", &TrafficBreakdownOptions{})
		return err
	})

	testNewRequestAndDoFailure(t, methodName, client, func() (*Response, error) {
		got, resp, err := client.Repositories.ListTrafficViews(ctx, "o", "r", nil)
		if got != nil {
			t.Errorf("testNewRequestAndDoFailure %v = %#v, want nil", methodName, got)
		}
		return resp, err
	})
}

func TestRepositoriesService_ListTrafficClones(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/traffic/clones", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprintf(w, `{"count": 7,
			"uniques": 6,
			"clones": [{
				"timestamp": "2016-05-31T16:00:00.00Z",
				"count": 7,
				"uniques": 6
		}]}`)
	})

	ctx := context.Background()
	got, _, err := client.Repositories.ListTrafficClones(ctx, "o", "r", nil)
	if err != nil {
		t.Errorf("Repositories.ListTrafficClones returned error: %+v", err)
	}

	want := &TrafficClones{
		Clones: []*TrafficData{{
			Timestamp: &Timestamp{time.Date(2016, time.May, 31, 16, 0, 0, 0, time.UTC)},
			Count:     Int(7),
			Uniques:   Int(6),
		}},
		Count:   Int(7),
		Uniques: Int(6),
	}

	if !cmp.Equal(got, want) {
		t.Errorf("Repositories.ListTrafficClones returned %+v, want %+v", got, want)
	}

	const methodName = "ListTrafficClones"
	testBadOptions(t, methodName, func() (err error) {
		_, _, err = client.Repositories.ListTrafficClones(ctx, "\n", "\n", &TrafficBreakdownOptions{})
		return err
	})

	testNewRequestAndDoFailure(t, methodName, client, func() (*Response, error) {
		got, resp, err := client.Repositories.ListTrafficClones(ctx, "o", "r", nil)
		if got != nil {
			t.Errorf("testNewRequestAndDoFailure %v = %#v, want nil", methodName, got)
		}
		return resp, err
	})
}

func TestTrafficReferrer_Marshal(t *testing.T) {
	testJSONMarshal(t, &TrafficReferrer{}, "{}")

	u := &TrafficReferrer{
		Referrer: String("referrer"),
		Count:    Int(0),
		Uniques:  Int(0),
	}

	want := `{
		"referrer" : "referrer",
		"count" : 0,
		"uniques" : 0
	}`

	testJSONMarshal(t, u, want)
}
<|endoftext|>


<commit_before>// Copyright 2020 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bzltestutil

// This package must have no deps beyond Go SDK.
import (
	"fmt"
	"os"
	"path/filepath"
	"runtime"
)

var (
	// Initialized by linker.
	RunDir string

	// Initial working directory.
	testExecDir string
)

// This initializer runs before any user packages.
func init() {
	var err error
	testExecDir, err = os.Getwd()
	if err != nil {
		panic(err)
	}

	// Check if we're being run by Bazel and change directories if so.
	// TEST_SRCDIR and TEST_WORKSPACE are set by the Bazel test runner, so that makes a decent proxy.
	testSrcDir, hasSrcDir := os.LookupEnv("TEST_SRCDIR")
	testWorkspace, hasWorkspace := os.LookupEnv("TEST_WORKSPACE")
	if hasSrcDir && hasWorkspace && RunDir != "" {
		abs := RunDir
		if !filepath.IsAbs(RunDir) {
			abs = filepath.Join(testSrcDir, testWorkspace, RunDir)
		}
		err := os.Chdir(abs)
		// Ignore the Chdir err when on Windows, since it might have have runfiles symlinks.
		// https://github.com/bazelbuild/rules_go/pull/1721#issuecomment-422145904
		if err != nil && runtime.GOOS != "windows" {
			panic(fmt.Sprintf("could not change to test directory: %v", err))
		}
		if err == nil {
			os.Setenv("PWD", abs)
		}
	}

	// Setup the bazel tmpdir as the go tmpdir.
	if tmpDir, ok := os.LookupEnv("TEST_TMPDIR"); ok {
		os.Setenv("TMPDIR", tmpDir)
	}
}
<commit_msg>dont set TEST_TMPDIR to TMPDIR in the test runner (#2777)<commit_after>// Copyright 2020 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bzltestutil

// This package must have no deps beyond Go SDK.
import (
	"fmt"
	"os"
	"path/filepath"
	"runtime"
)

var (
	// Initialized by linker.
	RunDir string

	// Initial working directory.
	testExecDir string
)

// This initializer runs before any user packages.
func init() {
	var err error
	testExecDir, err = os.Getwd()
	if err != nil {
		panic(err)
	}

	// Check if we're being run by Bazel and change directories if so.
	// TEST_SRCDIR and TEST_WORKSPACE are set by the Bazel test runner, so that makes a decent proxy.
	testSrcDir, hasSrcDir := os.LookupEnv("TEST_SRCDIR")
	testWorkspace, hasWorkspace := os.LookupEnv("TEST_WORKSPACE")
	if hasSrcDir && hasWorkspace && RunDir != "" {
		abs := RunDir
		if !filepath.IsAbs(RunDir) {
			abs = filepath.Join(testSrcDir, testWorkspace, RunDir)
		}
		err := os.Chdir(abs)
		// Ignore the Chdir err when on Windows, since it might have have runfiles symlinks.
		// https://github.com/bazelbuild/rules_go/pull/1721#issuecomment-422145904
		if err != nil && runtime.GOOS != "windows" {
			panic(fmt.Sprintf("could not change to test directory: %v", err))
		}
		if err == nil {
			os.Setenv("PWD", abs)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package worker

import (
	"errors"
	"fmt"

	"github.com/youtube/vitess/go/sqltypes"

	"github.com/youtube/vitess/go/vt/key"
	"github.com/youtube/vitess/go/vt/mysqlctl/tmutils"
	tabletmanagerdatapb "github.com/youtube/vitess/go/vt/proto/tabletmanagerdata"
	topodatapb "github.com/youtube/vitess/go/vt/proto/topodata"
	"github.com/youtube/vitess/go/vt/topo"
	"github.com/youtube/vitess/go/vt/vtgate/vindexes"
)

// This file defines the interface and implementations of sharding key resolvers.

// keyspaceIDResolver defines the interface that needs to be satisifed to get a
// keyspace ID from a database row.
type keyspaceIDResolver interface {
	// keyspaceID takes a table row, and returns the keyspace id as bytes.
	// It will return an error if no sharding key can be found.
	keyspaceID(row []sqltypes.Value) ([]byte, error)
}

// v2Resolver is the keyspace id resolver that is used by VTGate V2 deployments.
// In V2, the sharding key column name and type is the same for all tables,
// and the keyspace ID is stored in the sharding key database column.
type v2Resolver struct {
	keyspaceInfo        *topo.KeyspaceInfo
	shardingColumnIndex int
}

// newV2Resolver returns a keyspaceIDResolver for a v2 table.
// V2 keyspaces have a preset sharding column name and type.
func newV2Resolver(keyspaceInfo *topo.KeyspaceInfo, td *tabletmanagerdatapb.TableDefinition) (keyspaceIDResolver, error) {
	if keyspaceInfo.ShardingColumnName == "" {
		return nil, errors.New("ShardingColumnName needs to be set for a v2 sharding key")
	}
	if keyspaceInfo.ShardingColumnType == topodatapb.KeyspaceIdType_UNSET {
		return nil, errors.New("ShardingColumnType needs to be set for a v2 sharding key")
	}
	if td.Type != tmutils.TableBaseTable {
		return nil, fmt.Errorf("a keyspaceID resolver can only be created for a base table, got %v", td.Type)
	}

	// Find the sharding key column index.
	columnIndex, ok := tmutils.TableDefinitionGetColumn(td, keyspaceInfo.ShardingColumnName)
	if !ok {
		return nil, fmt.Errorf("table %v doesn't have a column named '%v'", td.Name, keyspaceInfo.ShardingColumnName)
	}

	return &v2Resolver{keyspaceInfo, columnIndex}, nil
}

// keyspaceID implements the keyspaceIDResolver interface.
func (r *v2Resolver) keyspaceID(row []sqltypes.Value) ([]byte, error) {
	v := row[r.shardingColumnIndex]
	switch r.keyspaceInfo.ShardingColumnType {
	case topodatapb.KeyspaceIdType_BYTES:
		return v.Raw(), nil
	case topodatapb.KeyspaceIdType_UINT64:
		i, err := v.ParseUint64()
		if err != nil {
			return nil, fmt.Errorf("Non numerical value: %v", err)
		}
		return key.Uint64Key(i).Bytes(), nil
	default:
		return nil, fmt.Errorf("unsupported ShardingColumnType: %v", r.keyspaceInfo.ShardingColumnType)
	}
}

// v3Resolver is the keyspace id resolver that is used by VTGate V3 deployments.
// In V3, we use the VSchema to find a Unique VIndex of cost 0 or 1 for each
// table.
type v3Resolver struct {
	shardingColumnIndex int
	vindex              vindexes.Unique
}

// newV3ResolverFromTableDefinition returns a keyspaceIDResolver for a v3 table.
func newV3ResolverFromTableDefinition(keyspaceSchema *vindexes.KeyspaceSchema, td *tabletmanagerdatapb.TableDefinition) (keyspaceIDResolver, error) {
	if td.Type != tmutils.TableBaseTable {
		return nil, fmt.Errorf("a keyspaceID resolver can only be created for a base table, got %v", td.Type)
	}
	tableSchema, ok := keyspaceSchema.Tables[td.Name]
	if !ok {
		return nil, fmt.Errorf("no vschema definition for table %v", td.Name)
	}
	// the primary vindex is most likely the sharding key, and has to
	// be unique.
	if len(tableSchema.ColumnVindexes) == 0 {
		return nil, fmt.Errorf("no vindex definition for table %v", td.Name)
	}
	colVindex := tableSchema.ColumnVindexes[0]
	if colVindex.Vindex.Cost() > 1 {
		return nil, fmt.Errorf("primary vindex cost is too high for table %v", td.Name)
	}
	unique, ok := colVindex.Vindex.(vindexes.Unique)
	if !ok {
		return nil, fmt.Errorf("primary vindex is not unique for table %v", td.Name)
	}

	// Find the sharding key column index.
	columnIndex, ok := tmutils.TableDefinitionGetColumn(td, colVindex.Column.String())
	if !ok {
		return nil, fmt.Errorf("table %v has a Vindex on unknown column %v", td.Name, colVindex.Column)
	}

	return &v3Resolver{
		shardingColumnIndex: columnIndex,
		vindex:              unique,
	}, nil
}

// newV3ResolverFromColumnList returns a keyspaceIDResolver for a v3 table.
func newV3ResolverFromColumnList(keyspaceSchema *vindexes.KeyspaceSchema, name string, columns []string) (keyspaceIDResolver, error) {
	tableSchema, ok := keyspaceSchema.Tables[name]
	if !ok {
		return nil, fmt.Errorf("no vschema definition for table %v", name)
	}
	// the primary vindex is most likely the sharding key, and has to
	// be unique.
	if len(tableSchema.ColumnVindexes) == 0 {
		return nil, fmt.Errorf("no vindex definition for table %v", name)
	}
	colVindex := tableSchema.ColumnVindexes[0]
	if colVindex.Vindex.Cost() > 1 {
		return nil, fmt.Errorf("primary vindex cost is too high for table %v", name)
	}
	unique, ok := colVindex.Vindex.(vindexes.Unique)
	if !ok {
		return nil, fmt.Errorf("primary vindex is not unique for table %v", name)
	}

	// Find the sharding key column index.
	columnIndex := -1
	for i, n := range columns {
		if colVindex.Column.EqualString(n) {
			columnIndex = i
			break
		}
	}
	if columnIndex == -1 {
		return nil, fmt.Errorf("table %v has a Vindex on unknown column %v", name, colVindex.Column)
	}

	return &v3Resolver{
		shardingColumnIndex: columnIndex,
		vindex:              unique,
	}, nil
}

// keyspaceID implements the keyspaceIDResolver interface.
func (r *v3Resolver) keyspaceID(row []sqltypes.Value) ([]byte, error) {
	v := row[r.shardingColumnIndex]
	ids := []interface{}{v}
	ksids, err := r.vindex.Map(nil, ids)
	if err != nil {
		return nil, err
	}
	if len(ksids) != 1 {
		return nil, fmt.Errorf("maping row to keyspace id returned an invalid array of keyspace ids: %v", ksids)
	}
	return ksids[0], nil
}
<commit_msg>Fixing import section.<commit_after>// Copyright 2015, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package worker

import (
	"errors"
	"fmt"

	"github.com/youtube/vitess/go/sqltypes"

	"github.com/youtube/vitess/go/vt/key"
	"github.com/youtube/vitess/go/vt/mysqlctl/tmutils"
	"github.com/youtube/vitess/go/vt/topo"
	"github.com/youtube/vitess/go/vt/vtgate/vindexes"

	tabletmanagerdatapb "github.com/youtube/vitess/go/vt/proto/tabletmanagerdata"
	topodatapb "github.com/youtube/vitess/go/vt/proto/topodata"
)

// This file defines the interface and implementations of sharding key resolvers.

// keyspaceIDResolver defines the interface that needs to be satisifed to get a
// keyspace ID from a database row.
type keyspaceIDResolver interface {
	// keyspaceID takes a table row, and returns the keyspace id as bytes.
	// It will return an error if no sharding key can be found.
	keyspaceID(row []sqltypes.Value) ([]byte, error)
}

// v2Resolver is the keyspace id resolver that is used by VTGate V2 deployments.
// In V2, the sharding key column name and type is the same for all tables,
// and the keyspace ID is stored in the sharding key database column.
type v2Resolver struct {
	keyspaceInfo        *topo.KeyspaceInfo
	shardingColumnIndex int
}

// newV2Resolver returns a keyspaceIDResolver for a v2 table.
// V2 keyspaces have a preset sharding column name and type.
func newV2Resolver(keyspaceInfo *topo.KeyspaceInfo, td *tabletmanagerdatapb.TableDefinition) (keyspaceIDResolver, error) {
	if keyspaceInfo.ShardingColumnName == "" {
		return nil, errors.New("ShardingColumnName needs to be set for a v2 sharding key")
	}
	if keyspaceInfo.ShardingColumnType == topodatapb.KeyspaceIdType_UNSET {
		return nil, errors.New("ShardingColumnType needs to be set for a v2 sharding key")
	}
	if td.Type != tmutils.TableBaseTable {
		return nil, fmt.Errorf("a keyspaceID resolver can only be created for a base table, got %v", td.Type)
	}

	// Find the sharding key column index.
	columnIndex, ok := tmutils.TableDefinitionGetColumn(td, keyspaceInfo.ShardingColumnName)
	if !ok {
		return nil, fmt.Errorf("table %v doesn't have a column named '%v'", td.Name, keyspaceInfo.ShardingColumnName)
	}

	return &v2Resolver{keyspaceInfo, columnIndex}, nil
}

// keyspaceID implements the keyspaceIDResolver interface.
func (r *v2Resolver) keyspaceID(row []sqltypes.Value) ([]byte, error) {
	v := row[r.shardingColumnIndex]
	switch r.keyspaceInfo.ShardingColumnType {
	case topodatapb.KeyspaceIdType_BYTES:
		return v.Raw(), nil
	case topodatapb.KeyspaceIdType_UINT64:
		i, err := v.ParseUint64()
		if err != nil {
			return nil, fmt.Errorf("Non numerical value: %v", err)
		}
		return key.Uint64Key(i).Bytes(), nil
	default:
		return nil, fmt.Errorf("unsupported ShardingColumnType: %v", r.keyspaceInfo.ShardingColumnType)
	}
}

// v3Resolver is the keyspace id resolver that is used by VTGate V3 deployments.
// In V3, we use the VSchema to find a Unique VIndex of cost 0 or 1 for each
// table.
type v3Resolver struct {
	shardingColumnIndex int
	vindex              vindexes.Unique
}

// newV3ResolverFromTableDefinition returns a keyspaceIDResolver for a v3 table.
func newV3ResolverFromTableDefinition(keyspaceSchema *vindexes.KeyspaceSchema, td *tabletmanagerdatapb.TableDefinition) (keyspaceIDResolver, error) {
	if td.Type != tmutils.TableBaseTable {
		return nil, fmt.Errorf("a keyspaceID resolver can only be created for a base table, got %v", td.Type)
	}
	tableSchema, ok := keyspaceSchema.Tables[td.Name]
	if !ok {
		return nil, fmt.Errorf("no vschema definition for table %v", td.Name)
	}
	// the primary vindex is most likely the sharding key, and has to
	// be unique.
	if len(tableSchema.ColumnVindexes) == 0 {
		return nil, fmt.Errorf("no vindex definition for table %v", td.Name)
	}
	colVindex := tableSchema.ColumnVindexes[0]
	if colVindex.Vindex.Cost() > 1 {
		return nil, fmt.Errorf("primary vindex cost is too high for table %v", td.Name)
	}
	unique, ok := colVindex.Vindex.(vindexes.Unique)
	if !ok {
		return nil, fmt.Errorf("primary vindex is not unique for table %v", td.Name)
	}

	// Find the sharding key column index.
	columnIndex, ok := tmutils.TableDefinitionGetColumn(td, colVindex.Column.String())
	if !ok {
		return nil, fmt.Errorf("table %v has a Vindex on unknown column %v", td.Name, colVindex.Column)
	}

	return &v3Resolver{
		shardingColumnIndex: columnIndex,
		vindex:              unique,
	}, nil
}

// newV3ResolverFromColumnList returns a keyspaceIDResolver for a v3 table.
func newV3ResolverFromColumnList(keyspaceSchema *vindexes.KeyspaceSchema, name string, columns []string) (keyspaceIDResolver, error) {
	tableSchema, ok := keyspaceSchema.Tables[name]
	if !ok {
		return nil, fmt.Errorf("no vschema definition for table %v", name)
	}
	// the primary vindex is most likely the sharding key, and has to
	// be unique.
	if len(tableSchema.ColumnVindexes) == 0 {
		return nil, fmt.Errorf("no vindex definition for table %v", name)
	}
	colVindex := tableSchema.ColumnVindexes[0]
	if colVindex.Vindex.Cost() > 1 {
		return nil, fmt.Errorf("primary vindex cost is too high for table %v", name)
	}
	unique, ok := colVindex.Vindex.(vindexes.Unique)
	if !ok {
		return nil, fmt.Errorf("primary vindex is not unique for table %v", name)
	}

	// Find the sharding key column index.
	columnIndex := -1
	for i, n := range columns {
		if colVindex.Column.EqualString(n) {
			columnIndex = i
			break
		}
	}
	if columnIndex == -1 {
		return nil, fmt.Errorf("table %v has a Vindex on unknown column %v", name, colVindex.Column)
	}

	return &v3Resolver{
		shardingColumnIndex: columnIndex,
		vindex:              unique,
	}, nil
}

// keyspaceID implements the keyspaceIDResolver interface.
func (r *v3Resolver) keyspaceID(row []sqltypes.Value) ([]byte, error) {
	v := row[r.shardingColumnIndex]
	ids := []interface{}{v}
	ksids, err := r.vindex.Map(nil, ids)
	if err != nil {
		return nil, err
	}
	if len(ksids) != 1 {
		return nil, fmt.Errorf("maping row to keyspace id returned an invalid array of keyspace ids: %v", ksids)
	}
	return ksids[0], nil
}
<|endoftext|>


<commit_before>package main

import (
	. "github.com/gocircuit/circuit/gocircuit.org/render"
)

func RenderIndexPage() string {
	figs := A{
		"FigFacade": RenderFigurePngSvg(
			"Circuit API provides a dynamic hierarchical view of a compute cluster.", "view", "550px"),
	}
	return RenderHtml(
		"Circuit: Self-managed infrastructure, programmatic monitoring and orchestration",
		Render(indexBody, figs),
	)
}

const indexBody = `

The circuit is a minimal distributed operating system that enables programmatic, reactive control
over hosts, processes and connections within a compute cluster.

{{.FigFacade}}

The circuit is unique in one respect: Once a circuit cluster is formed, the circuit system itself cannot
fail—only individual hosts can. In contrast, comparable systems
(like
CoreOS,
Consul and
Mesosphere)
can fail if the hardware hosting the system's own software fails.

Sources

Find the source repository for Circuit on GitHub.
Follow us on Twitter @gocircuit.

Submit issues to our GitHub repo. For discussions about using and developing
the Circuit visit the Circuit User Group and
the Circuit Developer Group, respectively.

Documentation

Building and installing

Running Circuit servers

Programming metaphor

Command-line client

Using processes

Using containers

Using subscriptions

Using name servers

Using servers

Using channel

Go client

Connecting to a circuit cluster

Navigating and using the anchor hierarchy

Using processes

Using containers

Using subscriptions

Using name servers

Using servers

Using channels

Security and networking

History and bibliography

Tutorials

Orchestrating a typical web app: Node.js using MySQL running on Amazon EC2

Overview

Prepare host images

Boot the circuit cluster

Write the circuit app

Run the app on the cluster

        `
<commit_msg>index<commit_after>package main

import (
	. "github.com/gocircuit/circuit/gocircuit.org/render"
)

func RenderIndexPage() string {
	figs := A{
		"FigFacade": RenderFigurePngSvg(
			"Circuit API provides a dynamic hierarchical view of a compute cluster.", "view", "550px"),
	}
	return RenderHtml(
		"Circuit: Self-managed infrastructure, programmatic monitoring and orchestration",
		Render(indexBody, figs),
	)
}

const indexBody = `

The circuit is a minimal distributed operating system that enables programmatic, reactive control
over hosts, processes and connections within a compute cluster.

{{.FigFacade}}

The circuit is unique in one respect: Once a circuit cluster is formed, the circuit system itself cannot
fail—only individual hosts can. In contrast, comparable systems
(like
CoreOS,
Consul and
Mesosphere)
can fail if the hardware hosting the system's own software fails.

Sources

Find the source repository for Circuit on GitHub.
Follow us on Twitter @gocircuit.

Submit issues to our GitHub repo. For discussions about using and developing
the Circuit visit the Circuit User Group and
the Circuit Developer Group, respectively.

Documentation

Building and installing

Running Circuit servers

Programming metaphor

Command-line client

Using processes

Using containers

Using subscriptions

Using name servers

Using servers

Using channel

Go client

Connecting to a circuit cluster

Navigating and using the anchor hierarchy

Using processes

Using containers

Using subscriptions

Using name servers

Using servers

Using channels

Security and networking

History and bibliography

Tutorials

Orchestrating a typical web app: Node.js using MySQL running on Amazon EC2

Overview

Prepare host images

Boot the circuit cluster

Write the circuit app

Run the app on the cluster

        `
<|endoftext|>


<commit_before>// package monit provides a metric reporting mechanism for webapps
//
// Basic usage:
//
//		m = monit.NewMonitor(monit.Config{
//			Host: "https://myhost.com/reporting/",
//			Base: map[string]interface{}{
//				"auth": "maybeINeedThis?"
//			},
//		})
//		m.Start()
package monit

import (
	"github.com/swhite24/envreader"
	"strconv"
	"time"
	"runtime"
	"encoding/json"
	"bytes"
	"net/http"
	"crypto/tls"
)

var (
	stats runtime.MemStats
	client http.Client
)

type (

	// Config contains configuration values for monit.  Values default to environment
	// variables where appropriate.
	Config struct {

		// Fully qualified url to send monitoring data to.
		// Can be provided in MONIT_HOST environment variable
		Host string

		// Interval, in seconds, at which to report metrics.
		// Can be provided in MONIT_INTERVAL environment variable
		Interval int

		// Base contains key value pairs to include as part of base object reported
		Base map[string]interface{}
	}

	// Monit exposes monitoring func
	Monit struct {
		config Config
		requests int
		cont bool
		start int64
	}
)

// NewMonitor provides an instance of Monit.
//
// Any zero-valued Config properties will use environment variables described above where appropriate.
func NewMonitor (c Config) (m Monit) {
	// Load environment
	vals := envreader.Read("MONIT_HOST", "MONIT_INTERVAL")

	// Check c
	if c.Host == "" { c.Host = vals["MONIT_HOST"] }
	if c.Interval == 0 {
		i, err := strconv.Atoi(vals["MONIT_INTERVAL"])
		if (err != nil) { panic(err) }
		c.Interval = i
	}
	if len(c.Base) == 0 { c.Base = make(map[string]interface{}) }

	m = Monit{ c , 0, true, time.Now().Unix() }

	return m
}

// Start starts a goroutine to report metrics to host based on Config value.
func (m *Monit) Start () {
	go (func (m *Monit) {
		for m.cont {
			// Sleep for specified interval
			time.Sleep(time.Duration(m.config.Interval) * time.Second)

			m.report()

			// Reset
			m.requests = 0
		}
	})(m)
}

func (m Monit) report () {
	// Get current stats
	m.getStat()

	// Get json buffer
	stat, _ := json.Marshal(m.config.Base)
	buf := bytes.NewBuffer(stat)

	// Issue request
	client.Post(m.config.Host, "application/json", buf)
}

func (m Monit) getStat () {
	runtime.ReadMemStats(&stats)

	// Mem_used in MB
	m.config.Base["mem_used"] = float64(stats.HeapAlloc) / 1000000
	m.config.Base["uptime"] = time.Now().Unix() - m.start
}

// Stop stops all reporting.  Call Start to begin again.
func (m *Monit) Stop () {
	m.cont = false
}

// Request increments count of requests to report for current interval.
func (m *Monit) Request () {
	m.requests = m.requests + 1
}

func init () {
	// Setup transport to ignore invalid certs
	transport := &http.Transport{
		TLSClientConfig: &tls.Config{ InsecureSkipVerify: true },
	}
	client = http.Client{ Transport: transport }
}<commit_msg>Update memory property<commit_after>// package monit provides a metric reporting mechanism for webapps
//
// Basic usage:
//
//		m = monit.NewMonitor(monit.Config{
//			Host: "https://myhost.com/reporting/",
//			Base: map[string]interface{}{
//				"auth": "maybeINeedThis?"
//			},
//		})
//		m.Start()
package monit

import (
	"github.com/swhite24/envreader"
	"strconv"
	"time"
	"runtime"
	"encoding/json"
	"bytes"
	"net/http"
	"crypto/tls"
)

var (
	stats runtime.MemStats
	client http.Client
)

type (

	// Config contains configuration values for monit.  Values default to environment
	// variables where appropriate.
	Config struct {

		// Fully qualified url to send monitoring data to.
		// Can be provided in MONIT_HOST environment variable
		Host string

		// Interval, in seconds, at which to report metrics.
		// Can be provided in MONIT_INTERVAL environment variable
		Interval int

		// Base contains key value pairs to include as part of base object reported
		Base map[string]interface{}
	}

	// Monit exposes monitoring func
	Monit struct {
		config Config
		requests int
		cont bool
		start int64
	}
)

// NewMonitor provides an instance of Monit.
//
// Any zero-valued Config properties will use environment variables described above where appropriate.
func NewMonitor (c Config) (m Monit) {
	// Load environment
	vals := envreader.Read("MONIT_HOST", "MONIT_INTERVAL")

	// Check c
	if c.Host == "" { c.Host = vals["MONIT_HOST"] }
	if c.Interval == 0 {
		i, err := strconv.Atoi(vals["MONIT_INTERVAL"])
		if (err != nil) { panic(err) }
		c.Interval = i
	}
	if len(c.Base) == 0 { c.Base = make(map[string]interface{}) }

	m = Monit{ c , 0, true, time.Now().Unix() }

	return m
}

// Start starts a goroutine to report metrics to host based on Config value.
func (m *Monit) Start () {
	go (func (m *Monit) {
		for m.cont {
			// Sleep for specified interval
			time.Sleep(time.Duration(m.config.Interval) * time.Second)

			m.report()

			// Reset
			m.requests = 0
		}
	})(m)
}

func (m Monit) report () {
	// Get current stats
	m.getStat()

	// Get json buffer
	stat, _ := json.Marshal(m.config.Base)
	buf := bytes.NewBuffer(stat)

	// Issue request
	client.Post(m.config.Host, "application/json", buf)
}

func (m Monit) getStat () {
	runtime.ReadMemStats(&stats)

	// Mem_used in MB
	m.config.Base["app_used_memory"] = float64(stats.HeapAlloc) / 1000000
	m.config.Base["uptime"] = time.Now().Unix() - m.start
}

// Stop stops all reporting.  Call Start to begin again.
func (m *Monit) Stop () {
	m.cont = false
}

// Request increments count of requests to report for current interval.
func (m *Monit) Request () {
	m.requests = m.requests + 1
}

func init () {
	// Setup transport to ignore invalid certs
	transport := &http.Transport{
		TLSClientConfig: &tls.Config{ InsecureSkipVerify: true },
	}
	client = http.Client{ Transport: transport }
}<|endoftext|>


<commit_before>// An example FTP server build on top of graval. graval handles the details
// of the FTP protocol, we just provide a persistence driver for rackspace
// cloud files.
//
// If you're looking to create a custom graval driver, this example is a
// reasonable starting point. I suggest copying this file and changing the
// function bodies as required.
package main

import (
	"errors"
	"github.com/ncw/swift"
	"github.com/yob/graval"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"
)

// A minimal driver for graval that stores all data on rackspace cloudfiles. The
// authentication are ignored, any username and password will work.
//
// This really just exists as a minimal demonstration of the interface graval
// drivers are required to implement.
type SwiftDriver struct{
	connection *swift.Connection
	container  string
	user       string
}

func (driver *SwiftDriver) Authenticate(user string, pass string) bool {
	log.Printf("Authenticate: %s %s", user, pass)
	driver.user = user
	return true
}
func (driver *SwiftDriver) Bytes(path string) (bytes int) {
	log.Printf("Bytes: %s", path)
	bytes = -1
	return
}
func (driver *SwiftDriver) ModifiedTime(path string) (time.Time, error) {
	log.Printf("ModifiedTime: %s", path)
	return time.Now(), nil
}
func (driver *SwiftDriver) ChangeDir(path string) bool {
	log.Printf("ChangeDir: %s", path)
	return false
}
func (driver *SwiftDriver) DirContents(path string) (files []os.FileInfo) {
	path = scoped_path_with_trailing_slash(driver.user, path)
	log.Printf("DirContents: %s", path)
	opts    := &swift.ObjectsOpts{Prefix:path}
	objects, err := driver.connection.ObjectsAll(driver.container, opts)
	if err != nil {
		return // error connecting to cloud files
	}
	for _, object := range objects {
		tail  := strings.Replace(object.Name, path, "", 1)
        basename := strings.Split(tail, "/")[0]
		if object.ContentType == "application/directory" {
			files = append(files, graval.NewDirItem(basename))
		} else  {
			files = append(files, graval.NewFileItem(basename, int(object.Bytes)))
		}
	}
	return
}

func (driver *SwiftDriver) DeleteDir(path string) bool {
	log.Printf("DeleteDir: %s", path)
	return false
}
func (driver *SwiftDriver) DeleteFile(path string) bool {
	log.Printf("DeleteFile: %s", path)
	return false
}
func (driver *SwiftDriver) Rename(fromPath string, toPath string) bool {
	log.Printf("Rename: %s %s", fromPath, toPath)
	return false
}
func (driver *SwiftDriver) MakeDir(path string) bool {
	path = scoped_path_with_trailing_slash(driver.user, path)
	log.Printf("MakeDir: %s", path)
	opts    := &swift.ObjectsOpts{Prefix:path}
	objects, err := driver.connection.ObjectNames(driver.container, opts)
	if err != nil {
		return false // error connection to cloud files
	}
	if len(objects) > 0 {
		return false // the dir already exists
	}
	driver.connection.ObjectPutString(driver.container, path, "", "application/directory")
	return true
}
func (driver *SwiftDriver) GetFile(path string) (data string, err error) {
	log.Printf("GetFile: %d", len(data))
	return "", errors.New("foo")
}
func (driver *SwiftDriver) PutFile(destPath string, data io.Reader) bool {
	destPath = scoped_path(driver.user, destPath)
	log.Printf("PutFile: %s", destPath)
	contents, err := ioutil.ReadAll(data)
	if err != nil {
		return false
	}
	err = driver.connection.ObjectPutBytes(driver.container, destPath, contents, "application/octet-stream")
	if err != nil {
		return false
	}
	return true
}

func scoped_path_with_trailing_slash(user string, path string) string {
	path = scoped_path(user, path)
	if !strings.HasSuffix(path, "/") {
		path += "/"
	}
	if path == "/" {
		return ""
	}
	return path
}

func scoped_path(user string, path string) string {
	if path == "/" {
		path = ""
	}
	return filepath.Join("/", user, path)
}

// graval requires a factory that will create a new driver instance for each
// client connection. Generally the factory will be fairly minimal. This is
// a good place to read any required config for your driver.
type SwiftDriverFactory struct{}

func (factory *SwiftDriverFactory) NewDriver() (graval.FTPDriver, error) {
	driver := &SwiftDriver{}
	driver.container  = "rba-uploads"
	driver.connection = &swift.Connection{
		UserName: os.Getenv("UserName"),
		ApiKey:   os.Getenv("ApiKey"),
		AuthUrl:  os.Getenv("AuthUrl"),
	}
	err := driver.connection.Authenticate()
	if err != nil {
		return nil, err
	}
	return driver, nil
}

// it's alive!
func main() {
	factory := &SwiftDriverFactory{}
	ftpServer := graval.NewFTPServer(&graval.FTPServerOpts{ Factory: factory })
	err := ftpServer.ListenAndServe()
	if err != nil {
		log.Fatal("Error starting server!")
		log.Fatal(err)
	}
}
<commit_msg>implement some more of the graval-swift example<commit_after>// An example FTP server build on top of graval. graval handles the details
// of the FTP protocol, we just provide a persistence driver for rackspace
// cloud files.
//
// If you're looking to create a custom graval driver, this example is a
// reasonable starting point. I suggest copying this file and changing the
// function bodies as required.
package main

import (
	"errors"
	"github.com/ncw/swift"
	"github.com/yob/graval"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"
)

// A minimal driver for graval that stores all data on rackspace cloudfiles. The
// authentication are ignored, any username and password will work.
//
// This really just exists as a minimal demonstration of the interface graval
// drivers are required to implement.
type SwiftDriver struct{
	connection *swift.Connection
	container  string
	user       string
}

func (driver *SwiftDriver) Authenticate(user string, pass string) bool {
	log.Printf("Authenticate: %s %s", user, pass)
	driver.user = user
	return true
}
func (driver *SwiftDriver) Bytes(path string) (bytes int) {
	path = scoped_path(driver.user, path)
	log.Printf("Bytes: %s", path)
	object, _, err := driver.connection.Object(driver.container, path)
	if err != nil {
		return -1
	}
	return int(object.Bytes)
}
func (driver *SwiftDriver) ModifiedTime(path string) (time.Time, error) {
	path = scoped_path(driver.user, path)
	log.Printf("ModifiedTime: %s", path)
	object, _, err := driver.connection.Object(driver.container, path)
	if err != nil {
		return time.Now(), err
	}
	return object.LastModified, nil
}
func (driver *SwiftDriver) ChangeDir(path string) bool {
	log.Printf("ChangeDir: %s", path)
	return false
}
func (driver *SwiftDriver) DirContents(path string) (files []os.FileInfo) {
	path = scoped_path_with_trailing_slash(driver.user, path)
	log.Printf("DirContents: %s", path)
	opts    := &swift.ObjectsOpts{Prefix:path}
	objects, err := driver.connection.ObjectsAll(driver.container, opts)
	if err != nil {
		return // error connecting to cloud files
	}
	for _, object := range objects {
		tail  := strings.Replace(object.Name, path, "", 1)
        basename := strings.Split(tail, "/")[0]
		if object.ContentType == "application/directory" {
			files = append(files, graval.NewDirItem(basename))
		} else  {
			files = append(files, graval.NewFileItem(basename, int(object.Bytes)))
		}
	}
	return
}

func (driver *SwiftDriver) DeleteDir(path string) bool {
	log.Printf("DeleteDir: %s", path)
	return false
}
func (driver *SwiftDriver) DeleteFile(path string) bool {
	log.Printf("DeleteFile: %s", path)
	return false
}
func (driver *SwiftDriver) Rename(fromPath string, toPath string) bool {
	log.Printf("Rename: %s %s", fromPath, toPath)
	return false
}
func (driver *SwiftDriver) MakeDir(path string) bool {
	path = scoped_path_with_trailing_slash(driver.user, path)
	log.Printf("MakeDir: %s", path)
	opts    := &swift.ObjectsOpts{Prefix:path}
	objects, err := driver.connection.ObjectNames(driver.container, opts)
	if err != nil {
		return false // error connection to cloud files
	}
	if len(objects) > 0 {
		return false // the dir already exists
	}
	driver.connection.ObjectPutString(driver.container, path, "", "application/directory")
	return true
}
func (driver *SwiftDriver) GetFile(path string) (data string, err error) {
	path = scoped_path(driver.user, path)
	log.Printf("GetFile: %s", path)
	data, err = driver.connection.ObjectGetString(driver.container, path)
	if err != nil {
		return "", errors.New("foo")
	}
	return
}
func (driver *SwiftDriver) PutFile(destPath string, data io.Reader) bool {
	destPath = scoped_path(driver.user, destPath)
	log.Printf("PutFile: %s", destPath)
	contents, err := ioutil.ReadAll(data)
	if err != nil {
		return false
	}
	err = driver.connection.ObjectPutBytes(driver.container, destPath, contents, "application/octet-stream")
	if err != nil {
		return false
	}
	return true
}

func scoped_path_with_trailing_slash(user string, path string) string {
	path = scoped_path(user, path)
	if !strings.HasSuffix(path, "/") {
		path += "/"
	}
	if path == "/" {
		return ""
	}
	return path
}

func scoped_path(user string, path string) string {
	if path == "/" {
		path = ""
	}
	return filepath.Join("/", user, path)
}

// graval requires a factory that will create a new driver instance for each
// client connection. Generally the factory will be fairly minimal. This is
// a good place to read any required config for your driver.
type SwiftDriverFactory struct{}

func (factory *SwiftDriverFactory) NewDriver() (graval.FTPDriver, error) {
	driver := &SwiftDriver{}
	driver.container  = "rba-uploads"
	driver.connection = &swift.Connection{
		UserName: os.Getenv("UserName"),
		ApiKey:   os.Getenv("ApiKey"),
		AuthUrl:  os.Getenv("AuthUrl"),
	}
	err := driver.connection.Authenticate()
	if err != nil {
		return nil, err
	}
	return driver, nil
}

// it's alive!
func main() {
	factory := &SwiftDriverFactory{}
	ftpServer := graval.NewFTPServer(&graval.FTPServerOpts{ Factory: factory })
	err := ftpServer.ListenAndServe()
	if err != nil {
		log.Fatal("Error starting server!")
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package controllers

import (
	// Don't forget this first import or nothing will work
	_ "crypto/sha512"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"os"

	"github.com/labstack/echo"
	"golang.org/x/oauth2"
)

func CallbackHandler(c echo.Context) error {
	domain := "jessemillar.auth0.com"

	// Instantiating the OAuth2 package to exchange the Code for a Token
	conf := &oauth2.Config{
		ClientID:     os.Getenv("AUTH0_CLIENT_ID"),
		ClientSecret: os.Getenv("AUTH0_CLIENT_SECRET"),
		RedirectURL:  "http://localhost:8000/callback",
		Scopes:       []string{"openid", "name", "email", "nickname"},
		Endpoint: oauth2.Endpoint{
			AuthURL:  "https://" + domain + "/authorize",
			TokenURL: "https://" + domain + "/oauth/token",
		},
	}

	// Getting the Code that we got from Auth0
	code := c.QueryParam("code")

	// Exchanging the code for a token
	token, err := conf.Exchange(oauth2.NoContext, code)
	if err != nil {
		return c.String(http.StatusInternalServerError, err.Error())
	}

	// Getting now the User information
	client := conf.Client(oauth2.NoContext, token)
	resp, err := client.Get("https://" + domain + "/userinfo")
	if err != nil {
		return c.String(http.StatusInternalServerError, err.Error())
	}

	// Reading the body
	raw, err := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()
	if err != nil {
		return c.String(http.StatusInternalServerError, err.Error())
	}

	// Unmarshalling the JSON of the Profile
	var profile map[string]interface{}
	if err := json.Unmarshal(raw, &profile); err != nil {
		return c.String(http.StatusInternalServerError, err.Error())
	}

	// Saving the information to the session.
	// We're using https://github.com/astaxie/beego/tree/master/session
	// The GlobalSessions variable is initialized in another file
	// Check https://github.com/auth0/auth0-golang/blob/master/examples/regular-web-app/app/app.go
	// session, _ := app.GlobalSessions.SessionStart(c, c.Request())
	// defer session.SessionRelease(c)

	// session.Set("id_token", token.Extra("id_token"))
	// session.Set("access_token", token.AccessToken)
	// session.Set("profile", profile)

	// Redirect to logged in page
	c.Redirect(http.StatusMovedPermanently, "/health")

	return c.String(http.StatusOK, "DONE")
}
<commit_msg>Sessions might be fixed<commit_after>package controllers

import (
	// Don't forget this first import or nothing will work
	_ "crypto/sha512"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"

	"github.com/jessemillar/byudzhet/packages/app"
	"github.com/labstack/echo"
	"github.com/labstack/echo/engine/standard"
	"golang.org/x/oauth2"
)

func CallbackHandler(c echo.Context) error {
	domain := "jessemillar.auth0.com"

	// Instantiating the OAuth2 package to exchange the Code for a Token
	conf := &oauth2.Config{
		ClientID:     os.Getenv("AUTH0_CLIENT_ID"),
		ClientSecret: os.Getenv("AUTH0_CLIENT_SECRET"),
		RedirectURL:  "http://localhost:8000/callback",
		Scopes:       []string{"openid", "name", "email", "nickname"},
		Endpoint: oauth2.Endpoint{
			AuthURL:  "https://" + domain + "/authorize",
			TokenURL: "https://" + domain + "/oauth/token",
		},
	}

	// Getting the Code that we got from Auth0
	code := c.QueryParam("code")

	// Exchanging the code for a token
	token, err := conf.Exchange(oauth2.NoContext, code)
	if err != nil {
		return c.String(http.StatusInternalServerError, err.Error())
	}

	// Getting now the User information
	client := conf.Client(oauth2.NoContext, token)
	resp, err := client.Get("https://" + domain + "/userinfo")
	if err != nil {
		return c.String(http.StatusInternalServerError, err.Error())
	}

	// Reading the body
	raw, err := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()
	if err != nil {
		return c.String(http.StatusInternalServerError, err.Error())
	}

	// Unmarshalling the JSON of the Profile
	var profile map[string]interface{}
	if err := json.Unmarshal(raw, &profile); err != nil {
		return c.String(http.StatusInternalServerError, err.Error())
	}

	// Saving the information to the session.
	// We're using https://github.com/astaxie/beego/tree/master/session
	// The GlobalSessions variable is initialized in another file
	// Check https://github.com/auth0/auth0-golang/blob/master/examples/regular-web-app/app/app.go
	session, _ := app.GlobalSessions.SessionStart(c.Response().(*standard.Response).ResponseWriter, c.Request().(*standard.Request).Request)
	defer session.SessionRelease(c.Response().(*standard.Response).ResponseWriter)

	session.Set("id_token", token.Extra("id_token"))
	session.Set("access_token", token.AccessToken)
	session.Set("profile", profile)

	stuff := session.Get("profile")
	fmt.Printf("Profile: %+v", stuff)

	// Redirect to logged in page
	c.Redirect(http.StatusMovedPermanently, "/health")

	return c.String(http.StatusOK, "DONE")
}
<|endoftext|>