<commit_before>package inbound

import (
	"context"
	"io"
	"sync"
	"sync/atomic"
	"time"

	"v2ray.com/core/app/dispatcher"
	"v2ray.com/core/app/log"
	"v2ray.com/core/app/proxyman"
	"v2ray.com/core/common/buf"
	"v2ray.com/core/common/net"
	"v2ray.com/core/proxy"
	"v2ray.com/core/transport/internet"
	"v2ray.com/core/transport/internet/tcp"
	"v2ray.com/core/transport/internet/udp"
)

type worker interface {
	Start() error
	Close()
	Port() net.Port
	Proxy() proxy.Inbound
}

type tcpWorker struct {
	address      net.Address
	port         net.Port
	proxy        proxy.Inbound
	stream       *internet.StreamConfig
	recvOrigDest bool
	tag          string
	dispatcher   dispatcher.Interface
	sniffers     []proxyman.KnownProtocols

	ctx    context.Context
	cancel context.CancelFunc
	hub    internet.Listener
}

func (w *tcpWorker) callback(conn internet.Connection) {
	ctx, cancel := context.WithCancel(w.ctx)
	if w.recvOrigDest {
		dest, err := tcp.GetOriginalDestination(conn)
		if err != nil {
			log.Trace(newError("failed to get original destination").Base(err))
		}
		if dest.IsValid() {
			ctx = proxy.ContextWithOriginalTarget(ctx, dest)
		}
	}
	if len(w.tag) > 0 {
		ctx = proxy.ContextWithInboundTag(ctx, w.tag)
	}
	ctx = proxy.ContextWithInboundEntryPoint(ctx, net.TCPDestination(w.address, w.port))
	ctx = proxy.ContextWithSource(ctx, net.DestinationFromAddr(conn.RemoteAddr()))
	if len(w.sniffers) > 0 {
		ctx = proxyman.ContextWithProtocolSniffers(ctx, w.sniffers)
	}
	if err := w.proxy.Process(ctx, net.Network_TCP, conn, w.dispatcher); err != nil {
		log.Trace(newError("connection ends").Base(err))
	}
	cancel()
	conn.Close()
}

func (w *tcpWorker) Proxy() proxy.Inbound {
	return w.proxy
}

func (w *tcpWorker) Start() error {
	ctx, cancel := context.WithCancel(context.Background())
	w.ctx = ctx
	w.cancel = cancel
	ctx = internet.ContextWithStreamSettings(ctx, w.stream)
	conns := make(chan internet.Connection, 16)
	hub, err := internet.ListenTCP(ctx, w.address, w.port, conns)
	if err != nil {
		return newError("failed to listen TCP on ", w.port).Base(err)
	}
	go w.handleConnections(conns)
	w.hub = hub
	return nil
}

func (w *tcpWorker) handleConnections(conns <-chan internet.Connection) {
	for {
		select {
		case <-w.ctx.Done():
			w.hub.Close()
		L:
			for {
				select {
				case conn := <-conns:
					conn.Close()
				default:
					break L
				}
			}
			return
		case conn := <-conns:
			go w.callback(conn)
		}
	}
}

func (w *tcpWorker) Close() {
	if w.hub != nil {
		w.cancel()
	}
}

func (w *tcpWorker) Port() net.Port {
	return w.port
}

type udpConn struct {
	lastActivityTime int64 // in seconds
	input            chan *buf.Buffer
	output           func([]byte) (int, error)
	remote           net.Addr
	local            net.Addr
	cancel           context.CancelFunc
}

func (c *udpConn) updateActivity() {
	atomic.StoreInt64(&c.lastActivityTime, time.Now().Unix())
}

func (c *udpConn) Read(buf []byte) (int, error) {
	in, open := <-c.input
	if !open {
		return 0, io.EOF
	}
	defer in.Release()
	c.updateActivity()
	return copy(buf, in.Bytes()), nil
}

// Write implements io.Writer.
func (c *udpConn) Write(buf []byte) (int, error) {
	n, err := c.output(buf)
	if err == nil {
		c.updateActivity()
	}
	return n, err
}

func (c *udpConn) Close() error {
	return nil
}

func (c *udpConn) RemoteAddr() net.Addr {
	return c.remote
}

func (c *udpConn) LocalAddr() net.Addr {
	return c.remote
}

func (*udpConn) SetDeadline(time.Time) error {
	return nil
}

func (*udpConn) SetReadDeadline(time.Time) error {
	return nil
}

func (*udpConn) SetWriteDeadline(time.Time) error {
	return nil
}

type udpWorker struct {
	sync.RWMutex

	proxy        proxy.Inbound
	hub          *udp.Hub
	address      net.Address
	port         net.Port
	recvOrigDest bool
	tag          string
	dispatcher   dispatcher.Interface

	ctx        context.Context
	cancel     context.CancelFunc
	activeConn map[net.Destination]*udpConn
}

func (w *udpWorker) getConnection(src net.Destination) (*udpConn, bool) {
	w.Lock()
	defer w.Unlock()

	if conn, found := w.activeConn[src]; found {
		return conn, true
	}

	conn := &udpConn{
		input: make(chan *buf.Buffer, 32),
		output: func(b []byte) (int, error) {
			return w.hub.WriteTo(b, src)
		},
		remote: &net.UDPAddr{
			IP:   src.Address.IP(),
			Port: int(src.Port),
		},
		local: &net.UDPAddr{
			IP:   w.address.IP(),
			Port: int(w.port),
		},
	}
	w.activeConn[src] = conn

	conn.updateActivity()
	return conn, false
}

func (w *udpWorker) callback(b *buf.Buffer, source net.Destination, originalDest net.Destination) {
	conn, existing := w.getConnection(source)
	select {
	case conn.input <- b:
	default:
		b.Release()
	}

	if !existing {
		go func() {
			ctx := w.ctx
			ctx, cancel := context.WithCancel(ctx)
			conn.cancel = cancel
			if originalDest.IsValid() {
				ctx = proxy.ContextWithOriginalTarget(ctx, originalDest)
			}
			if len(w.tag) > 0 {
				ctx = proxy.ContextWithInboundTag(ctx, w.tag)
			}
			ctx = proxy.ContextWithSource(ctx, source)
			ctx = proxy.ContextWithInboundEntryPoint(ctx, net.UDPDestination(w.address, w.port))
			if err := w.proxy.Process(ctx, net.Network_UDP, conn, w.dispatcher); err != nil {
				log.Trace(newError("connection ends").Base(err))
			}
			w.removeConn(source)
			cancel()
		}()
	}
}

func (w *udpWorker) removeConn(src net.Destination) {
	w.Lock()
	delete(w.activeConn, src)
	w.Unlock()
}

func (w *udpWorker) Start() error {
	w.activeConn = make(map[net.Destination]*udpConn)
	ctx, cancel := context.WithCancel(context.Background())
	w.ctx = ctx
	w.cancel = cancel
	h, err := udp.ListenUDP(w.address, w.port, udp.ListenOption{
		Callback:            w.callback,
		ReceiveOriginalDest: w.recvOrigDest,
	})
	if err != nil {
		return err
	}
	go w.monitor()
	w.hub = h
	return nil
}

func (w *udpWorker) Close() {
	if w.hub != nil {
		w.hub.Close()
		w.cancel()
	}
}

func (w *udpWorker) monitor() {
	timer := time.NewTicker(time.Second * 16)
	defer timer.Stop()

	for {
		select {
		case <-w.ctx.Done():
			return
		case <-timer.C:
			nowSec := time.Now().Unix()
			w.Lock()
			for addr, conn := range w.activeConn {
				if nowSec-atomic.LoadInt64(&conn.lastActivityTime) > 8 {
					delete(w.activeConn, addr)
					conn.cancel()
				}
			}
			w.Unlock()
		}
	}
}

func (w *udpWorker) Port() net.Port {
	return w.port
}

func (w *udpWorker) Proxy() proxy.Inbound {
	return w.proxy
}
<commit_msg>fix udp in transparent proxy<commit_after>package inbound

import (
	"context"
	"io"
	"sync"
	"sync/atomic"
	"time"

	"v2ray.com/core/app/dispatcher"
	"v2ray.com/core/app/log"
	"v2ray.com/core/app/proxyman"
	"v2ray.com/core/common/buf"
	"v2ray.com/core/common/net"
	"v2ray.com/core/proxy"
	"v2ray.com/core/transport/internet"
	"v2ray.com/core/transport/internet/tcp"
	"v2ray.com/core/transport/internet/udp"
)

type worker interface {
	Start() error
	Close()
	Port() net.Port
	Proxy() proxy.Inbound
}

type tcpWorker struct {
	address      net.Address
	port         net.Port
	proxy        proxy.Inbound
	stream       *internet.StreamConfig
	recvOrigDest bool
	tag          string
	dispatcher   dispatcher.Interface
	sniffers     []proxyman.KnownProtocols

	ctx    context.Context
	cancel context.CancelFunc
	hub    internet.Listener
}

func (w *tcpWorker) callback(conn internet.Connection) {
	ctx, cancel := context.WithCancel(w.ctx)
	if w.recvOrigDest {
		dest, err := tcp.GetOriginalDestination(conn)
		if err != nil {
			log.Trace(newError("failed to get original destination").Base(err))
		}
		if dest.IsValid() {
			ctx = proxy.ContextWithOriginalTarget(ctx, dest)
		}
	}
	if len(w.tag) > 0 {
		ctx = proxy.ContextWithInboundTag(ctx, w.tag)
	}
	ctx = proxy.ContextWithInboundEntryPoint(ctx, net.TCPDestination(w.address, w.port))
	ctx = proxy.ContextWithSource(ctx, net.DestinationFromAddr(conn.RemoteAddr()))
	if len(w.sniffers) > 0 {
		ctx = proxyman.ContextWithProtocolSniffers(ctx, w.sniffers)
	}
	if err := w.proxy.Process(ctx, net.Network_TCP, conn, w.dispatcher); err != nil {
		log.Trace(newError("connection ends").Base(err))
	}
	cancel()
	conn.Close()
}

func (w *tcpWorker) Proxy() proxy.Inbound {
	return w.proxy
}

func (w *tcpWorker) Start() error {
	ctx, cancel := context.WithCancel(context.Background())
	w.ctx = ctx
	w.cancel = cancel
	ctx = internet.ContextWithStreamSettings(ctx, w.stream)
	conns := make(chan internet.Connection, 16)
	hub, err := internet.ListenTCP(ctx, w.address, w.port, conns)
	if err != nil {
		return newError("failed to listen TCP on ", w.port).Base(err)
	}
	go w.handleConnections(conns)
	w.hub = hub
	return nil
}

func (w *tcpWorker) handleConnections(conns <-chan internet.Connection) {
	for {
		select {
		case <-w.ctx.Done():
			w.hub.Close()
		L:
			for {
				select {
				case conn := <-conns:
					conn.Close()
				default:
					break L
				}
			}
			return
		case conn := <-conns:
			go w.callback(conn)
		}
	}
}

func (w *tcpWorker) Close() {
	if w.hub != nil {
		w.cancel()
	}
}

func (w *tcpWorker) Port() net.Port {
	return w.port
}

type udpConn struct {
	lastActivityTime int64 // in seconds
	input            chan *buf.Buffer
	output           func([]byte) (int, error)
	remote           net.Addr
	local            net.Addr
	cancel           context.CancelFunc
}

func (c *udpConn) updateActivity() {
	atomic.StoreInt64(&c.lastActivityTime, time.Now().Unix())
}

func (c *udpConn) Read(buf []byte) (int, error) {
	in, open := <-c.input
	if !open {
		return 0, io.EOF
	}
	defer in.Release()
	c.updateActivity()
	return copy(buf, in.Bytes()), nil
}

// Write implements io.Writer.
func (c *udpConn) Write(buf []byte) (int, error) {
	n, err := c.output(buf)
	if err == nil {
		c.updateActivity()
	}
	return n, err
}

func (c *udpConn) Close() error {
	return nil
}

func (c *udpConn) RemoteAddr() net.Addr {
	return c.remote
}

func (c *udpConn) LocalAddr() net.Addr {
	return c.remote
}

func (*udpConn) SetDeadline(time.Time) error {
	return nil
}

func (*udpConn) SetReadDeadline(time.Time) error {
	return nil
}

func (*udpConn) SetWriteDeadline(time.Time) error {
	return nil
}

type connId struct {
	src  net.Destination
	dest net.Destination
}

type udpWorker struct {
	sync.RWMutex

	proxy        proxy.Inbound
	hub          *udp.Hub
	address      net.Address
	port         net.Port
	recvOrigDest bool
	tag          string
	dispatcher   dispatcher.Interface

	ctx        context.Context
	cancel     context.CancelFunc
	activeConn map[connId]*udpConn
}

func (w *udpWorker) getConnection(id connId) (*udpConn, bool) {
	w.Lock()
	defer w.Unlock()

	if conn, found := w.activeConn[id]; found {
		return conn, true
	}

	conn := &udpConn{
		input: make(chan *buf.Buffer, 32),
		output: func(b []byte) (int, error) {
			return w.hub.WriteTo(b, id.src)
		},
		remote: &net.UDPAddr{
			IP:   id.src.Address.IP(),
			Port: int(id.src.Port),
		},
		local: &net.UDPAddr{
			IP:   w.address.IP(),
			Port: int(w.port),
		},
	}
	w.activeConn[id] = conn

	conn.updateActivity()
	return conn, false
}

func (w *udpWorker) callback(b *buf.Buffer, source net.Destination, originalDest net.Destination) {
	id := connId{
		src:  source,
		dest: originalDest,
	}
	conn, existing := w.getConnection(id)
	select {
	case conn.input <- b:
	default:
		b.Release()
	}

	if !existing {
		go func() {
			ctx := w.ctx
			ctx, cancel := context.WithCancel(ctx)
			conn.cancel = cancel
			if originalDest.IsValid() {
				ctx = proxy.ContextWithOriginalTarget(ctx, originalDest)
			}
			if len(w.tag) > 0 {
				ctx = proxy.ContextWithInboundTag(ctx, w.tag)
			}
			ctx = proxy.ContextWithSource(ctx, source)
			ctx = proxy.ContextWithInboundEntryPoint(ctx, net.UDPDestination(w.address, w.port))
			if err := w.proxy.Process(ctx, net.Network_UDP, conn, w.dispatcher); err != nil {
				log.Trace(newError("connection ends").Base(err))
			}
			w.removeConn(id)
			cancel()
		}()
	}
}

func (w *udpWorker) removeConn(id connId) {
	w.Lock()
	delete(w.activeConn, id)
	w.Unlock()
}

func (w *udpWorker) Start() error {
	w.activeConn = make(map[connId]*udpConn, 16)
	ctx, cancel := context.WithCancel(context.Background())
	w.ctx = ctx
	w.cancel = cancel
	h, err := udp.ListenUDP(w.address, w.port, udp.ListenOption{
		Callback:            w.callback,
		ReceiveOriginalDest: w.recvOrigDest,
	})
	if err != nil {
		return err
	}
	go w.monitor()
	w.hub = h
	return nil
}

func (w *udpWorker) Close() {
	if w.hub != nil {
		w.hub.Close()
		w.cancel()
	}
}

func (w *udpWorker) monitor() {
	timer := time.NewTicker(time.Second * 16)
	defer timer.Stop()

	for {
		select {
		case <-w.ctx.Done():
			return
		case <-timer.C:
			nowSec := time.Now().Unix()
			w.Lock()
			for addr, conn := range w.activeConn {
				if nowSec-atomic.LoadInt64(&conn.lastActivityTime) > 8 {
					delete(w.activeConn, addr)
					conn.cancel()
				}
			}
			w.Unlock()
		}
	}
}

func (w *udpWorker) Port() net.Port {
	return w.port
}

func (w *udpWorker) Proxy() proxy.Inbound {
	return w.proxy
}
<|endoftext|>


<commit_before>// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package storage

import (
	"context"
	"errors"
	"fmt"
	"io"
	"net/url"
	"os"

	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/setting"
)

var (
	// ErrURLNotSupported represents url is not supported
	ErrURLNotSupported = errors.New("url method not supported")
)

// ErrInvalidConfiguration is called when there is invalid configuration for a storage
type ErrInvalidConfiguration struct {
	cfg interface{}
	err error
}

func (err ErrInvalidConfiguration) Error() string {
	if err.err != nil {
		return fmt.Sprintf("Invalid Configuration Argument: %v: Error: %v", err.cfg, err.err)
	}
	return fmt.Sprintf("Invalid Configuration Argument: %v", err.cfg)
}

// IsErrInvalidConfiguration checks if an error is an ErrInvalidConfiguration
func IsErrInvalidConfiguration(err error) bool {
	_, ok := err.(ErrInvalidConfiguration)
	return ok
}

// Type is a type of Storage
type Type string

// NewStorageFunc is a function that creates a storage
type NewStorageFunc func(ctx context.Context, cfg interface{}) (ObjectStorage, error)

var storageMap = map[Type]NewStorageFunc{}

// RegisterStorageType registers a provided storage type with a function to create it
func RegisterStorageType(typ Type, fn func(ctx context.Context, cfg interface{}) (ObjectStorage, error)) {
	storageMap[typ] = fn
}

// Object represents the object on the storage
type Object interface {
	io.ReadCloser
	io.Seeker
	Stat() (os.FileInfo, error)
}

// ObjectStorage represents an object storage to handle a bucket and files
type ObjectStorage interface {
	Open(path string) (Object, error)
	// Save store a object, if size is unknown set -1
	Save(path string, r io.Reader, size int64) (int64, error)
	Stat(path string) (os.FileInfo, error)
	Delete(path string) error
	URL(path, name string) (*url.URL, error)
	IterateObjects(func(path string, obj Object) error) error
}

// Copy copies a file from source ObjectStorage to dest ObjectStorage
func Copy(dstStorage ObjectStorage, dstPath string, srcStorage ObjectStorage, srcPath string) (int64, error) {
	f, err := srcStorage.Open(srcPath)
	if err != nil {
		return 0, err
	}
	defer f.Close()

	size := int64(-1)
	fsinfo, err := f.Stat()
	if err == nil {
		size = fsinfo.Size()
	}

	return dstStorage.Save(dstPath, f, size)
}

// Clean delete all the objects in this storage
func Clean(storage ObjectStorage) error {
	return storage.IterateObjects(func(path string, obj Object) error {
		return storage.Delete(path)
	})
}

// SaveFrom saves data to the ObjectStorage with path p from the callback
func SaveFrom(objStorage ObjectStorage, p string, callback func(w io.Writer) error) error {
	pr, pw := io.Pipe()
	defer pr.Close()
	go func() {
		defer pw.Close()
		if err := callback(pw); err != nil {
			_ = pw.CloseWithError(err)
		}
	}()

	_, err := objStorage.Save(p, pr, -1)
	return err
}

var (
	// Attachments represents attachments storage
	Attachments ObjectStorage

	// LFS represents lfs storage
	LFS ObjectStorage

	// Avatars represents user avatars storage
	Avatars ObjectStorage
	// RepoAvatars represents repository avatars storage
	RepoAvatars ObjectStorage

	// RepoArchives represents repository archives storage
	RepoArchives ObjectStorage
)

// Init init the stoarge
func Init() error {
	if err := initAttachments(); err != nil {
		return err
	}

	if err := initAvatars(); err != nil {
		return err
	}

	if err := initRepoAvatars(); err != nil {
		return err
	}

	if err := initLFS(); err != nil {
		return err
	}

	return initRepoArchives()
}

// NewStorage takes a storage type and some config and returns an ObjectStorage or an error
func NewStorage(typStr string, cfg interface{}) (ObjectStorage, error) {
	if len(typStr) == 0 {
		typStr = string(LocalStorageType)
	}
	fn, ok := storageMap[Type(typStr)]
	if !ok {
		return nil, fmt.Errorf("Unsupported storage type: %s", typStr)
	}

	return fn(context.Background(), cfg)
}

func initAvatars() (err error) {
	log.Info("Initialising Avatar storage with type: %s", setting.Avatar.Storage.Type)
	Avatars, err = NewStorage(setting.Avatar.Storage.Type, &setting.Avatar.Storage)
	return
}

func initAttachments() (err error) {
	log.Info("Initialising Attachment storage with type: %s", setting.Attachment.Storage.Type)
	Attachments, err = NewStorage(setting.Attachment.Storage.Type, &setting.Attachment.Storage)
	return
}

func initLFS() (err error) {
	log.Info("Initialising LFS storage with type: %s", setting.LFS.Storage.Type)
	LFS, err = NewStorage(setting.LFS.Storage.Type, &setting.LFS.Storage)
	return
}

func initRepoAvatars() (err error) {
	log.Info("Initialising Repository Avatar storage with type: %s", setting.RepoAvatar.Storage.Type)
	RepoAvatars, err = NewStorage(setting.RepoAvatar.Storage.Type, &setting.RepoAvatar.Storage)
	return
}

func initRepoArchives() (err error) {
	log.Info("Initialising Repository Archive storage with type: %s", setting.RepoArchive.Storage.Type)
	RepoArchives, err = NewStorage(setting.RepoArchive.Storage.Type, &setting.RepoArchive.Storage)
	return
}
<commit_msg>Close storage objects before cleaning (#16934)<commit_after>// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package storage

import (
	"context"
	"errors"
	"fmt"
	"io"
	"net/url"
	"os"

	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/setting"
)

var (
	// ErrURLNotSupported represents url is not supported
	ErrURLNotSupported = errors.New("url method not supported")
)

// ErrInvalidConfiguration is called when there is invalid configuration for a storage
type ErrInvalidConfiguration struct {
	cfg interface{}
	err error
}

func (err ErrInvalidConfiguration) Error() string {
	if err.err != nil {
		return fmt.Sprintf("Invalid Configuration Argument: %v: Error: %v", err.cfg, err.err)
	}
	return fmt.Sprintf("Invalid Configuration Argument: %v", err.cfg)
}

// IsErrInvalidConfiguration checks if an error is an ErrInvalidConfiguration
func IsErrInvalidConfiguration(err error) bool {
	_, ok := err.(ErrInvalidConfiguration)
	return ok
}

// Type is a type of Storage
type Type string

// NewStorageFunc is a function that creates a storage
type NewStorageFunc func(ctx context.Context, cfg interface{}) (ObjectStorage, error)

var storageMap = map[Type]NewStorageFunc{}

// RegisterStorageType registers a provided storage type with a function to create it
func RegisterStorageType(typ Type, fn func(ctx context.Context, cfg interface{}) (ObjectStorage, error)) {
	storageMap[typ] = fn
}

// Object represents the object on the storage
type Object interface {
	io.ReadCloser
	io.Seeker
	Stat() (os.FileInfo, error)
}

// ObjectStorage represents an object storage to handle a bucket and files
type ObjectStorage interface {
	Open(path string) (Object, error)
	// Save store a object, if size is unknown set -1
	Save(path string, r io.Reader, size int64) (int64, error)
	Stat(path string) (os.FileInfo, error)
	Delete(path string) error
	URL(path, name string) (*url.URL, error)
	IterateObjects(func(path string, obj Object) error) error
}

// Copy copies a file from source ObjectStorage to dest ObjectStorage
func Copy(dstStorage ObjectStorage, dstPath string, srcStorage ObjectStorage, srcPath string) (int64, error) {
	f, err := srcStorage.Open(srcPath)
	if err != nil {
		return 0, err
	}
	defer f.Close()

	size := int64(-1)
	fsinfo, err := f.Stat()
	if err == nil {
		size = fsinfo.Size()
	}

	return dstStorage.Save(dstPath, f, size)
}

// Clean delete all the objects in this storage
func Clean(storage ObjectStorage) error {
	return storage.IterateObjects(func(path string, obj Object) error {
		_ = obj.Close()
		return storage.Delete(path)
	})
}

// SaveFrom saves data to the ObjectStorage with path p from the callback
func SaveFrom(objStorage ObjectStorage, p string, callback func(w io.Writer) error) error {
	pr, pw := io.Pipe()
	defer pr.Close()
	go func() {
		defer pw.Close()
		if err := callback(pw); err != nil {
			_ = pw.CloseWithError(err)
		}
	}()

	_, err := objStorage.Save(p, pr, -1)
	return err
}

var (
	// Attachments represents attachments storage
	Attachments ObjectStorage

	// LFS represents lfs storage
	LFS ObjectStorage

	// Avatars represents user avatars storage
	Avatars ObjectStorage
	// RepoAvatars represents repository avatars storage
	RepoAvatars ObjectStorage

	// RepoArchives represents repository archives storage
	RepoArchives ObjectStorage
)

// Init init the stoarge
func Init() error {
	if err := initAttachments(); err != nil {
		return err
	}

	if err := initAvatars(); err != nil {
		return err
	}

	if err := initRepoAvatars(); err != nil {
		return err
	}

	if err := initLFS(); err != nil {
		return err
	}

	return initRepoArchives()
}

// NewStorage takes a storage type and some config and returns an ObjectStorage or an error
func NewStorage(typStr string, cfg interface{}) (ObjectStorage, error) {
	if len(typStr) == 0 {
		typStr = string(LocalStorageType)
	}
	fn, ok := storageMap[Type(typStr)]
	if !ok {
		return nil, fmt.Errorf("Unsupported storage type: %s", typStr)
	}

	return fn(context.Background(), cfg)
}

func initAvatars() (err error) {
	log.Info("Initialising Avatar storage with type: %s", setting.Avatar.Storage.Type)
	Avatars, err = NewStorage(setting.Avatar.Storage.Type, &setting.Avatar.Storage)
	return
}

func initAttachments() (err error) {
	log.Info("Initialising Attachment storage with type: %s", setting.Attachment.Storage.Type)
	Attachments, err = NewStorage(setting.Attachment.Storage.Type, &setting.Attachment.Storage)
	return
}

func initLFS() (err error) {
	log.Info("Initialising LFS storage with type: %s", setting.LFS.Storage.Type)
	LFS, err = NewStorage(setting.LFS.Storage.Type, &setting.LFS.Storage)
	return
}

func initRepoAvatars() (err error) {
	log.Info("Initialising Repository Avatar storage with type: %s", setting.RepoAvatar.Storage.Type)
	RepoAvatars, err = NewStorage(setting.RepoAvatar.Storage.Type, &setting.RepoAvatar.Storage)
	return
}

func initRepoArchives() (err error) {
	log.Info("Initialising Repository Archive storage with type: %s", setting.RepoArchive.Storage.Type)
	RepoArchives, err = NewStorage(setting.RepoArchive.Storage.Type, &setting.RepoArchive.Storage)
	return
}
<|endoftext|>


<commit_before>//----------------------------------------
//
// Copyright © ying32. All Rights Reserved.
//
// Licensed under Apache License 2.0
//
//----------------------------------------

// +build !windows
// +build cgo

package vcl

// #cgo darwin CFLAGS: -mmacosx-version-min=10.5 -DMACOSX_DEPLOYMENT_TARGET=10.5
// #cgo darwin LDFLAGS: -mmacosx-version-min=10.7
//
// extern void* doEventCallbackProc(void* f, void* args, long argcount);
// static void* doGetEventCallbackAddr() {
//    return &doEventCallbackProc;
// }
//
// extern void* doMessageCallbackProc(void* f, void* msg);
// static void* doGetMessageCallbackAddr() {
//    return &doMessageCallbackProc;
// }
//
// extern void* doThreadSyncCallbackProc();
// static void* doGetThreadSyncCallbackAddr() {
//    return &doThreadSyncCallbackProc;
// }
import "C"

import (
	"unsafe"
)

//export doEventCallbackProc
func doEventCallbackProc(f unsafe.Pointer, args unsafe.Pointer, argcount C.long) unsafe.Pointer {
	eventCallbackProc(uintptr(f), uintptr(args), int(argcount))
	return nullptr
}

//export doMessageCallbackProc
func doMessageCallbackProc(f unsafe.Pointer, msg unsafe.Pointer) unsafe.Pointer {
	messageCallbackProc(uintptr(f), uintptr(msg))
	return nullptr
}

//export doThreadSyncCallbackProc
func doThreadSyncCallbackProc() unsafe.Pointer {
	threadSyncCallbackProc()
	return nullptr
}

var (
	eventCallback      = uintptr(C.doGetEventCallbackAddr())
	messageCallback    = uintptr(C.doGetMessageCallbackAddr())
	threadSyncCallback = uintptr(C.doGetThreadSyncCallbackAddr())
)
<commit_msg>Update LDFLAGS<commit_after>//----------------------------------------
//
// Copyright © ying32. All Rights Reserved.
//
// Licensed under Apache License 2.0
//
//----------------------------------------

// +build !windows
// +build cgo

package vcl

// #cgo darwin CFLAGS: -mmacosx-version-min=10.5 -DMACOSX_DEPLOYMENT_TARGET=10.5
// #cgo darwin LDFLAGS: -mmacosx-version-min=10.5
//
// extern void* doEventCallbackProc(void* f, void* args, long argcount);
// static void* doGetEventCallbackAddr() {
//    return &doEventCallbackProc;
// }
//
// extern void* doMessageCallbackProc(void* f, void* msg);
// static void* doGetMessageCallbackAddr() {
//    return &doMessageCallbackProc;
// }
//
// extern void* doThreadSyncCallbackProc();
// static void* doGetThreadSyncCallbackAddr() {
//    return &doThreadSyncCallbackProc;
// }
import "C"

import (
	"unsafe"
)

//export doEventCallbackProc
func doEventCallbackProc(f unsafe.Pointer, args unsafe.Pointer, argcount C.long) unsafe.Pointer {
	eventCallbackProc(uintptr(f), uintptr(args), int(argcount))
	return nullptr
}

//export doMessageCallbackProc
func doMessageCallbackProc(f unsafe.Pointer, msg unsafe.Pointer) unsafe.Pointer {
	messageCallbackProc(uintptr(f), uintptr(msg))
	return nullptr
}

//export doThreadSyncCallbackProc
func doThreadSyncCallbackProc() unsafe.Pointer {
	threadSyncCallbackProc()
	return nullptr
}

var (
	eventCallback      = uintptr(C.doGetEventCallbackAddr())
	messageCallback    = uintptr(C.doGetMessageCallbackAddr())
	threadSyncCallback = uintptr(C.doGetThreadSyncCallbackAddr())
)
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"crypto/x509"
	"encoding/pem"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"strings"

	"github.com/pkg/sftp"
	"golang.org/x/crypto/ssh"
)

//catapult [-keyfile=... [-passphrase=..] | -password=.. ] user@server:port

var password string
var passphrase string
var keyfile string

func usage() {
	fmt.Fprintf(os.Stderr, "catapult <flags> user@server:port\n")
	fmt.Fprintln(os.Stderr, "")
	fmt.Fprintln(os.Stderr, "Note: flags must come before connection string")
}

func list(conn *ssh.Client, args []string) {
	if len(args) != 1 {
		fmt.Fprintln(os.Stderr, "USAGE: list/[pattern] dir")
		return
	}

	p, m := path.Split(args[0])

	if m == "" {
		m = "*"
	}

	c, err := sftp.NewClient(conn)
	if err != nil {
		panic(err)
	}
	defer c.Close()

	files, err := c.ReadDir(p)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}
	for _, file := range files {
		name := file.Name()
		matched, _ := path.Match(m, name)
		if matched {
			fmt.Println(path.Join(p, name))
		}
	}
}

func get(client *sftp.Client, src string, dest string) error {
	srcFile, err := client.Open(src)
	if err != nil {
		return err
	}
	defer srcFile.Close()

	destFile, err := os.Create(dest) //note, will truncate existing
	if err != nil {
		return err
	}
	defer destFile.Close()

	bytes, err := srcFile.WriteTo(destFile)
	if err != nil {
		return err
	}

	fmt.Printf("GET: %s %s %dbytes\n", src, dest, bytes)
	return nil
}

func exists(name string, alts []string) (bool, error) {
	for _, a := range alts {
		full := path.Join(a, name)
		_, err := os.Stat(full)
		if !(os.IsNotExist(err)) {
			return true, err
		}
	}
	return false, nil
}

func gets(conn *ssh.Client, args []string) {
	if len(args) < 2 {
		fmt.Fprintln(os.Stderr, "USAGE: gets from/[pattern] to [alts]")
		fmt.Fprintln(os.Stderr, "       note that from must end in a / unless it has a match pattern")
		return
	}

	frm, m := path.Split(args[0])
	local := args[1]
	alts := args[1:] //include local in this to simplify logic

	ls, err := os.Stat(local)
	if err != nil || !(ls.IsDir()) {
		fmt.Fprintln(os.Stderr, "local (to) directory doesn't exist")
		return
	}

	if m == "" {
		m = "*"
	}

	c, err := sftp.NewClient(conn)
	if err != nil {
		panic(err)
	}
	defer c.Close()

	files, err := c.ReadDir(frm)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}
	for _, file := range files {
		name := file.Name()
		matched, _ := path.Match(m, name)
		if matched {
			done, err := exists(name, alts)
			if err != nil {
				fmt.Fprintln(os.Stderr, "Failed to check existence: ", name, err)
				continue //move on to next file
			}
			if done {
				//already have this file
				continue
			}
			src := path.Join(frm, name)
			dest := path.Join(local, name)
			err = get(c, src, dest)
			if err != nil {
				fmt.Fprintln(os.Stderr, "Failed get the file: ", name)
			}
		}
	}
}

func put(client *sftp.Client, src string, dest string) error {
	srcFile, err := os.Open(src)
	if err != nil {
		return err
	}
	defer srcFile.Close()

	destFile, err := client.Create(dest) //note, will truncate existing
	if err != nil {
		return err
	}
	defer destFile.Close()

	bytes, err := io.Copy(destFile, srcFile)
	if err != nil {
		return err
	}

	fmt.Printf("PUT: %s %s %dbytes\n", src, dest, bytes)
	return nil
}

func puts(conn *ssh.Client, args []string) {
	if len(args) != 3 {
		fmt.Fprintln(os.Stderr, "USAGE: puts outbox/[pattern] to sentbox")
		fmt.Fprintln(os.Stderr, "       note that outbox must end in a / unless it has a match pattern")
		return
	}

	frm, m := path.Split(args[0])
	remote := args[1]
	sent := args[2]

	if m == "" {
		m = "*"
	}

	ls, err := os.Stat(sent)
	if err != nil || !(ls.IsDir()) {
		fmt.Fprintln(os.Stderr, "sentbox directory doesn't exist")
		return
	}

	c, err := sftp.NewClient(conn)
	if err != nil {
		panic(err)
	}
	defer c.Close()

	files, err := ioutil.ReadDir(frm)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}
	for _, file := range files {
		name := file.Name()
		matched, _ := path.Match(m, name)
		if matched {
			src := path.Join(frm, name)
			dest := path.Join(remote, name)
			archive := path.Join(sent, name)
			err = put(c, src, dest)
			if err != nil {
				fmt.Fprintln(os.Stderr, "Failed to send: ", name, err)
				continue //move on to next file
			}
			err = os.Rename(src, archive)
			if err != nil {
				fmt.Fprintln(os.Stderr, "Failed to move file to sent: ", name)
			}
		}
	}
}

func init() {
	flag.StringVar(&password, "password", "", "password for sftp connection")
	flag.StringVar(&passphrase, "passphrase", "", "passphrase for keyfile")
	flag.StringVar(&keyfile, "keyfile", "", "keyfile path")
}

func encryptedBlock(block *pem.Block) bool {
	return strings.Contains(block.Headers["Proc-Type"], "ENCRYPTED")
}

func ParsePrivateKey(file string, passphrase string) (interface{}, error) {
	pemBytes, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}

	block, _ := pem.Decode(pemBytes)
	if block == nil {
		return nil, errors.New("ssh: no key found in keyfile")
	}

	if encryptedBlock(block) {
		bytes, err := x509.DecryptPEMBlock(block, []byte(passphrase))
		if err != nil || bytes == nil {
			return nil, errors.New("ssh: could not decrypt keyfile")
		}

		key, err := x509.ParsePKCS8PrivateKey(bytes)
		if err == nil {
			return key, nil
		}

		key, err = x509.ParsePKCS1PrivateKey(bytes)
		if err == nil {
			return key, nil
		}

		return nil, errors.New("ssh: key decryption failed")

	}

	return ssh.ParseRawPrivateKey(pemBytes)
}

func main() {
	flag.Parse()

	if flag.NArg() != 1 {
		usage()
		os.Exit(2)
	}

	connection := strings.SplitN(flag.Arg(0), "@", 2)
	if len(connection) != 2 {
		usage()
		os.Exit(2)
	}
	username := connection[0]
	address := connection[1]

	var auths []ssh.AuthMethod

	if keyfile != "" {
		key, err := ParsePrivateKey(keyfile, passphrase)
		if err != nil {
			panic(err)
		}
		signer, err := ssh.NewSignerFromKey(key)
		if err != nil {
			panic(err)
		}
		auths = append(auths, ssh.PublicKeys(signer))
	}

	if password != "" {
		auths = append(auths, ssh.Password(password))
	}

	if password == "" && keyfile == "" {
		fmt.Fprintln(os.Stderr, "Need password or keyfile")
	}

	config := &ssh.ClientConfig{
		User: username,
		Auth: auths,
	}

	hasPort, err := regexp.MatchString(".+:\\d+", address)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(2)
	}
	if !hasPort {
		address = address + ":22"
	}

	client, err := ssh.Dial("tcp", address, config)
	if err != nil {
		fmt.Fprintln(os.Stderr, "Failed to connect to server")
		fmt.Fprintln(os.Stderr, err)
		os.Exit(2)
	}
	defer client.Close()

	//split strings with escape of \ for spaces in arguments
	r := regexp.MustCompile("(\\\\.|[^\\s])+")
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		input := scanner.Text()
		args := r.FindAllString(input, -1)
		switch cmd := args[0]; cmd {
		case "gets":
			gets(client, args[1:])
		case "puts":
			puts(client, args[1:])
		case "list":
			list(client, args[1:])
		default:
			fmt.Println("Unknown command: %s", cmd)
		}
	}
}
<commit_msg>added clean command<commit_after>package main

import (
	"bufio"
	"crypto/x509"
	"encoding/pem"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"strings"

	"github.com/pkg/sftp"
	"golang.org/x/crypto/ssh"
)

//catapult [-keyfile=... [-passphrase=..] | -password=.. ] user@server:port

var password string
var passphrase string
var keyfile string

func usage() {
	fmt.Fprintf(os.Stderr, "catapult <flags> user@server:port\n")
	fmt.Fprintln(os.Stderr, "")
	fmt.Fprintln(os.Stderr, "Note: flags must come before connection string")
}

func list(conn *ssh.Client, args []string) {
	if len(args) != 1 {
		fmt.Fprintln(os.Stderr, "USAGE: list/[pattern] dir")
		return
	}

	p, m := path.Split(args[0])

	if m == "" {
		m = "*"
	}

	c, err := sftp.NewClient(conn)
	if err != nil {
		panic(err)
	}
	defer c.Close()

	files, err := c.ReadDir(p)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}
	for _, file := range files {
		name := file.Name()
		matched, _ := path.Match(m, name)
		if matched {
			fmt.Println(path.Join(p, name))
		}
	}
}

func get(client *sftp.Client, src string, dest string) error {
	srcFile, err := client.Open(src)
	if err != nil {
		return err
	}
	defer srcFile.Close()

	destFile, err := os.Create(dest) //note, will truncate existing
	if err != nil {
		return err
	}
	defer destFile.Close()

	bytes, err := srcFile.WriteTo(destFile)
	if err != nil {
		return err
	}

	fmt.Printf("GET: %s %s %dbytes\n", src, dest, bytes)
	return nil
}

func exists(name string, alts []string) (bool, error) {
	for _, a := range alts {
		full := path.Join(a, name)
		_, err := os.Stat(full)
		if !(os.IsNotExist(err)) {
			return true, err
		}
	}
	return false, nil
}

func gets(conn *ssh.Client, args []string) {
	if len(args) < 2 {
		fmt.Fprintln(os.Stderr, "USAGE: gets from/[pattern] to [alts]")
		fmt.Fprintln(os.Stderr, "       note that from must end in a / unless it has a match pattern")
		return
	}

	frm, m := path.Split(args[0])
	local := args[1]
	alts := args[1:] //include local in this to simplify logic

	ls, err := os.Stat(local)
	if err != nil || !(ls.IsDir()) {
		fmt.Fprintln(os.Stderr, "local (to) directory doesn't exist")
		return
	}

	if m == "" {
		m = "*"
	}

	c, err := sftp.NewClient(conn)
	if err != nil {
		panic(err)
	}
	defer c.Close()

	files, err := c.ReadDir(frm)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}
	for _, file := range files {
		name := file.Name()
		matched, _ := path.Match(m, name)
		if matched {
			done, err := exists(name, alts)
			if err != nil {
				fmt.Fprintln(os.Stderr, "Failed to check existence: ", name, err)
				continue //move on to next file
			}
			if done {
				//already have this file
				continue
			}
			src := path.Join(frm, name)
			dest := path.Join(local, name)
			err = get(c, src, dest)
			if err != nil {
				fmt.Fprintln(os.Stderr, "Failed get the file: ", name)
			}
		}
	}
}

func put(client *sftp.Client, src string, dest string) error {
	srcFile, err := os.Open(src)
	if err != nil {
		return err
	}
	defer srcFile.Close()

	destFile, err := client.Create(dest) //note, will truncate existing
	if err != nil {
		return err
	}
	defer destFile.Close()

	bytes, err := io.Copy(destFile, srcFile)
	if err != nil {
		return err
	}

	fmt.Printf("PUT: %s %s %dbytes\n", src, dest, bytes)
	return nil
}

func puts(conn *ssh.Client, args []string) {
	if len(args) != 3 {
		fmt.Fprintln(os.Stderr, "USAGE: puts outbox/[pattern] to sentbox")
		fmt.Fprintln(os.Stderr, "       note that outbox must end in a / unless it has a match pattern")
		return
	}

	frm, m := path.Split(args[0])
	remote := args[1]
	sent := args[2]

	if m == "" {
		m = "*"
	}

	ls, err := os.Stat(sent)
	if err != nil || !(ls.IsDir()) {
		fmt.Fprintln(os.Stderr, "sentbox directory doesn't exist")
		return
	}

	c, err := sftp.NewClient(conn)
	if err != nil {
		panic(err)
	}
	defer c.Close()

	files, err := ioutil.ReadDir(frm)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}
	for _, file := range files {
		name := file.Name()
		matched, _ := path.Match(m, name)
		if matched {
			src := path.Join(frm, name)
			dest := path.Join(remote, name)
			archive := path.Join(sent, name)
			err = put(c, src, dest)
			if err != nil {
				fmt.Fprintln(os.Stderr, "Failed to send: ", name, err)
				continue //move on to next file
			}
			err = os.Rename(src, archive)
			if err != nil {
				fmt.Fprintln(os.Stderr, "Failed to move file to sent: ", name)
			}
		}
	}
}

func remove(client *sftp.Client, dest string) error {
	destFile, err := client.Create(dest) //note, will truncate existing
	if err != nil {
		return err
	}
	defer destFile.Close()

	bytes, err := io.Copy(destFile, srcFile)
	if err != nil {
		return err
	}

	fmt.Printf("PUT: %s %s %dbytes\n", src, dest, bytes)
	return nil
}

//Remove files that exist
func clean(conn *ssh.Client, args []string) {
	if len(args) != 2 {
		fmt.Fprintln(os.Stderr, "USAGE: clean remote_path/ local_processed_path/[pattern]")
		fmt.Fprintln(os.Stderr, "       processed path must end in a / or pattern")
		return
	}

	remote := args[0]
	local,lm := path.Split(args[1])

	if m == "" {
		m = "*"
	}

	if lm == "" {
		lm = "*"
	}

	ls, err := os.Stat(local)
	if err != nil || !(ls.IsDir()) {
		fmt.Fprintln(os.Stderr, "processed directory doesn't exist")
		return
	}

	c, err := sftp.NewClient(conn)
	if err != nil {
		panic(err)
	}
	defer c.Close()

	files, err := ioutil.ReadDir(local)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}
	for _, file := range files {
		name := file.Name()
		matched, _ := path.Match(m, name)
		if matched {
			src := path.Join(local, name)
			dest := path.Join(remote, name)
			err = c.removeFile(dest)
			if err != nil {
				fmt.Fprintln(os.Stderr, "Failed to remove remote file: ", dest, err)
				continue //move on to next file ??? Do we want to remove processed
			}
			err = os.Remove(src)
			if err != nil {
				fmt.Fprintln(os.Stderr, "Failed to remove local file: ", src, err)
			}
		}
	}
}

func init() {
	flag.StringVar(&password, "password", "", "password for sftp connection")
	flag.StringVar(&passphrase, "passphrase", "", "passphrase for keyfile")
	flag.StringVar(&keyfile, "keyfile", "", "keyfile path")
	flag.StringVar(&fingerprint, "", "fingerprint of server")
}

func encryptedBlock(block *pem.Block) bool {
	return strings.Contains(block.Headers["Proc-Type"], "ENCRYPTED")
}

func ParsePrivateKey(file string, passphrase string) (interface{}, error) {
	pemBytes, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}

	block, _ := pem.Decode(pemBytes)
	if block == nil {
		return nil, errors.New("ssh: no key found in keyfile")
	}

	if encryptedBlock(block) {
		bytes, err := x509.DecryptPEMBlock(block, []byte(passphrase))
		if err != nil || bytes == nil {
			return nil, errors.New("ssh: could not decrypt keyfile")
		}

		key, err := x509.ParsePKCS8PrivateKey(bytes)
		if err == nil {
			return key, nil
		}

		key, err = x509.ParsePKCS1PrivateKey(bytes)
		if err == nil {
			return key, nil
		}

		return nil, errors.New("ssh: key decryption failed")

	}

	return ssh.ParseRawPrivateKey(pemBytes)
}

func main() {
	flag.Parse()

	if flag.NArg() != 1 {
		usage()
		os.Exit(2)
	}

	connection := strings.SplitN(flag.Arg(0), "@", 2)
	if len(connection) != 2 {
		usage()
		os.Exit(2)
	}
	username := connection[0]
	address := connection[1]

	var auths []ssh.AuthMethod

	if keyfile != "" {
		key, err := ParsePrivateKey(keyfile, passphrase)
		if err != nil {
			panic(err)
		}
		signer, err := ssh.NewSignerFromKey(key)
		if err != nil {
			panic(err)
		}
		auths = append(auths, ssh.PublicKeys(signer))
	}

	if password != "" {
		auths = append(auths, ssh.Password(password))
	}

	if password == "" && keyfile == "" {
		fmt.Fprintln(os.Stderr, "Need password or keyfile")
	}

	config := &ssh.ClientConfig{
		User: username,
		Auth: auths,
	}

	hasPort, err := regexp.MatchString(".+:\\d+", address)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(2)
	}
	if !hasPort {
		address = address + ":22"
	}

	client, err := ssh.Dial("tcp", address, config)
	if err != nil {
		fmt.Fprintln(os.Stderr, "Failed to connect to server")
		fmt.Fprintln(os.Stderr, err)
		os.Exit(2)
	}
	defer client.Close()

	//split strings with escape of \ for spaces in arguments
	r := regexp.MustCompile("(\\\\.|[^\\s])+")
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		input := scanner.Text()
		args := r.FindAllString(input, -1)
		switch cmd := args[0]; cmd {
		case "gets":
			gets(client, args[1:])
		case "puts":
			puts(client, args[1:])
		case "list":
			list(client, args[1:])
		case "clean":
			fmt.Println("TODO: implement clean")
		default:
			fmt.Println("Unknown command: %s", cmd)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd.

// +build ignore

// Command print fetches and prints package.
//
// Usage: go run print.go importPath
package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"strings"

	"github.com/golang/gddo/gosrc"
)

var (
	etag    = flag.String("etag", "", "Etag")
	local   = flag.String("local", "", "Get package from local workspace.")
	present = flag.Bool("present", false, "Get presentation.")
)

func main() {
	flag.Parse()
	if len(flag.Args()) != 1 {
		log.Fatal("Usage: go run print.go importPath")
	}
	if *present {
		printPresentation(flag.Args()[0])
	} else {
		printDir(flag.Args()[0])
	}
}

func printDir(path string) {
	if *local != "" {
		gosrc.SetLocalDevMode(*local)
	}
	dir, err := gosrc.Get(http.DefaultClient, path, *etag)
	if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
		log.Fatalf("redirect to %s", e.Redirect)
	} else if err != nil {
		log.Fatalf("%+v", err)
	}

	fmt.Println("ImportPath    ", dir.ImportPath)
	fmt.Println("ResovledPath  ", dir.ResolvedPath)
	fmt.Println("ProjectRoot   ", dir.ProjectRoot)
	fmt.Println("ProjectName   ", dir.ProjectName)
	fmt.Println("ProjectURL    ", dir.ProjectURL)
	fmt.Println("VCS           ", dir.VCS)
	fmt.Println("Etag          ", dir.Etag)
	fmt.Println("BrowseURL     ", dir.BrowseURL)
	fmt.Println("Subdirectories", strings.Join(dir.Subdirectories, ", "))
	fmt.Println("LineFmt       ", dir.LineFmt)
	fmt.Println("Files:")
	for _, file := range dir.Files {
		fmt.Printf("%30s %5d %s\n", file.Name, len(file.Data), file.BrowseURL)
	}
}

func printPresentation(path string) {
	pres, err := gosrc.GetPresentation(http.DefaultClient, path)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s\n", pres.Files[pres.Filename])
	for name, data := range pres.Files {
		if name != pres.Filename {
			fmt.Printf("---------- %s ----------\n%s\n", name, data)
		}
	}
}
<commit_msg>gosrc: Setup GitHub authentication for print command<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd.

// +build ignore

// Command print fetches and prints package.
//
// Usage: go run print.go importPath
package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"strings"

	"github.com/golang/gddo/gosrc"
	"github.com/golang/gddo/httputil"
)

var (
	etag    = flag.String("etag", "", "Etag")
	local   = flag.String("local", "", "Get package from local workspace.")
	present = flag.Bool("present", false, "Get presentation.")
)

func main() {
	flag.Parse()
	if len(flag.Args()) != 1 {
		log.Fatal("Usage: go run print.go importPath")
	}
	if *present {
		printPresentation(flag.Args()[0])
	} else {
		printDir(flag.Args()[0])
	}
}

func printDir(path string) {
	if *local != "" {
		gosrc.SetLocalDevMode(*local)
	}
	c := &http.Client{
		Transport: httputil.NewAuthTransport(&http.Transport{}),
	}
	dir, err := gosrc.Get(c, path, *etag)
	if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
		log.Fatalf("redirect to %s", e.Redirect)
	} else if err != nil {
		log.Fatalf("%+v", err)
	}

	fmt.Println("ImportPath    ", dir.ImportPath)
	fmt.Println("ResovledPath  ", dir.ResolvedPath)
	fmt.Println("ProjectRoot   ", dir.ProjectRoot)
	fmt.Println("ProjectName   ", dir.ProjectName)
	fmt.Println("ProjectURL    ", dir.ProjectURL)
	fmt.Println("VCS           ", dir.VCS)
	fmt.Println("Etag          ", dir.Etag)
	fmt.Println("BrowseURL     ", dir.BrowseURL)
	fmt.Println("Subdirectories", strings.Join(dir.Subdirectories, ", "))
	fmt.Println("LineFmt       ", dir.LineFmt)
	fmt.Println("Files:")
	for _, file := range dir.Files {
		fmt.Printf("%30s %5d %s\n", file.Name, len(file.Data), file.BrowseURL)
	}
}

func printPresentation(path string) {
	pres, err := gosrc.GetPresentation(http.DefaultClient, path)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s\n", pres.Files[pres.Filename])
	for name, data := range pres.Files {
		if name != pres.Filename {
			fmt.Printf("---------- %s ----------\n%s\n", name, data)
		}
	}
}
<|endoftext|>


<commit_before>package gaestatic

import (
	"strings"
	"fmt"
	"strconv"
	"net/http"
	"text/template"
	"bytes"
	"net/url"
)

const PLIST_TEMPLATE string = `<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
	<dict>
		<key>items</key>
		<array>
			<dict>
				<key>assets</key>
				<array>
					{{if .IpaUrl}}
					<dict>
						<key>kind</key>
						<string>software-package</string>
						<key>url</key>
						<string>{{.IpaUrl}}</string>
					</dict>
					{{end}}
					{{if .DisplayImageUrl}}
					<dict>
						<key>kind</key>
						<string>display-image</string>
						<key>url</key>
						<string>{{.DisplayImageUrl}}</string>
					</dict>
					{{end}}
					{{if .FullSizeImageUrl}}
					<dict>
						<key>kind</key>
						<string>full-size-image</string>
						<key>url</key>
						<string>{{.FullSizeImageUrl}}</string>
					</dict>
					{{end}}
				</array>
				<key>metadata</key>
				<dict>
					{{if .BundleIdentifer}}
					<key>bundle-identifier</key>
					<string>{{.BundleIdentifer}}</string>
					{{end}}
					{{if .BundleVersion}}
					<key>bundle-version</key>
					<string>{{.BundleVersion}}</string>
					<key>kind</key>
					{{end}}
					{{if .Title}}
					<string>software</string>
					<key>title</key>
					<string>{{.Title}}</string>
					{{end}}
				</dict>
			</dict>
		</array>
	</dict>
</plist>
`

type PlistTemplateParams struct {
	// eg. https://example.com/apps/ios/sample.ipa
	IpaUrl string
	// eg. https://example.com/apps/ios/image.png
	DisplayImageUrl string
	// eg. https://example.com/apps/ios/full-image.png
	FullSizeImageUrl string
	// eg. com.example.sample
	BundleIdentifer string
	// eg. 1.0
	BundleVersion string
	// eg. Sample App
	Title string
}

/**
 * Dynamic Plist Handler
 */
func plistHandler(w http.ResponseWriter, r *http.Request) bool {

	isDone := true

	config := GetAppConfig()
	if config == nil {
		// Internal Server Errror
		w.WriteHeader(500)
		w.Write([]byte("No Config"))
		return isDone
	}

	filePath := strings.Replace(r.URL.Path, config.PlistDir, "", 1)
	tmp := strings.SplitN(filePath, "/", 2)
	if len(tmp) < 2 {
		// Bad Request
		w.WriteHeader(400)
		w.Write([]byte("invalid path #1"))
		return isDone
	}
	bundleIdentifer := tmp[0]
	if strings.Contains(tmp[1], "..") {
		// Bad Request
		w.WriteHeader(400)
		w.Write([]byte("invalid path #2"))
		return isDone
	}

	ipaUrl, _ := url.Parse(r.RequestURI)
	ipaUrl.Host = r.Host
	ipaUrl.Scheme = "https"
	ipaUrl.Path = "/" + tmp[1]

	params := PlistTemplateParams{}
	// http://example.com/{filePath}/{bundleId}/{IpaPath}?title={title}&version={bundleVersion}
	params.Title = r.URL.Query().Get("title")
	params.BundleVersion = r.URL.Query().Get("version")
	params.BundleIdentifer = bundleIdentifer
	params.IpaUrl = ipaUrl.String()

	tmpl, err := template.New("plist").Parse(PLIST_TEMPLATE)

	if err != nil {
		// Not Found
		w.WriteHeader(501)
		w.Write([]byte(fmt.Sprintf("plist template is invalid.")))
		return isDone
	}

	writer := new(bytes.Buffer)
	err	= tmpl.Execute(writer, params)

	var contentLength string
	if err != nil {
		// Forbidden : サイズ取得失敗
		w.WriteHeader(403)
		w.Write([]byte(fmt.Sprintf("plist params is invalid.")))
		return isDone
	} else {
		contentLength = strconv.FormatInt(int64(writer.Len()), 10)
	}
	contentLength = contentLength + "bytes"

	contentType := GetContentType("_.plist")
	if contentType != "" {
		w.Header().Set("Content-Type", contentType)
	}
	w.Write(writer.Bytes())
	isDone = true
	return isDone
}
<commit_msg>dynamic plist : snapshot<commit_after>package gaestatic

import (
	"strings"
	"fmt"
	"strconv"
	"net/http"
	"text/template"
	"bytes"
	"net/url"
)

const PLIST_TEMPLATE string = `<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
	<dict>
		<key>items</key>
		<array>
			<dict>
				<key>assets</key>
				<array>
					{{if .IpaUrl}}
					<dict>
						<key>kind</key>
						<string>software-package</string>
						<key>url</key>
						<string>{{.IpaUrl}}</string>
					</dict>
					{{end}}
					{{if .DisplayImageUrl}}
					<dict>
						<key>kind</key>
						<string>display-image</string>
						<key>url</key>
						<string>{{.DisplayImageUrl}}</string>
					</dict>
					{{end}}
					{{if .FullSizeImageUrl}}
					<dict>
						<key>kind</key>
						<string>full-size-image</string>
						<key>url</key>
						<string>{{.FullSizeImageUrl}}</string>
					</dict>
					{{end}}
				</array>
				<key>metadata</key>
				<dict>
					{{if .BundleIdentifer}}
					<key>bundle-identifier</key>
					<string>{{.BundleIdentifer}}</string>
					{{end}}
					{{if .BundleVersion}}
					<key>bundle-version</key>
					<string>{{.BundleVersion}}</string>
					<key>kind</key>
					{{end}}
					{{if .Title}}
					<string>software</string>
					<key>title</key>
					<string>{{.Title}}</string>
					{{end}}
				</dict>
			</dict>
		</array>
	</dict>
</plist>
`

type PlistTemplateParams struct {
	// eg. https://example.com/apps/ios/sample.ipa
	IpaUrl string
	// eg. https://example.com/apps/ios/image.png
	DisplayImageUrl string
	// eg. https://example.com/apps/ios/full-image.png
	FullSizeImageUrl string
	// eg. com.example.sample
	BundleIdentifer string
	// eg. 1.0
	BundleVersion string
	// eg. Sample App
	Title string
}

/**
 * Dynamic Plist Handler
 */
func plistHandler(w http.ResponseWriter, r *http.Request) bool {

	isDone := true

	config := GetAppConfig()
	if config == nil {
		// Internal Server Errror
		w.WriteHeader(500)
		w.Write([]byte("No Config"))
		return isDone
	}

	filePath := strings.Replace(r.URL.Path, config.PlistDir, "", 1)
	tmp := strings.SplitN(filePath, "/", 2)
	if len(tmp) < 2 {
		// Bad Request
		w.WriteHeader(400)
		w.Write([]byte("invalid path #1"))
		return isDone
	}
	bundleIdentifer := tmp[0]
	if strings.Contains(tmp[1], "..") {
		// Bad Request
		w.WriteHeader(400)
		w.Write([]byte("invalid path #2"))
		return isDone
	}

	ipaUrl, _ := url.Parse(r.RequestURI)
	ipaUrl.Host = r.Host
	ipaUrl.Scheme = r.URL.Scheme
	ipaUrl.Path = "/" + tmp[1]

	params := PlistTemplateParams{}
	// http://example.com/{filePath}/{bundleId}/{IpaPath}?title={title}&version={bundleVersion}
	params.Title = r.URL.Query().Get("title")
	params.BundleVersion = r.URL.Query().Get("version")
	params.BundleIdentifer = bundleIdentifer
	params.IpaUrl = ipaUrl.String()

	tmpl, err := template.New("plist").Parse(PLIST_TEMPLATE)

	if err != nil {
		// Not Found
		w.WriteHeader(501)
		w.Write([]byte(fmt.Sprintf("plist template is invalid.")))
		return isDone
	}

	writer := new(bytes.Buffer)
	err	= tmpl.Execute(writer, params)

	var contentLength string
	if err != nil {
		// Forbidden : サイズ取得失敗
		w.WriteHeader(403)
		w.Write([]byte(fmt.Sprintf("plist params is invalid.")))
		return isDone
	} else {
		contentLength = strconv.FormatInt(int64(writer.Len()), 10)
	}
	contentLength = contentLength + "bytes"

	contentType := GetContentType("_.plist")
	if contentType != "" {
		w.Header().Set("Content-Type", contentType)
	}
	w.Write(writer.Bytes())
	isDone = true
	return isDone
}
<|endoftext|>


<commit_before>package graph

// Graph describes the methods of graph operations.
// It assumes that the identifier of a Vertex is string and unique.
// And weight values is float64.
type Graph interface {
	// GetVertices returns a map of all vertices.
	GetVertices() map[string]bool

	// FindVertex returns true if the vertex already
	// exists in the graph.
	FindVertex(vtx string) bool

	// AddVertex adds a vertex to a graph, and returns false
	// if the vertex already existed in the graph.
	AddVertex(vtx string) bool

	// DeleteVertex deletes a vertex from a graph.
	// It returns true if it got deleted.
	// And false if it didn't get deleted.
	DeleteVertex(vtx string) bool

	// AddEdge adds an edge from vtx1 to vtx2 with the weight.
	AddEdge(vtx1, vtx2 string, weight float64) error

	// ReplaceEdge replaces an edge from vtx1 to vtx2 with the weight.
	ReplaceEdge(vtx1, vtx2 string, weight float64) error

	// DeleteEdge deletes an edge from vtx1 to vtx2.
	DeleteEdge(vtx1, vtx2 string) error

	// GetWeight returns the weight from vtx1 to vtx2.
	GetWeight(vtx1, vtx2 string) (float64, error)

	// GetParents returns the map of parent vertices.
	// (Vertices that comes to the argument vertex.)
	GetParents(vtx string) (map[string]bool, error)

	// GetChildren returns the map of child vertices.
	// (Vertices that goes out of the argument vertex.)
	GetChildren(vtx string) (map[string]bool, error)

	// String describes the Graph.
	String() string
}
<commit_msg>graph: add Init to Graph interface<commit_after>package graph

// Graph describes the methods of graph operations.
// It assumes that the identifier of a Vertex is string and unique.
// And weight values is float64.
type Graph interface {
	// Init initializes a Graph.
	Init()

	// GetVertices returns a map of all vertices.
	GetVertices() map[string]bool

	// FindVertex returns true if the vertex already
	// exists in the graph.
	FindVertex(vtx string) bool

	// AddVertex adds a vertex to a graph, and returns false
	// if the vertex already existed in the graph.
	AddVertex(vtx string) bool

	// DeleteVertex deletes a vertex from a graph.
	// It returns true if it got deleted.
	// And false if it didn't get deleted.
	DeleteVertex(vtx string) bool

	// AddEdge adds an edge from vtx1 to vtx2 with the weight.
	AddEdge(vtx1, vtx2 string, weight float64) error

	// ReplaceEdge replaces an edge from vtx1 to vtx2 with the weight.
	ReplaceEdge(vtx1, vtx2 string, weight float64) error

	// DeleteEdge deletes an edge from vtx1 to vtx2.
	DeleteEdge(vtx1, vtx2 string) error

	// GetWeight returns the weight from vtx1 to vtx2.
	GetWeight(vtx1, vtx2 string) (float64, error)

	// GetParents returns the map of parent vertices.
	// (Vertices that comes to the argument vertex.)
	GetParents(vtx string) (map[string]bool, error)

	// GetChildren returns the map of child vertices.
	// (Vertices that goes out of the argument vertex.)
	GetChildren(vtx string) (map[string]bool, error)

	// String describes the Graph.
	String() string
}
<|endoftext|>


<commit_before>package errorsx

import (
	"fmt"
	"runtime/debug"
	"sort"
	"strings"
)

type kvPairsMapType map[interface{}]interface{}

type Error interface {
	Error() string
	Stack() []byte
}

type Err struct {
	err     error
	kvPairs kvPairsMapType
	stack   []byte
}

func (err *Err) Stack() []byte {
	return err.stack
}

func (err *Err) Error() string {
	var s = err.err.Error()
	var kvStrings []string
	for key, val := range err.kvPairs {
		kvStrings = append(kvStrings, fmt.Sprintf("%s=%q", key, val))
	}
	if len(kvStrings) > 0 {
		sort.Slice(kvStrings, func(i, j int) bool {
			return kvStrings[i] < kvStrings[j]
		})
		s += fmt.Sprintf(" [%s]", strings.Join(kvStrings, ", "))
	}
	return s
}

func Errorf(message string, args ...interface{}) Error {
	return &Err{
		fmt.Errorf(message, args...),
		make(kvPairsMapType),
		debug.Stack(),
	}
}

func Wrap(err error, kvPairs ...interface{}) Error {
	if err == nil {
		return nil
	}

	kvPairsMap := make(kvPairsMapType)
	for i := 0; i < len(kvPairs); i = i + 2 {
		k := kvPairs[i]
		v := kvPairs[i+1]
		kvPairsMap[k] = v
	}

	errType, ok := err.(*Err)
	if !ok {
		return &Err{
			err,
			kvPairsMap,
			debug.Stack(),
		}
	}

	// merge in kv map
	for k, v := range kvPairsMap {
		errType.kvPairs[k] = v
	}

	return errType
}

// Cause fetches the underlying cause of the error
// this should be used with errors wrapped from errors.New()
func Cause(err error) error {
	errErr, ok := err.(*Err)
	if ok {
		return Cause(errErr.err)
	}

	return err
}
<commit_msg>deterministic ordering of error kv pairs<commit_after>package errorsx

import (
	"fmt"
	"runtime/debug"
	"sort"
	"strings"
)

type kvPairsMapType map[interface{}]interface{}

type Error interface {
	Error() string
	Stack() []byte
}

type Err struct {
	err     error
	kvPairs kvPairsMapType
	stack   []byte
}

func (err *Err) Stack() []byte {
	return err.stack
}

func (err *Err) Error() string {
	var s = err.err.Error()
	var kvStrings []string
	for key, val := range err.kvPairs {
		kvStrings = append(kvStrings, fmt.Sprintf("%s=%#v", key, val))
	}
	sort.Slice(kvStrings, func(i, j int) bool {
		return kvStrings[i] < kvStrings[j]
	})
	if len(kvStrings) > 0 {
		sort.Slice(kvStrings, func(i, j int) bool {
			return kvStrings[i] < kvStrings[j]
		})
		s += fmt.Sprintf(" [%s]", strings.Join(kvStrings, ", "))
	}
	return s
}

func Errorf(message string, args ...interface{}) Error {
	return &Err{
		fmt.Errorf(message, args...),
		make(kvPairsMapType),
		debug.Stack(),
	}
}

func Wrap(err error, kvPairs ...interface{}) Error {
	if err == nil {
		return nil
	}

	kvPairsMap := make(kvPairsMapType)
	for i := 0; i < len(kvPairs); i = i + 2 {
		k := kvPairs[i]
		v := kvPairs[i+1]
		kvPairsMap[k] = v
	}

	errType, ok := err.(*Err)
	if !ok {
		return &Err{
			err,
			kvPairsMap,
			debug.Stack(),
		}
	}

	// merge in kv map
	for k, v := range kvPairsMap {
		errType.kvPairs[k] = v
	}

	return errType
}

// Cause fetches the underlying cause of the error
// this should be used with errors wrapped from errors.New()
func Cause(err error) error {
	errErr, ok := err.(*Err)
	if ok {
		return Cause(errErr.err)
	}

	return err
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testapi

import (
	"strings"

	"k8s.io/kubernetes/pkg/apis/experimental/latest"
)

// Returns the appropriate path for the given prefix (watch, proxy, redirect, etc), resource, namespace and name.
// For example, this is of the form:
// /experimental/v1/watch/namespaces/foo/pods/pod0 for v1.
func ResourcePathWithPrefix(prefix, resource, namespace, name string) string {
	path := "/experimental/" + latest.GroupOrDie("").Version
	if prefix != "" {
		path = path + "/" + prefix
	}
	if namespace != "" {
		path = path + "/namespaces/" + namespace
	}
	// Resource names are lower case.
	resource = strings.ToLower(resource)
	if resource != "" {
		path = path + "/" + resource
	}
	if name != "" {
		path = path + "/" + name
	}
	return path
}

// Returns the appropriate path for the given resource, namespace and name.
// For example, this is of the form:
// /experimental/v1/namespaces/foo/pods/pod0 for v1.
func ResourcePath(resource, namespace, name string) string {
	return ResourcePathWithPrefix("", resource, namespace, name)
}
<commit_msg>remove pkg/expapi/testapi/testapi.go<commit_after><|endoftext|>


<commit_before>package services

import (
	"encoding/json"
	"errors"
	"fmt"
	"path"

	"github.com/franela/goreq"
	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
	m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models"
)

var IoHelper m.IoUtil = IoUtilImp{}

func ListAllPlugins(repoUrl string) (m.PluginRepo, error) {
	fullUrl := repoUrl + "/repo"
	res, err := goreq.Request{Uri: fullUrl, MaxRedirects: 3}.Do()
	if err != nil {
		return m.PluginRepo{}, err
	}
	if res.StatusCode != 200 {
		return m.PluginRepo{}, fmt.Errorf("Could not access %s statuscode %v", fullUrl, res.StatusCode)
	}

	var resp m.PluginRepo
	err = res.Body.FromJsonTo(&resp)
	if err != nil {
		return m.PluginRepo{}, errors.New("Could not load plugin data")
	}

	return resp, nil
}

func ReadPlugin(pluginDir, pluginName string) (m.InstalledPlugin, error) {
	pluginDataPath := path.Join(pluginDir, pluginName, "plugin.json")
	pluginData, _ := IoHelper.ReadFile(pluginDataPath)

	res := m.InstalledPlugin{}
	json.Unmarshal(pluginData, &res)

	if res.Info.Version == "" {
		res.Info.Version = "0.0.0"
	}

	if res.Id == "" {
		return m.InstalledPlugin{}, errors.New("could not find plugin " + pluginName + " in " + pluginDir)
	}

	return res, nil
}

func GetLocalPlugins(pluginDir string) []m.InstalledPlugin {
	result := make([]m.InstalledPlugin, 0)
	files, _ := IoHelper.ReadDir(pluginDir)
	for _, f := range files {
		res, err := ReadPlugin(pluginDir, f.Name())
		if err == nil {
			result = append(result, res)
		}
	}

	return result
}

func RemoveInstalledPlugin(pluginPath, id string) error {
	logger.Infof("Removing plugin: %v\n", id)
	return IoHelper.RemoveAll(path.Join(pluginPath, id))
}

func GetPlugin(pluginId, repoUrl string) (m.Plugin, error) {
	fullUrl := repoUrl + "/repo/" + pluginId

	res, err := goreq.Request{Uri: fullUrl, MaxRedirects: 3}.Do()
	if err != nil {
		return m.Plugin{}, err
	}
	if res.StatusCode != 200 {
		return m.Plugin{}, fmt.Errorf("Could not access %s statuscode %v", fullUrl, res.StatusCode)
	}

	var resp m.Plugin
	err = res.Body.FromJsonTo(&resp)
	if err != nil {
		return m.Plugin{}, errors.New("Could not load plugin data")
	}

	return resp, nil
}
<commit_msg>feat(cli): adds support for dist/plugin.json location for plugins<commit_after>package services

import (
	"encoding/json"
	"errors"
	"fmt"
	"path"

	"github.com/franela/goreq"
	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
	m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models"
)

var IoHelper m.IoUtil = IoUtilImp{}

func ListAllPlugins(repoUrl string) (m.PluginRepo, error) {
	fullUrl := repoUrl + "/repo"
	res, err := goreq.Request{Uri: fullUrl, MaxRedirects: 3}.Do()
	if err != nil {
		return m.PluginRepo{}, err
	}
	if res.StatusCode != 200 {
		return m.PluginRepo{}, fmt.Errorf("Could not access %s statuscode %v", fullUrl, res.StatusCode)
	}

	var resp m.PluginRepo
	err = res.Body.FromJsonTo(&resp)
	if err != nil {
		return m.PluginRepo{}, errors.New("Could not load plugin data")
	}

	return resp, nil
}

func ReadPlugin(pluginDir, pluginName string) (m.InstalledPlugin, error) {
	distPluginDataPath := path.Join(pluginDir, pluginName, "dist", "plugin.json")

	var data []byte
	var err error
	data, err = IoHelper.ReadFile(distPluginDataPath)

	if err != nil {
		pluginDataPath := path.Join(pluginDir, pluginName, "plugin.json")
		data, err = IoHelper.ReadFile(pluginDataPath)

		if err != nil {
			return m.InstalledPlugin{}, errors.New("Could not find dist/plugin.json or plugin.json on  " + pluginName + " in " + pluginDir)
		}
	}

	res := m.InstalledPlugin{}
	json.Unmarshal(data, &res)

	if res.Info.Version == "" {
		res.Info.Version = "0.0.0"
	}

	if res.Id == "" {
		return m.InstalledPlugin{}, errors.New("could not find plugin " + pluginName + " in " + pluginDir)
	}

	return res, nil
}

func GetLocalPlugins(pluginDir string) []m.InstalledPlugin {
	result := make([]m.InstalledPlugin, 0)
	files, _ := IoHelper.ReadDir(pluginDir)
	for _, f := range files {
		res, err := ReadPlugin(pluginDir, f.Name())
		if err == nil {
			result = append(result, res)
		}
	}

	return result
}

func RemoveInstalledPlugin(pluginPath, id string) error {
	logger.Infof("Removing plugin: %v\n", id)
	return IoHelper.RemoveAll(path.Join(pluginPath, id))
}

func GetPlugin(pluginId, repoUrl string) (m.Plugin, error) {
	fullUrl := repoUrl + "/repo/" + pluginId

	res, err := goreq.Request{Uri: fullUrl, MaxRedirects: 3}.Do()
	if err != nil {
		return m.Plugin{}, err
	}
	if res.StatusCode != 200 {
		return m.Plugin{}, fmt.Errorf("Could not access %s statuscode %v", fullUrl, res.StatusCode)
	}

	var resp m.Plugin
	err = res.Body.FromJsonTo(&resp)
	if err != nil {
		return m.Plugin{}, errors.New("Could not load plugin data")
	}

	return resp, nil
}
<|endoftext|>


<commit_before>/*

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package xgboostjob

import (
	"fmt"
	"github.com/kubeflow/common/job_controller"
	"github.com/kubeflow/common/job_controller/api/v1"
	"github.com/kubeflow/xgboost-operator/pkg/apis/xgboostjob/v1alpha1"
	"github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"sigs.k8s.io/controller-runtime/pkg/event"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
)

// satisfiedExpectations returns true if the required adds/dels for the given job have been observed.
// Add/del counts are established by the controller at sync time, and updated as controllees are observed by the controller
// manager.
func (r *ReconcileXGBoostJob) satisfiedExpectations(xgbJob *v1alpha1.XGBoostJob) bool {
	satisfied := false
	key, err := job_controller.KeyFunc(xgbJob)
	if err != nil {
		utilruntime.HandleError(fmt.Errorf("couldn't get key for job object %#v: %v", xgbJob, err))
		return false
	}
	for rtype := range xgbJob.Spec.XGBReplicaSpecs {
		// Check the expectations of the pods.
		expectationPodsKey := job_controller.GenExpectationPodsKey(key, string(rtype))
		satisfied = satisfied || r.xgbJobController.Expectations.SatisfiedExpectations(expectationPodsKey)
		// Check the expectations of the services.
		expectationServicesKey := job_controller.GenExpectationServicesKey(key, string(rtype))
		satisfied = satisfied || r.xgbJobController.Expectations.SatisfiedExpectations(expectationServicesKey)
	}
	return satisfied
}

// onDependentCreateFunc modify expectations when dependent (pod/service) creation observed.
func onDependentCreateFunc(r reconcile.Reconciler) func(event.CreateEvent) bool {
	return func(e event.CreateEvent) bool {
		xgbr, ok := r.(*ReconcileXGBoostJob)
		if !ok {
			return true
		}
		rtype := e.Meta.GetLabels()[v1.ReplicaTypeLabel]
		if len(rtype) == 0 {
			return false
		}

		logrus.Info("Update on create function ", xgbr.ControllerName(), " create object ", e.Meta.GetName())
		if controllerRef := metav1.GetControllerOf(e.Meta); controllerRef != nil {
			expectKey := job_controller.GenExpectationPodsKey(e.Meta.GetNamespace()+"/"+controllerRef.Name, rtype)
			xgbr.xgbJobController.Expectations.CreationObserved(expectKey)
			return true
		}

		return true
	}
}

// onDependentDeleteFunc modify expectations when dependent (pod/service) deletion observed.
func onDependentDeleteFunc(r reconcile.Reconciler) func(event.DeleteEvent) bool {
	return func(e event.DeleteEvent) bool {
		xgbr, ok := r.(*ReconcileXGBoostJob)
		if !ok {
			return true
		}

		rtype := e.Meta.GetLabels()[v1.ReplicaTypeLabel]
		if len(rtype) == 0 {
			return false
		}

		logrus.Info("Update on deleting function ", xgbr.ControllerName(), " delete object ", e.Meta.GetName())
		if controllerRef := metav1.GetControllerOf(e.Meta); controllerRef != nil {
			expectKey := job_controller.GenExpectationPodsKey(e.Meta.GetNamespace()+"/"+controllerRef.Name, rtype)
			xgbr.xgbJobController.Expectations.DeleteExpectations(expectKey)
			return true
		}

		return true
	}
}
<commit_msg>Set different expectation keys for pod and service (#79)<commit_after>/*

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package xgboostjob

import (
	"fmt"
	"github.com/kubeflow/common/job_controller"
	"github.com/kubeflow/common/job_controller/api/v1"
	"github.com/kubeflow/xgboost-operator/pkg/apis/xgboostjob/v1alpha1"
	"github.com/sirupsen/logrus"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"sigs.k8s.io/controller-runtime/pkg/event"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
)

// satisfiedExpectations returns true if the required adds/dels for the given job have been observed.
// Add/del counts are established by the controller at sync time, and updated as controllees are observed by the controller
// manager.
func (r *ReconcileXGBoostJob) satisfiedExpectations(xgbJob *v1alpha1.XGBoostJob) bool {
	satisfied := false
	key, err := job_controller.KeyFunc(xgbJob)
	if err != nil {
		utilruntime.HandleError(fmt.Errorf("couldn't get key for job object %#v: %v", xgbJob, err))
		return false
	}
	for rtype := range xgbJob.Spec.XGBReplicaSpecs {
		// Check the expectations of the pods.
		expectationPodsKey := job_controller.GenExpectationPodsKey(key, string(rtype))
		satisfied = satisfied || r.xgbJobController.Expectations.SatisfiedExpectations(expectationPodsKey)
		// Check the expectations of the services.
		expectationServicesKey := job_controller.GenExpectationServicesKey(key, string(rtype))
		satisfied = satisfied || r.xgbJobController.Expectations.SatisfiedExpectations(expectationServicesKey)
	}
	return satisfied
}

// onDependentCreateFunc modify expectations when dependent (pod/service) creation observed.
func onDependentCreateFunc(r reconcile.Reconciler) func(event.CreateEvent) bool {
	return func(e event.CreateEvent) bool {
		xgbr, ok := r.(*ReconcileXGBoostJob)
		if !ok {
			return true
		}
		rtype := e.Meta.GetLabels()[v1.ReplicaTypeLabel]
		if len(rtype) == 0 {
			return false
		}

		logrus.Info("Update on create function ", xgbr.ControllerName(), " create object ", e.Meta.GetName())
		if controllerRef := metav1.GetControllerOf(e.Meta); controllerRef != nil {
			var expectKey string
			jobKey := e.Meta.GetNamespace() + "/" + controllerRef.Name
			if _, ok := e.Object.(*corev1.Pod); ok {
				expectKey = job_controller.GenExpectationPodsKey(jobKey, rtype)
			}
			if _, ok := e.Object.(*corev1.Service); ok {
				expectKey = job_controller.GenExpectationServicesKey(e.Meta.GetNamespace()+"/"+controllerRef.Name, rtype)
			}
			xgbr.xgbJobController.Expectations.CreationObserved(expectKey)
			return true
		}

		return true
	}
}

// onDependentDeleteFunc modify expectations when dependent (pod/service) deletion observed.
func onDependentDeleteFunc(r reconcile.Reconciler) func(event.DeleteEvent) bool {
	return func(e event.DeleteEvent) bool {
		xgbr, ok := r.(*ReconcileXGBoostJob)
		if !ok {
			return true
		}

		rtype := e.Meta.GetLabels()[v1.ReplicaTypeLabel]
		if len(rtype) == 0 {
			return false
		}

		logrus.Info("Update on deleting function ", xgbr.ControllerName(), " delete object ", e.Meta.GetName())
		if controllerRef := metav1.GetControllerOf(e.Meta); controllerRef != nil {
			var expectKey string
			jobKey := e.Meta.GetNamespace() + "/" + controllerRef.Name
			if _, ok := e.Object.(*corev1.Pod); ok {
				expectKey = job_controller.GenExpectationPodsKey(jobKey, rtype)
			}
			if _, ok := e.Object.(*corev1.Service); ok {
				expectKey = job_controller.GenExpectationServicesKey(jobKey, rtype)
			}
			xgbr.xgbJobController.Expectations.DeleteExpectations(expectKey)
			return true
		}

		return true
	}
}
<|endoftext|>