<commit_before>package input

import (
	"fmt"
	"sync"

	"github.com/brunoga/go-pipeliner/datatypes"

	deluge "github.com/brunoga/go-deluge"
	base_modules "github.com/brunoga/go-modules"
	pipeliner_modules "github.com/brunoga/go-pipeliner/modules"
)

type DelugeOutputModule struct {
	*base_modules.GenericModule

	inputChannel chan *datatypes.PipelineItem
	quitChannel  chan struct{}

	delugeClient *deluge.Deluge
}

func NewDelugeOutputModule(specificId string) *DelugeOutputModule {
	return &DelugeOutputModule{
		base_modules.NewGenericModule("Deluge Output Module", "1.0.0",
			"deluge", specificId, "pipeliner-output"),
		make(chan *datatypes.PipelineItem),
		make(chan struct{}),
		nil,
	}
}

func (m *DelugeOutputModule) Configure(params *base_modules.ParameterMap) error {
	serverParam, ok := (*params)["server"]
	if !ok || len(serverParam) == 0 {
		return fmt.Errorf("required server parameter not found")
	}

	passwordParam, ok := (*params)["password"]
	if !ok || len(passwordParam) == 0 {
		return fmt.Errorf("required password parameter not found")
	}

	delugeClient, err := deluge.New(serverParam, passwordParam)
	if err != nil {
		return err
	}

	m.delugeClient = delugeClient

	m.SetReady(true)

	return nil
}

func (m *DelugeOutputModule) Parameters() *base_modules.ParameterMap {
	return &base_modules.ParameterMap{
		"server":   "",
		"password": "",
	}
}

func (m *DelugeOutputModule) Duplicate(specificId string) (base_modules.Module, error) {
	duplicate := NewDelugeOutputModule(specificId)
	err := pipeliner_modules.RegisterPipelinerOutputModule(duplicate)
	if err != nil {
		return nil, err
	}

	return duplicate, nil
}

func (m *DelugeOutputModule) GetInputChannel() chan<- *datatypes.PipelineItem {
	return m.inputChannel
}

func (m *DelugeOutputModule) Start(waitGroup *sync.WaitGroup) error {
	if !m.Ready() {
		waitGroup.Done()
		return fmt.Errorf("not ready")
	}

	if m.inputChannel == nil {
		waitGroup.Done()
		return fmt.Errorf("input channel not connected")
	}

	go m.doWork(waitGroup)

	return nil
}

func (m *DelugeOutputModule) Stop() {
	close(m.quitChannel)
	m.quitChannel = make(chan struct{})
}

func (m *DelugeOutputModule) doWork(waitGroup *sync.WaitGroup) {
	defer waitGroup.Done()
L:
	for {
		select {
		case item, ok := <-m.inputChannel:
			if ok {
				m.sendToDeluge(item)
			} else {
				m.inputChannel = nil
				break L
			}
		case <-m.quitChannel:
			break L
		}
	}
}

func (m *DelugeOutputModule) sendToDeluge(item *datatypes.PipelineItem) {
	// Use first URL available.
	// TODO(bga): Allow user to define preferred hosts or URL types (e.g.
	// prefer magnet links).
	torrentUrl, err := item.GetUrl(0)
	if err != nil {
		// TODO(bga): Log error.
		return
	}

	// TODO(bga): Empty configuration for now.
	options := map[string]interface{}{}

	// TODO(bga): Handle errors when addding torrent.
	switch torrentUrl.Scheme {
	case "magnet":
		m.delugeClient.CoreAddTorrentMagnet(
			torrentUrl.String(), options)
	case "http":
		m.delugeClient.CoreAddTorrentUrl(
			torrentUrl.String(), options)
	default:
		// TODO(bga): Add handling of other types.
	}
}

func init() {
	pipeliner_modules.RegisterPipelinerOutputModule(NewDelugeOutputModule(""))
}
<commit_msg>Make DelugeOutputModule use GenericOutputModule.<commit_after>package input

import (
	"fmt"

	"github.com/brunoga/go-pipeliner/datatypes"

	deluge "github.com/brunoga/go-deluge"
	base_modules "github.com/brunoga/go-modules"
	pipeliner_modules "github.com/brunoga/go-pipeliner/modules"
)

type DelugeOutputModule struct {
	*pipeliner_modules.GenericOutputModule

	delugeClient *deluge.Deluge
}

func NewDelugeOutputModule(specificId string) *DelugeOutputModule {
	delugeOutputModule := &DelugeOutputModule{
		pipeliner_modules.NewGenericOutputModule("Deluge Output Module",
			"1.0.0", "deluge", specificId, nil),
		nil,
	}
	delugeOutputModule.SetConsumerFunc(delugeOutputModule.sendItemToDeluge)

	return delugeOutputModule
}

func (m *DelugeOutputModule) Configure(
	params *base_modules.ParameterMap) error {
	serverParam, ok := (*params)["server"]
	if !ok || len(serverParam) == 0 {
		return fmt.Errorf("required server parameter not found")
	}

	passwordParam, ok := (*params)["password"]
	if !ok || len(passwordParam) == 0 {
		return fmt.Errorf("required password parameter not found")
	}

	delugeClient, err := deluge.New(serverParam, passwordParam)
	if err != nil {
		return err
	}

	m.delugeClient = delugeClient

	m.SetReady(true)

	return nil
}

func (m *DelugeOutputModule) Parameters() *base_modules.ParameterMap {
	return &base_modules.ParameterMap{
		"server":   "",
		"password": "",
	}
}

func (m *DelugeOutputModule) Duplicate(specificId string) (base_modules.Module,
	error) {
	duplicate := NewDelugeOutputModule(specificId)
	err := pipeliner_modules.RegisterPipelinerOutputModule(duplicate)
	if err != nil {
		return nil, err
	}

	return duplicate, nil
}

func (m *DelugeOutputModule) sendItemToDeluge(
	item *datatypes.PipelineItem) bool {
	// Use first URL available.
	// TODO(bga): Allow user to define preferred hosts or URL types (e.g.
	// prefer magnet links).
	torrentUrl, err := item.GetUrl(0)
	if err != nil {
		// TODO(bga): Log error.
		return false
	}

	// TODO(bga): Empty configuration for now.
	options := map[string]interface{}{}

	// TODO(bga): Handle errors when addding torrent.
	switch torrentUrl.Scheme {
	case "magnet":
		m.delugeClient.CoreAddTorrentMagnet(
			torrentUrl.String(), options)
	case "http":
		m.delugeClient.CoreAddTorrentUrl(
			torrentUrl.String(), options)
	default:
		// TODO(bga): Add handling of other types.
		return false
	}

	return true
}

func init() {
	pipeliner_modules.RegisterPipelinerOutputModule(
		NewDelugeOutputModule(""))
}
<|endoftext|>


<commit_before>package renter

import (
	"io"
	"os"
	"sync"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/renter/hostdb"
	"github.com/NebulousLabs/Sia/types"
)

const (
	hostTimeout = 15 * time.Second
)

// When a file contract is within 'renewThreshold' blocks of expiring, the renter
// will attempt to renew the contract.
var renewThreshold = func() types.BlockHeight {
	switch build.Release {
	case "testing":
		return 20
	case "dev":
		return 200
	default:
		return 2000
	}
}()

// repair attempts to repair a file chunk by uploading its pieces to more
// hosts.
func (f *file) repair(chunkIndex uint64, missingPieces []uint64, r io.ReaderAt, hosts []hostdb.Uploader) error {
	// read chunk data and encode
	chunk := make([]byte, f.chunkSize())
	_, err := r.ReadAt(chunk, int64(chunkIndex*f.chunkSize()))
	if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
		return err
	}
	pieces, err := f.erasureCode.Encode(chunk)
	if err != nil {
		return err
	}
	// encrypt pieces
	for i := range pieces {
		key := deriveKey(f.masterKey, chunkIndex, uint64(i))
		pieces[i], err = key.EncryptBytes(pieces[i])
		if err != nil {
			return err
		}
	}

	// upload one piece per host
	numPieces := len(missingPieces)
	if len(hosts) < numPieces {
		numPieces = len(hosts)
	}
	var wg sync.WaitGroup
	wg.Add(numPieces)
	for i := 0; i < numPieces; i++ {
		// each goroutine gets a different host, index, and piece, so there
		// are no data race concerns
		pIndex := missingPieces[i]
		go func(host hostdb.Uploader, pieceIndex uint64, piece []byte) {
			defer wg.Done()

			// upload data to host
			offset, err := host.Upload(piece)
			if err != nil {
				return
			}

			// create contract entry, if necessary
			f.mu.Lock()
			defer f.mu.Unlock()
			contract, ok := f.contracts[host.ContractID()]
			if !ok {
				contract = fileContract{
					ID:          host.ContractID(),
					IP:          host.Address(),
					WindowStart: host.EndHeight(),
				}
			}

			// update contract
			contract.Pieces = append(contract.Pieces, pieceData{
				Chunk:  chunkIndex,
				Piece:  pieceIndex,
				Offset: offset,
			})
			f.contracts[host.ContractID()] = contract
		}(hosts[i], pIndex, pieces[pIndex])
	}
	wg.Wait()

	return nil
}

// threadedRepairLoop improves the health of files tracked by the renter by
// reuploading their missing pieces. Multiple repair attempts may be necessary
// before the file reaches full redundancy.
func (r *Renter) threadedRepairLoop() {
	for {
		time.Sleep(5 * time.Second)

		if !r.wallet.Unlocked() {
			continue
		}

		// make copy of repair set under lock
		repairing := make(map[string]trackedFile)
		id := r.mu.RLock()
		for name, meta := range r.tracking {
			repairing[name] = meta
		}
		r.mu.RUnlock(id)

		for name, meta := range repairing {
			r.threadedRepairFile(name, meta)
		}
	}
}

// incompleteChunks returns a map of chunks containing pieces that have not
// been uploaded.
func (f *file) incompleteChunks() map[uint64][]uint64 {
	f.mu.RLock()
	defer f.mu.RUnlock()

	present := make([][]bool, f.numChunks())
	for i := range present {
		present[i] = make([]bool, f.erasureCode.NumPieces())
	}
	for _, fc := range f.contracts {
		for _, p := range fc.Pieces {
			present[p.Chunk][p.Piece] = true
		}
	}

	incomplete := make(map[uint64][]uint64)
	for chunkIndex, pieceBools := range present {
		for pieceIndex, ok := range pieceBools {
			if !ok {
				incomplete[uint64(chunkIndex)] = append(incomplete[uint64(chunkIndex)], uint64(pieceIndex))
			}
		}
	}
	return incomplete
}

// chunkHosts returns the hosts storing the given chunk.
func (f *file) chunkHosts(chunk uint64) []modules.NetAddress {
	f.mu.RLock()
	defer f.mu.RUnlock()

	var old []modules.NetAddress
	for _, fc := range f.contracts {
		for _, p := range fc.Pieces {
			if p.Chunk == chunk {
				old = append(old, fc.IP)
				break
			}
		}
	}
	return old
}

// expiringContracts returns the contracts that will expire soon.
// TODO: what if contract has fully expired?
func (f *file) expiringContracts(height types.BlockHeight) []fileContract {
	f.mu.RLock()
	defer f.mu.RUnlock()

	var expiring []fileContract
	for _, fc := range f.contracts {
		if height >= fc.WindowStart-renewThreshold {
			expiring = append(expiring, fc)
		}
	}
	return expiring
}

// offlineChunks returns the chunks belonging to "offline" hosts -- hosts that
// do not meet uptime requirements. Importantly, only chunks missing more than
// half their redundancy are returned.
func (f *file) offlineChunks(hdb hostDB) map[uint64][]uint64 {
	f.mu.RLock()
	defer f.mu.RUnlock()

	// helper function for determining if a host is offline. A host is
	// considered offline if it is in AllHosts but not ActiveHosts. For good
	// order notation, we covert AllHosts and ActiveHosts to a single map. The
	// map lookup will return true if the host was in ActiveHosts.
	//
	// TODO: once the hostdb is persistent, checking for existence in
	// ActiveHosts should be sufficient.
	hostSet := make(map[modules.NetAddress]bool)
	for _, host := range hdb.AllHosts() {
		hostSet[host.IPAddress] = false
	}
	for _, host := range hdb.ActiveHosts() {
		hostSet[host.IPAddress] = true
	}
	isOffline := func(addr modules.NetAddress) bool {
		active, exists := hostSet[addr]
		return exists && !active
	}

	// mark all pieces belonging to offline hosts.
	offline := make(map[uint64][]uint64)
	for _, fc := range f.contracts {
		if isOffline(fc.IP) {
			for _, p := range fc.Pieces {
				offline[p.Chunk] = append(offline[p.Chunk], p.Piece)
			}
		}
	}
	// filter out chunks missing less than half of their redundancy
	filtered := make(map[uint64][]uint64)
	for chunk, pieces := range offline {
		if len(pieces) > f.erasureCode.NumPieces()/2 {
			filtered[chunk] = pieces
		}
	}
	return filtered
}

// threadedRepairFile repairs and saves an individual file.
func (r *Renter) threadedRepairFile(name string, meta trackedFile) {
	// helper function
	logAndRemove := func(fmt string, args ...interface{}) {
		r.log.Printf(fmt, args...)
		id := r.mu.Lock()
		delete(r.tracking, name)
		r.mu.Unlock(id)
	}

	id := r.mu.RLock()
	f, ok := r.files[name]
	r.mu.RUnlock(id)
	if !ok {
		logAndRemove("removing %v from repair set: no longer tracking that file", name)
		return
	}

	// check for expiration
	height := r.cs.Height()
	if !meta.Renew && meta.EndHeight < height {
		logAndRemove("removing %v from repair set: storage period has ended", name)
		return
	}

	// open file handle
	handle, err := os.Open(meta.RepairPath)
	if err != nil {
		logAndRemove("removing %v from repair set: %v", name, err)
		return
	}
	defer handle.Close()

	// repair incomplete chunks
	if badChunks := f.incompleteChunks(); len(badChunks) != 0 {
		r.log.Printf("repairing %v chunks of %v", len(badChunks), f.name)
		var duration types.BlockHeight
		if meta.Renew {
			duration = defaultDuration
		} else {
			duration = meta.EndHeight - height
		}
		r.repairChunks(f, handle, badChunks, duration)
	}

	// repair offline chunks
	if badChunks := f.offlineChunks(r.hostDB); len(badChunks) != 0 {
		r.log.Printf("reuploading %v offline chunks of %v", len(badChunks), f.name)
		var duration types.BlockHeight
		if meta.Renew {
			duration = defaultDuration
		} else {
			duration = meta.EndHeight - height
		}
		r.repairChunks(f, handle, badChunks, duration)
	}

	// renew expiring contracts
	if meta.Renew {
		if badContracts := f.expiringContracts(height); len(badContracts) != 0 {
			r.log.Printf("renewing %v contracts of %v", len(badContracts), f.name)
			newHeight := height + defaultDuration
			r.renewContracts(f, badContracts, newHeight)
		}
	}

	// save the repaired file data
	err = r.saveFile(f)
	if err != nil {
		// definitely bad, but we probably shouldn't delete from the
		// repair set if this happens
		r.log.Printf("failed to save repaired file %v: %v", name, err)
	}
}

// repairChunks uploads missing chunks of f to new hosts.
func (r *Renter) repairChunks(f *file, handle io.ReaderAt, chunks map[uint64][]uint64, duration types.BlockHeight) {
	// create host pool
	contractSize := (f.pieceSize + crypto.TwofishOverhead) * f.numChunks() // each host gets one piece of each chunk
	pool, err := r.hostDB.NewPool(contractSize, duration)
	if err != nil {
		r.log.Printf("failed to repair %v: %v", f.name, err)
		return
	}
	defer pool.Close() // heh

	for chunk, pieces := range chunks {
		// determine host set
		old := f.chunkHosts(chunk)
		hosts := pool.UniqueHosts(f.erasureCode.NumPieces()-len(old), old)
		if len(hosts) == 0 {
			r.log.Printf("aborting repair of %v: not enough hosts", f.name)
			return
		}
		// upload to new hosts
		err = f.repair(chunk, pieces, handle, hosts)
		if err != nil {
			r.log.Printf("aborting repair of %v: %v", f.name, err)
			return
		}
	}
}

// renewContracts renews each of the supplied contracts, replacing their entry
// in f with the new contract.
func (r *Renter) renewContracts(f *file, contracts []fileContract, newHeight types.BlockHeight) {
	for _, c := range contracts {
		newID, err := r.hostDB.Renew(c.ID, newHeight)
		if err != nil {
			r.log.Printf("failed to renew contract %v: %v", c.ID, err)
			continue
		}
		f.mu.Lock()
		f.contracts[newID] = fileContract{
			ID:          newID,
			IP:          c.IP,
			Pieces:      c.Pieces,
			WindowStart: newHeight,
		}
		// need to delete the old contract; otherwise f.expiringContracts
		// will continue to return it
		delete(f.contracts, c.ID)
		f.mu.Unlock()
	}
}
<commit_msg>prevent rename race with saveFile<commit_after>package renter

import (
	"io"
	"os"
	"sync"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/renter/hostdb"
	"github.com/NebulousLabs/Sia/types"
)

const (
	hostTimeout = 15 * time.Second
)

// When a file contract is within 'renewThreshold' blocks of expiring, the renter
// will attempt to renew the contract.
var renewThreshold = func() types.BlockHeight {
	switch build.Release {
	case "testing":
		return 20
	case "dev":
		return 200
	default:
		return 2000
	}
}()

// repair attempts to repair a file chunk by uploading its pieces to more
// hosts.
func (f *file) repair(chunkIndex uint64, missingPieces []uint64, r io.ReaderAt, hosts []hostdb.Uploader) error {
	// read chunk data and encode
	chunk := make([]byte, f.chunkSize())
	_, err := r.ReadAt(chunk, int64(chunkIndex*f.chunkSize()))
	if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
		return err
	}
	pieces, err := f.erasureCode.Encode(chunk)
	if err != nil {
		return err
	}
	// encrypt pieces
	for i := range pieces {
		key := deriveKey(f.masterKey, chunkIndex, uint64(i))
		pieces[i], err = key.EncryptBytes(pieces[i])
		if err != nil {
			return err
		}
	}

	// upload one piece per host
	numPieces := len(missingPieces)
	if len(hosts) < numPieces {
		numPieces = len(hosts)
	}
	var wg sync.WaitGroup
	wg.Add(numPieces)
	for i := 0; i < numPieces; i++ {
		// each goroutine gets a different host, index, and piece, so there
		// are no data race concerns
		pIndex := missingPieces[i]
		go func(host hostdb.Uploader, pieceIndex uint64, piece []byte) {
			defer wg.Done()

			// upload data to host
			offset, err := host.Upload(piece)
			if err != nil {
				return
			}

			// create contract entry, if necessary
			f.mu.Lock()
			defer f.mu.Unlock()
			contract, ok := f.contracts[host.ContractID()]
			if !ok {
				contract = fileContract{
					ID:          host.ContractID(),
					IP:          host.Address(),
					WindowStart: host.EndHeight(),
				}
			}

			// update contract
			contract.Pieces = append(contract.Pieces, pieceData{
				Chunk:  chunkIndex,
				Piece:  pieceIndex,
				Offset: offset,
			})
			f.contracts[host.ContractID()] = contract
		}(hosts[i], pIndex, pieces[pIndex])
	}
	wg.Wait()

	return nil
}

// threadedRepairLoop improves the health of files tracked by the renter by
// reuploading their missing pieces. Multiple repair attempts may be necessary
// before the file reaches full redundancy.
func (r *Renter) threadedRepairLoop() {
	for {
		time.Sleep(5 * time.Second)

		if !r.wallet.Unlocked() {
			continue
		}

		// make copy of repair set under lock
		repairing := make(map[string]trackedFile)
		id := r.mu.RLock()
		for name, meta := range r.tracking {
			repairing[name] = meta
		}
		r.mu.RUnlock(id)

		for name, meta := range repairing {
			r.threadedRepairFile(name, meta)
		}
	}
}

// incompleteChunks returns a map of chunks containing pieces that have not
// been uploaded.
func (f *file) incompleteChunks() map[uint64][]uint64 {
	f.mu.RLock()
	defer f.mu.RUnlock()

	present := make([][]bool, f.numChunks())
	for i := range present {
		present[i] = make([]bool, f.erasureCode.NumPieces())
	}
	for _, fc := range f.contracts {
		for _, p := range fc.Pieces {
			present[p.Chunk][p.Piece] = true
		}
	}

	incomplete := make(map[uint64][]uint64)
	for chunkIndex, pieceBools := range present {
		for pieceIndex, ok := range pieceBools {
			if !ok {
				incomplete[uint64(chunkIndex)] = append(incomplete[uint64(chunkIndex)], uint64(pieceIndex))
			}
		}
	}
	return incomplete
}

// chunkHosts returns the hosts storing the given chunk.
func (f *file) chunkHosts(chunk uint64) []modules.NetAddress {
	f.mu.RLock()
	defer f.mu.RUnlock()

	var old []modules.NetAddress
	for _, fc := range f.contracts {
		for _, p := range fc.Pieces {
			if p.Chunk == chunk {
				old = append(old, fc.IP)
				break
			}
		}
	}
	return old
}

// expiringContracts returns the contracts that will expire soon.
// TODO: what if contract has fully expired?
func (f *file) expiringContracts(height types.BlockHeight) []fileContract {
	f.mu.RLock()
	defer f.mu.RUnlock()

	var expiring []fileContract
	for _, fc := range f.contracts {
		if height >= fc.WindowStart-renewThreshold {
			expiring = append(expiring, fc)
		}
	}
	return expiring
}

// offlineChunks returns the chunks belonging to "offline" hosts -- hosts that
// do not meet uptime requirements. Importantly, only chunks missing more than
// half their redundancy are returned.
func (f *file) offlineChunks(hdb hostDB) map[uint64][]uint64 {
	f.mu.RLock()
	defer f.mu.RUnlock()

	// helper function for determining if a host is offline. A host is
	// considered offline if it is in AllHosts but not ActiveHosts. For good
	// order notation, we covert AllHosts and ActiveHosts to a single map. The
	// map lookup will return true if the host was in ActiveHosts.
	//
	// TODO: once the hostdb is persistent, checking for existence in
	// ActiveHosts should be sufficient.
	hostSet := make(map[modules.NetAddress]bool)
	for _, host := range hdb.AllHosts() {
		hostSet[host.IPAddress] = false
	}
	for _, host := range hdb.ActiveHosts() {
		hostSet[host.IPAddress] = true
	}
	isOffline := func(addr modules.NetAddress) bool {
		active, exists := hostSet[addr]
		return exists && !active
	}

	// mark all pieces belonging to offline hosts.
	offline := make(map[uint64][]uint64)
	for _, fc := range f.contracts {
		if isOffline(fc.IP) {
			for _, p := range fc.Pieces {
				offline[p.Chunk] = append(offline[p.Chunk], p.Piece)
			}
		}
	}
	// filter out chunks missing less than half of their redundancy
	filtered := make(map[uint64][]uint64)
	for chunk, pieces := range offline {
		if len(pieces) > f.erasureCode.NumPieces()/2 {
			filtered[chunk] = pieces
		}
	}
	return filtered
}

// threadedRepairFile repairs and saves an individual file.
func (r *Renter) threadedRepairFile(name string, meta trackedFile) {
	// helper function
	logAndRemove := func(fmt string, args ...interface{}) {
		r.log.Printf(fmt, args...)
		id := r.mu.Lock()
		delete(r.tracking, name)
		r.mu.Unlock(id)
	}

	id := r.mu.RLock()
	f, ok := r.files[name]
	r.mu.RUnlock(id)
	if !ok {
		logAndRemove("removing %v from repair set: no longer tracking that file", name)
		return
	}

	// check for expiration
	height := r.cs.Height()
	if !meta.Renew && meta.EndHeight < height {
		logAndRemove("removing %v from repair set: storage period has ended", name)
		return
	}

	// open file handle
	handle, err := os.Open(meta.RepairPath)
	if err != nil {
		logAndRemove("removing %v from repair set: %v", name, err)
		return
	}
	defer handle.Close()

	// repair incomplete chunks
	if badChunks := f.incompleteChunks(); len(badChunks) != 0 {
		r.log.Printf("repairing %v chunks of %v", len(badChunks), f.name)
		var duration types.BlockHeight
		if meta.Renew {
			duration = defaultDuration
		} else {
			duration = meta.EndHeight - height
		}
		r.repairChunks(f, handle, badChunks, duration)
	}

	// repair offline chunks
	if badChunks := f.offlineChunks(r.hostDB); len(badChunks) != 0 {
		r.log.Printf("reuploading %v offline chunks of %v", len(badChunks), f.name)
		var duration types.BlockHeight
		if meta.Renew {
			duration = defaultDuration
		} else {
			duration = meta.EndHeight - height
		}
		r.repairChunks(f, handle, badChunks, duration)
	}

	// renew expiring contracts
	if meta.Renew {
		if badContracts := f.expiringContracts(height); len(badContracts) != 0 {
			r.log.Printf("renewing %v contracts of %v", len(badContracts), f.name)
			newHeight := height + defaultDuration
			r.renewContracts(f, badContracts, newHeight)
		}
	}

	// save the repaired file data
	f.mu.RLock()
	err = r.saveFile(f)
	f.mu.RUnlock()
	if err != nil {
		// definitely bad, but we probably shouldn't delete from the
		// repair set if this happens
		r.log.Printf("failed to save repaired file %v: %v", name, err)
	}
}

// repairChunks uploads missing chunks of f to new hosts.
func (r *Renter) repairChunks(f *file, handle io.ReaderAt, chunks map[uint64][]uint64, duration types.BlockHeight) {
	// create host pool
	contractSize := (f.pieceSize + crypto.TwofishOverhead) * f.numChunks() // each host gets one piece of each chunk
	pool, err := r.hostDB.NewPool(contractSize, duration)
	if err != nil {
		r.log.Printf("failed to repair %v: %v", f.name, err)
		return
	}
	defer pool.Close() // heh

	for chunk, pieces := range chunks {
		// determine host set
		old := f.chunkHosts(chunk)
		hosts := pool.UniqueHosts(f.erasureCode.NumPieces()-len(old), old)
		if len(hosts) == 0 {
			r.log.Printf("aborting repair of %v: not enough hosts", f.name)
			return
		}
		// upload to new hosts
		err = f.repair(chunk, pieces, handle, hosts)
		if err != nil {
			r.log.Printf("aborting repair of %v: %v", f.name, err)
			return
		}
	}
}

// renewContracts renews each of the supplied contracts, replacing their entry
// in f with the new contract.
func (r *Renter) renewContracts(f *file, contracts []fileContract, newHeight types.BlockHeight) {
	for _, c := range contracts {
		newID, err := r.hostDB.Renew(c.ID, newHeight)
		if err != nil {
			r.log.Printf("failed to renew contract %v: %v", c.ID, err)
			continue
		}
		f.mu.Lock()
		f.contracts[newID] = fileContract{
			ID:          newID,
			IP:          c.IP,
			Pieces:      c.Pieces,
			WindowStart: newHeight,
		}
		// need to delete the old contract; otherwise f.expiringContracts
		// will continue to return it
		delete(f.contracts, c.ID)
		f.mu.Unlock()
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package social

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"

	"github.com/macaron-contrib/oauth2"

	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/modules/log"
	"github.com/gogits/gogs/modules/setting"
)

type BasicUserInfo struct {
	Identity string
	Name     string
	Email    string
}

type SocialConnector interface {
	Type() int
	UserInfo(*oauth2.Token, *url.URL) (*BasicUserInfo, error)
}

var (
	SocialMap = make(map[string]SocialConnector)
)

func NewOauthService() {
	if !setting.Cfg.Section("oauth").Key("ENABLED").MustBool() {
		return
	}

	oauth2.AppSubUrl = setting.AppSubUrl

	setting.OauthService = &setting.Oauther{}
	setting.OauthService.OauthInfos = make(map[string]*setting.OauthInfo)

	socialConfigs := make(map[string]*oauth2.Options)
	allOauthes := []string{"github", "google", "qq", "twitter", "weibo", "openshift"}
	// Load all OAuth config data.
	for _, name := range allOauthes {
		sec := setting.Cfg.Section("oauth." + name)
		if !sec.Key("ENABLED").MustBool() {
			continue
		}
		setting.OauthService.OauthInfos[name] = &setting.OauthInfo{
			Options: oauth2.Options{
				ClientID:     sec.Key("CLIENT_ID").String(),
				ClientSecret: sec.Key("CLIENT_SECRET").String(),
				Scopes:       sec.Key("SCOPES").Strings(" "),
				PathLogin:    "/user/login/oauth2/" + name,
				PathCallback: setting.AppSubUrl + "/user/login/" + name,
				RedirectURL:  setting.AppUrl + "user/login/" + name,
			},
			AuthUrl:  sec.Key("AUTH_URL").String(),
			TokenUrl: sec.Key("TOKEN_URL").String(),
		}
		socialConfigs[name] = &oauth2.Options{
			ClientID:     setting.OauthService.OauthInfos[name].ClientID,
			ClientSecret: setting.OauthService.OauthInfos[name].ClientSecret,
			Scopes:       setting.OauthService.OauthInfos[name].Scopes,
		}
	}
	enabledOauths := make([]string, 0, 10)

	// GitHub.
	if setting.Cfg.Section("oauth.github").Key("ENABLED").MustBool() {
		setting.OauthService.GitHub = true
		newGitHubOauth(socialConfigs["github"])
		enabledOauths = append(enabledOauths, "GitHub")
	}

	// Google.
	if setting.Cfg.Section("oauth.google").Key("ENABLED").MustBool() {
		setting.OauthService.Google = true
		newGoogleOauth(socialConfigs["google"])
		enabledOauths = append(enabledOauths, "Google")
	}

	// QQ.
	if setting.Cfg.Section("oauth.qq").Key("ENABLED").MustBool() {
		setting.OauthService.Tencent = true
		newTencentOauth(socialConfigs["qq"])
		enabledOauths = append(enabledOauths, "QQ")
	}

	// Twitter.
	// if setting.Cfg.Section("oauth.twitter").Key( "ENABLED").MustBool() {
	// 	setting.OauthService.Twitter = true
	// 	newTwitterOauth(socialConfigs["twitter"])
	// 	enabledOauths = append(enabledOauths, "Twitter")
	// }

	// Weibo.
	if setting.Cfg.Section("oauth.weibo").Key("ENABLED").MustBool() {
		setting.OauthService.Weibo = true
		newWeiboOauth(socialConfigs["weibo"])
		enabledOauths = append(enabledOauths, "Weibo")
	}

	// Openshift.
	if setting.Cfg.Section("oauth.openshift").Key("ENABLED").MustBool() {
		setting.OauthService.OpenShift = true
		newOpenShiftOauth(socialConfigs["openshift"])
		enabledOauths = append(enabledOauths, "OpenShift")
	}

	log.Info("Oauth Service Enabled %s", enabledOauths)
}

//   ________.__  __     ___ ___      ___.
//  /  _____/|__|/  |_  /   |   \ __ _\_ |__
// /   \  ___|  \   __\/    ~    \  |  \ __ \
// \    \_\  \  ||  |  \    Y    /  |  / \_\ \
//  \______  /__||__|   \___|_  /|____/|___  /
//         \/                 \/           \/

type SocialGithub struct {
	opts *oauth2.Options
}

func newGitHubOauth(opts *oauth2.Options) {
	SocialMap["github"] = &SocialGithub{opts}
}

func (s *SocialGithub) Type() int {
	return int(models.GITHUB)
}

func (s *SocialGithub) UserInfo(token *oauth2.Token, _ *url.URL) (*BasicUserInfo, error) {
	transport := s.opts.NewTransportFromToken(token)
	var data struct {
		Id    int    `json:"id"`
		Name  string `json:"login"`
		Email string `json:"email"`
	}
	r, err := transport.Client().Get("https://api.github.com/user")
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()
	if err = json.NewDecoder(r.Body).Decode(&data); err != nil {
		return nil, err
	}
	return &BasicUserInfo{
		Identity: strconv.Itoa(data.Id),
		Name:     data.Name,
		Email:    data.Email,
	}, nil
}

//   ________                     .__
//  /  _____/  ____   ____   ____ |  |   ____
// /   \  ___ /  _ \ /  _ \ / ___\|  | _/ __ \
// \    \_\  (  <_> |  <_> ) /_/  >  |_\  ___/
//  \______  /\____/ \____/\___  /|____/\___  >
//         \/             /_____/           \/

type SocialGoogle struct {
	opts *oauth2.Options
}

func (s *SocialGoogle) Type() int {
	return int(models.GOOGLE)
}

func newGoogleOauth(opts *oauth2.Options) {
	SocialMap["google"] = &SocialGoogle{opts}
}

func (s *SocialGoogle) UserInfo(token *oauth2.Token, _ *url.URL) (*BasicUserInfo, error) {
	transport := s.opts.NewTransportFromToken(token)
	var data struct {
		Id    string `json:"id"`
		Name  string `json:"name"`
		Email string `json:"email"`
	}
	r, err := transport.Client().Get("https://www.googleapis.com/userinfo/v2/me")
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()
	if err = json.NewDecoder(r.Body).Decode(&data); err != nil {
		return nil, err
	}
	return &BasicUserInfo{
		Identity: data.Id,
		Name:     data.Name,
		Email:    data.Email,
	}, nil
}

// ________   ________
// \_____  \  \_____  \
//  /  / \  \  /  / \  \
// /   \_/.  \/   \_/.  \
// \_____\ \_/\_____\ \_/
//        \__>       \__>

type SocialTencent struct {
	opts *oauth2.Options
}

func newTencentOauth(opts *oauth2.Options) {
	SocialMap["qq"] = &SocialTencent{opts}
}

func (s *SocialTencent) Type() int {
	return int(models.QQ)
}

func (s *SocialTencent) UserInfo(token *oauth2.Token, URL *url.URL) (*BasicUserInfo, error) {
	r, err := http.Get("https://graph.z.qq.com/moc2/me?access_token=" + url.QueryEscape(token.AccessToken))
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, err
	}
	vals, err := url.ParseQuery(string(body))
	if err != nil {
		return nil, err
	}

	return &BasicUserInfo{
		Identity: vals.Get("openid"),
	}, nil
}

// ___________       .__  __    __
// \__    ___/_  _  _|__|/  |__/  |_  ___________
//   |    |  \ \/ \/ /  \   __\   __\/ __ \_  __ \
//   |    |   \     /|  ||  |  |  | \  ___/|  | \/
//   |____|    \/\_/ |__||__|  |__|  \___  >__|
//                                       \/

// type SocialTwitter struct {
// 	Token *oauth2.Token
// 	*oauth2.Transport
// }

// func (s *SocialTwitter) Type() int {
// 	return int(models.TWITTER)
// }

// func newTwitterOauth(config *oauth2.Config) {
// 	SocialMap["twitter"] = &SocialTwitter{
// 		Transport: &oauth.Transport{
// 			Config:    config,
// 			Transport: http.DefaultTransport,
// 		},
// 	}
// }

// func (s *SocialTwitter) SetRedirectUrl(url string) {
// 	s.Transport.Config.RedirectURL = url
// }

// //https://github.com/mrjones/oauth
// func (s *SocialTwitter) UserInfo(token *oauth2.Token, _ *url.URL) (*BasicUserInfo, error) {
// 	// transport := &oauth.Transport{Token: token}
// 	// var data struct {
// 	// 	Id    string `json:"id"`
// 	// 	Name  string `json:"name"`
// 	// 	Email string `json:"email"`
// 	// }
// 	// var err error

// 	// reqUrl := "https://www.googleapis.com/oauth2/v1/userinfo"
// 	// r, err := transport.Client().Get(reqUrl)
// 	// if err != nil {
// 	// 	return nil, err
// 	// }
// 	// defer r.Body.Close()
// 	// if err = json.NewDecoder(r.Body).Decode(&data); err != nil {
// 	// 	return nil, err
// 	// }
// 	// return &BasicUserInfo{
// 	// 	Identity: data.Id,
// 	// 	Name:     data.Name,
// 	// 	Email:    data.Email,
// 	// }, nil
// 	return nil, nil
// }

//  __      __       ._____.
// /  \    /  \ ____ |__\_ |__   ____
// \   \/\/   // __ \|  || __ \ /  _ \
//  \        /\  ___/|  || \_\ (  <_> )
//   \__/\  /  \___  >__||___  /\____/
//        \/       \/        \/

type SocialWeibo struct {
	opts *oauth2.Options
}

func newWeiboOauth(opts *oauth2.Options) {
	SocialMap["weibo"] = &SocialWeibo{opts}
}

func (s *SocialWeibo) Type() int {
	return int(models.WEIBO)
}

func (s *SocialWeibo) UserInfo(token *oauth2.Token, _ *url.URL) (*BasicUserInfo, error) {
	transport := s.opts.NewTransportFromToken(token)
	var data struct {
		Name string `json:"name"`
	}
	var urls = url.Values{
		"access_token": {token.AccessToken},
		"uid":          {token.Extra("uid")},
	}
	reqUrl := "https://api.weibo.com/2/users/show.json"
	r, err := transport.Client().Get(reqUrl + "?" + urls.Encode())
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()

	if err = json.NewDecoder(r.Body).Decode(&data); err != nil {
		return nil, err
	}
	return &BasicUserInfo{
		Identity: token.Extra("uid"),
		Name:     data.Name,
	}, nil
}

// OpenShift

type SocialOpenShift struct {
	opts *oauth2.Options
}

func newOpenShiftOauth(opts *oauth2.Options) {
	SocialMap["openshift"] = &SocialOpenShift{opts}
}

func (s *SocialOpenShift) Type() int {
	return int(models.OPENSHIFT)
}

func (s *SocialOpenShift) UserInfo(token *oauth2.Token, _ *url.URL) (*BasicUserInfo, error) {
	log.Info("UserInfo for openshift being invoked!")
	transport := s.opts.NewTransportFromToken(token)
	var data struct {
		Metadata struct {
			Name string `json:"name"`
		} `json:"metadata"`
	}
	reqUrl := "https://os1.fabric8.io:8443/osapi/v1beta3/users/~"
	r, err := transport.Client().Get(reqUrl)
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()

	if err = json.NewDecoder(r.Body).Decode(&data); err != nil {
		return nil, err
	}
	return &BasicUserInfo{
		Identity: token.Extra("uid"),
		Name:     data.Metadata.Name,
	}, nil
}
<commit_msg>#1271 use environment variables to point at the openshift server to use to extract the identity and log it so users can check they have properly configured it<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package social

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"

	"github.com/macaron-contrib/oauth2"

	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/modules/log"
	"github.com/gogits/gogs/modules/setting"
	"os"
)

type BasicUserInfo struct {
	Identity string
	Name     string
	Email    string
}

type SocialConnector interface {
	Type() int
	UserInfo(*oauth2.Token, *url.URL) (*BasicUserInfo, error)
}

var (
	SocialMap = make(map[string]SocialConnector)
)

func NewOauthService() {
	if !setting.Cfg.Section("oauth").Key("ENABLED").MustBool() {
		return
	}

	oauth2.AppSubUrl = setting.AppSubUrl

	setting.OauthService = &setting.Oauther{}
	setting.OauthService.OauthInfos = make(map[string]*setting.OauthInfo)

	socialConfigs := make(map[string]*oauth2.Options)
	allOauthes := []string{"github", "google", "qq", "twitter", "weibo", "openshift"}
	// Load all OAuth config data.
	for _, name := range allOauthes {
		sec := setting.Cfg.Section("oauth." + name)
		if !sec.Key("ENABLED").MustBool() {
			continue
		}
		setting.OauthService.OauthInfos[name] = &setting.OauthInfo{
			Options: oauth2.Options{
				ClientID:     sec.Key("CLIENT_ID").String(),
				ClientSecret: sec.Key("CLIENT_SECRET").String(),
				Scopes:       sec.Key("SCOPES").Strings(" "),
				PathLogin:    "/user/login/oauth2/" + name,
				PathCallback: setting.AppSubUrl + "/user/login/" + name,
				RedirectURL:  setting.AppUrl + "user/login/" + name,
			},
			AuthUrl:  sec.Key("AUTH_URL").String(),
			TokenUrl: sec.Key("TOKEN_URL").String(),
		}
		socialConfigs[name] = &oauth2.Options{
			ClientID:     setting.OauthService.OauthInfos[name].ClientID,
			ClientSecret: setting.OauthService.OauthInfos[name].ClientSecret,
			Scopes:       setting.OauthService.OauthInfos[name].Scopes,
		}
	}
	enabledOauths := make([]string, 0, 10)

	// GitHub.
	if setting.Cfg.Section("oauth.github").Key("ENABLED").MustBool() {
		setting.OauthService.GitHub = true
		newGitHubOauth(socialConfigs["github"])
		enabledOauths = append(enabledOauths, "GitHub")
	}

	// Google.
	if setting.Cfg.Section("oauth.google").Key("ENABLED").MustBool() {
		setting.OauthService.Google = true
		newGoogleOauth(socialConfigs["google"])
		enabledOauths = append(enabledOauths, "Google")
	}

	// QQ.
	if setting.Cfg.Section("oauth.qq").Key("ENABLED").MustBool() {
		setting.OauthService.Tencent = true
		newTencentOauth(socialConfigs["qq"])
		enabledOauths = append(enabledOauths, "QQ")
	}

	// Twitter.
	// if setting.Cfg.Section("oauth.twitter").Key( "ENABLED").MustBool() {
	// 	setting.OauthService.Twitter = true
	// 	newTwitterOauth(socialConfigs["twitter"])
	// 	enabledOauths = append(enabledOauths, "Twitter")
	// }

	// Weibo.
	if setting.Cfg.Section("oauth.weibo").Key("ENABLED").MustBool() {
		setting.OauthService.Weibo = true
		newWeiboOauth(socialConfigs["weibo"])
		enabledOauths = append(enabledOauths, "Weibo")
	}

	// Openshift.
	if setting.Cfg.Section("oauth.openshift").Key("ENABLED").MustBool() {
		setting.OauthService.OpenShift = true
		newOpenShiftOauth(socialConfigs["openshift"])
		enabledOauths = append(enabledOauths, "OpenShift")
		log.Info("Using OpenShift request URL (%s)", getOpenShiftReqUrl())
	}

	log.Info("Oauth Service Enabled %s", enabledOauths)
}

//   ________.__  __     ___ ___      ___.
//  /  _____/|__|/  |_  /   |   \ __ _\_ |__
// /   \  ___|  \   __\/    ~    \  |  \ __ \
// \    \_\  \  ||  |  \    Y    /  |  / \_\ \
//  \______  /__||__|   \___|_  /|____/|___  /
//         \/                 \/           \/

type SocialGithub struct {
	opts *oauth2.Options
}

func newGitHubOauth(opts *oauth2.Options) {
	SocialMap["github"] = &SocialGithub{opts}
}

func (s *SocialGithub) Type() int {
	return int(models.GITHUB)
}

func (s *SocialGithub) UserInfo(token *oauth2.Token, _ *url.URL) (*BasicUserInfo, error) {
	transport := s.opts.NewTransportFromToken(token)
	var data struct {
		Id    int    `json:"id"`
		Name  string `json:"login"`
		Email string `json:"email"`
	}
	r, err := transport.Client().Get("https://api.github.com/user")
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()
	if err = json.NewDecoder(r.Body).Decode(&data); err != nil {
		return nil, err
	}
	return &BasicUserInfo{
		Identity: strconv.Itoa(data.Id),
		Name:     data.Name,
		Email:    data.Email,
	}, nil
}

//   ________                     .__
//  /  _____/  ____   ____   ____ |  |   ____
// /   \  ___ /  _ \ /  _ \ / ___\|  | _/ __ \
// \    \_\  (  <_> |  <_> ) /_/  >  |_\  ___/
//  \______  /\____/ \____/\___  /|____/\___  >
//         \/             /_____/           \/

type SocialGoogle struct {
	opts *oauth2.Options
}

func (s *SocialGoogle) Type() int {
	return int(models.GOOGLE)
}

func newGoogleOauth(opts *oauth2.Options) {
	SocialMap["google"] = &SocialGoogle{opts}
}

func (s *SocialGoogle) UserInfo(token *oauth2.Token, _ *url.URL) (*BasicUserInfo, error) {
	transport := s.opts.NewTransportFromToken(token)
	var data struct {
		Id    string `json:"id"`
		Name  string `json:"name"`
		Email string `json:"email"`
	}
	r, err := transport.Client().Get("https://www.googleapis.com/userinfo/v2/me")
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()
	if err = json.NewDecoder(r.Body).Decode(&data); err != nil {
		return nil, err
	}
	return &BasicUserInfo{
		Identity: data.Id,
		Name:     data.Name,
		Email:    data.Email,
	}, nil
}

// ________   ________
// \_____  \  \_____  \
//  /  / \  \  /  / \  \
// /   \_/.  \/   \_/.  \
// \_____\ \_/\_____\ \_/
//        \__>       \__>

type SocialTencent struct {
	opts *oauth2.Options
}

func newTencentOauth(opts *oauth2.Options) {
	SocialMap["qq"] = &SocialTencent{opts}
}

func (s *SocialTencent) Type() int {
	return int(models.QQ)
}

func (s *SocialTencent) UserInfo(token *oauth2.Token, URL *url.URL) (*BasicUserInfo, error) {
	r, err := http.Get("https://graph.z.qq.com/moc2/me?access_token=" + url.QueryEscape(token.AccessToken))
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, err
	}
	vals, err := url.ParseQuery(string(body))
	if err != nil {
		return nil, err
	}

	return &BasicUserInfo{
		Identity: vals.Get("openid"),
	}, nil
}

// ___________       .__  __    __
// \__    ___/_  _  _|__|/  |__/  |_  ___________
//   |    |  \ \/ \/ /  \   __\   __\/ __ \_  __ \
//   |    |   \     /|  ||  |  |  | \  ___/|  | \/
//   |____|    \/\_/ |__||__|  |__|  \___  >__|
//                                       \/

// type SocialTwitter struct {
// 	Token *oauth2.Token
// 	*oauth2.Transport
// }

// func (s *SocialTwitter) Type() int {
// 	return int(models.TWITTER)
// }

// func newTwitterOauth(config *oauth2.Config) {
// 	SocialMap["twitter"] = &SocialTwitter{
// 		Transport: &oauth.Transport{
// 			Config:    config,
// 			Transport: http.DefaultTransport,
// 		},
// 	}
// }

// func (s *SocialTwitter) SetRedirectUrl(url string) {
// 	s.Transport.Config.RedirectURL = url
// }

// //https://github.com/mrjones/oauth
// func (s *SocialTwitter) UserInfo(token *oauth2.Token, _ *url.URL) (*BasicUserInfo, error) {
// 	// transport := &oauth.Transport{Token: token}
// 	// var data struct {
// 	// 	Id    string `json:"id"`
// 	// 	Name  string `json:"name"`
// 	// 	Email string `json:"email"`
// 	// }
// 	// var err error

// 	// reqUrl := "https://www.googleapis.com/oauth2/v1/userinfo"
// 	// r, err := transport.Client().Get(reqUrl)
// 	// if err != nil {
// 	// 	return nil, err
// 	// }
// 	// defer r.Body.Close()
// 	// if err = json.NewDecoder(r.Body).Decode(&data); err != nil {
// 	// 	return nil, err
// 	// }
// 	// return &BasicUserInfo{
// 	// 	Identity: data.Id,
// 	// 	Name:     data.Name,
// 	// 	Email:    data.Email,
// 	// }, nil
// 	return nil, nil
// }

//  __      __       ._____.
// /  \    /  \ ____ |__\_ |__   ____
// \   \/\/   // __ \|  || __ \ /  _ \
//  \        /\  ___/|  || \_\ (  <_> )
//   \__/\  /  \___  >__||___  /\____/
//        \/       \/        \/

type SocialWeibo struct {
	opts *oauth2.Options
}

func newWeiboOauth(opts *oauth2.Options) {
	SocialMap["weibo"] = &SocialWeibo{opts}
}

func (s *SocialWeibo) Type() int {
	return int(models.WEIBO)
}

func (s *SocialWeibo) UserInfo(token *oauth2.Token, _ *url.URL) (*BasicUserInfo, error) {
	transport := s.opts.NewTransportFromToken(token)
	var data struct {
		Name string `json:"name"`
	}
	var urls = url.Values{
		"access_token": {token.AccessToken},
		"uid":          {token.Extra("uid")},
	}
	reqUrl := "https://api.weibo.com/2/users/show.json"
	r, err := transport.Client().Get(reqUrl + "?" + urls.Encode())
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()

	if err = json.NewDecoder(r.Body).Decode(&data); err != nil {
		return nil, err
	}
	return &BasicUserInfo{
		Identity: token.Extra("uid"),
		Name:     data.Name,
	}, nil
}

// OpenShift

type SocialOpenShift struct {
	opts *oauth2.Options
}

func newOpenShiftOauth(opts *oauth2.Options) {
	SocialMap["openshift"] = &SocialOpenShift{opts}
}

func (s *SocialOpenShift) Type() int {
	return int(models.OPENSHIFT)
}

func (s *SocialOpenShift) UserInfo(token *oauth2.Token, _ *url.URL) (*BasicUserInfo, error) {
	log.Info("UserInfo for openshift being invoked!")
	transport := s.opts.NewTransportFromToken(token)
	var data struct {
		Metadata struct {
			Name string `json:"name"`
		} `json:"metadata"`
	}
	reqUrl := getOpenShiftReqUrl()
	log.Info("Using OpenShift URL %s", reqUrl)
	r, err := transport.Client().Get(reqUrl)
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()

	if err = json.NewDecoder(r.Body).Decode(&data); err != nil {
		return nil, err
	}
	return &BasicUserInfo{
		Identity: token.Extra("uid"),
		Name:     data.Metadata.Name,
	}, nil
}

func getOpenShiftReqUrl() string {
	return os.ExpandEnv("https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}/osapi/v1beta3/users/~")
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/Sirupsen/logrus"
)

var log = logrus.New()

func init() {
	log.Formatter = new(logrus.JSONFormatter)
	log.Formatter = new(logrus.TextFormatter) // default
}

func main() {
	defer func() {
		err := recover()
		if err != nil {
			log.WithFields(logrus.Fields{
				"omg":    true,
				"err":    err,
				"number": 100,
			}).Fatal("The ice breaks!")
		}
	}()

	log.WithFields(logrus.Fields{
		"animal": "walrus",
		"size":   10,
	}).Info("A group of walrus emerges from the ocean")

	log.WithFields(logrus.Fields{
		"omg":    true,
		"number": 122,
	}).Warn("The group's number increased tremendously!")

	log.WithFields(logrus.Fields{
		"animal": "orca",
		"size":   9009,
	}).Panic("It's over 9000!")
}
<commit_msg>examples/basic: add debug level<commit_after>package main

import (
	"github.com/Sirupsen/logrus"
)

var log = logrus.New()

func init() {
	log.Formatter = new(logrus.JSONFormatter)
	log.Formatter = new(logrus.TextFormatter) // default
	log.Level = logrus.DebugLevel
}

func main() {
	defer func() {
		err := recover()
		if err != nil {
			log.WithFields(logrus.Fields{
				"omg":    true,
				"err":    err,
				"number": 100,
			}).Fatal("The ice breaks!")
		}
	}()

	log.WithFields(logrus.Fields{
		"animal": "walrus",
		"number": 8,
	}).Debug("Started observing beach")

	log.WithFields(logrus.Fields{
		"animal": "walrus",
		"size":   10,
	}).Info("A group of walrus emerges from the ocean")

	log.WithFields(logrus.Fields{
		"omg":    true,
		"number": 122,
	}).Warn("The group's number increased tremendously!")

	log.WithFields(logrus.Fields{
		"temperature": -4,
	}).Debug("Temperature changes")

	log.WithFields(logrus.Fields{
		"animal": "orca",
		"size":   9009,
	}).Panic("It's over 9000!")
}
<|endoftext|>


<commit_before>package moveserver

import (
	"fmt"
	"kodi_automation/transmission"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	//"strconv"
	"strings"
	"sync"
	"time"
)

func directoryListing(dirname string, levels int, dirs_only bool) ([]string, error) {
	res := []string{}
	err := filepath.Walk(dirname, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if dirs_only && !info.IsDir() {
			return nil
		}
		tmp := path
		for i := 0; i < levels; i++ {
			tmp = filepath.Dir(tmp)
			if tmp == dirname {
				res = append(res, path)
				break
			}
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	log.Printf("directoryListing %s %v", dirname, res)
	return res, nil
}

type MoveRequest struct {
	Path string
	To   string
}

type MoveListenerRequest struct {
	Request MoveRequest
}

func moveListener(s *MoveServer, ch chan MoveListenerRequest) {
	for {
		req := <-ch
		log.Printf("Received move requests %v", req)

		bytes, err := exec.Command("mv", req.Request.Path, req.Request.To).Output()
		log.Printf("Move result: err: %v; output: %s", err, string(bytes))
		s.SetPathMoveResult(req.Request.Path, err, string(bytes))
	}
}

func updateDiskStats(s *MoveServer) {
	regex := regexp.MustCompile(`([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)%\s+(.*)`)
	bytes, err := exec.Command("df", "-h").Output()
	if err != nil {
		log.Printf("df -h: %v", err)
		s.setDiskStats(nil)
	} else {
		ds := []DiskStats{}
		output := string(bytes)
		for _, line := range strings.Split(string(output), "\n")[1:] {
			grs := regex.FindStringSubmatch(line)
			if grs == nil {
				continue
			}
			//size, err1 := strconv.ParseInt(grs[2], 10, 64)
			//used, err2 := strconv.ParseInt(grs[3], 10, 64)
			//avail, err3 := strconv.ParseInt(grs[4], 10, 64)
			//if err1 != nil || err2 != nil || err3 != nil {
			//	log.Printf("Atoi errors: %v, %v, %v", err1, err2, err3)
			//	continue
			//}
			if grs != nil {
				ds = append(ds, DiskStats{
					Path:       grs[6],
					Filesystem: grs[1],
					Size:       grs[2], //size,
					Used:       grs[3], // used,
					Avail:      grs[4], //avail,
				})
			}
		}
		log.Print("Disk stats:")
		for _, x := range ds {
			log.Printf("   %#v", x)
		}
		s.setDiskStats(ds)
	}
}

func diskStatsUpdater(s *MoveServer, d time.Duration) {
	// regex := regexp.MustCompile(`([^\s]+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)%\s+(.*)`)
	for {
		updateDiskStats(s)
		time.Sleep(d)
	}
}

func updateCache(s *MoveServer) {
	log.Printf("Updating cached info.")
	paths, err := directoryListing(s.sourceDir, 1, false)
	sl, slErr := getSeriesTargetListing(s.seriesTarget)
	tis, _ := s.transmissionCli.GetTorrents(s.sourceDir)

	if err != nil || slErr != nil {
		log.Printf(fmt.Sprintf("Update cached info errors: %v; %v", err, slErr))
	} else {
		s.setCachedInfo(paths, tis, sl)
	}
}

func cacheUpdater(s *MoveServer, d time.Duration) {
	for {
		updateCache(s)
		time.Sleep(d)
	}
}

func (s *MoveServer) UpdateCacheAsync() {
	go func() {
		updateCache(s)
	}()
}

func (s *MoveServer) UpdateDiskStatsAsync() {
	go func() {
		updateDiskStats(s)
	}()
}

// PathInfo has all the information kept on Server about paths in the Download directory.
type PathMoveInfo struct {
	Moving          bool
	Target          string
	LastError       error
	LastErrorOutput string
}

type PathInfo struct {
	Name        string
	Path        string
	AllowMove   bool
	MoveInfo    PathMoveInfo
	TorrentInfo transmission.TorrentInfo
}

type DiskStats struct {
	Path       string
	Filesystem string
	Size       string
	Used       string
	Avail      string
}

type MoveServer struct {
	transmissionCli *transmission.Transmission

	// Directory to scan
	sourceDir string

	// Targets for movies
	moviesTarget string

	// Target for series and directory listing.
	seriesTarget        string
	seriesTargetListing []string

	// old path info kept for history - sorted by when things were moved
	pathInfoHistory []PathInfo

	// Path info is kept separately.
	pathInfo        map[string]PathInfo
	cacheRefreshed  time.Time
	refreshDuration time.Duration

	// Disk stats
	diskStats []DiskStats

	// chennels
	moveChannel chan MoveListenerRequest

	lock sync.Mutex
}

func New(sourceDir string, moviesTarget string, seriesTarget string, maxMvComands int, mvBufferSize int) (*MoveServer, error) {
	s := &MoveServer{
		transmissionCli: transmission.New("", 0, "", ""),
		sourceDir:       sourceDir,
		moviesTarget:    moviesTarget,
		seriesTarget:    seriesTarget,
		pathInfoHistory: []PathInfo{},
		pathInfo:        map[string]PathInfo{},
		refreshDuration: 5 * time.Minute,
		cacheRefreshed:  time.Now(),
		moveChannel:     make(chan MoveListenerRequest, mvBufferSize),
		lock:            sync.Mutex{},
	}

	for i := 0; i < maxMvComands; i++ {
		go moveListener(s, s.moveChannel)
	}
	go cacheUpdater(s, 5*time.Minute)
	go diskStatsUpdater(s, 5*time.Minute)

	return s, nil
}

func (s *MoveServer) GetSeriesTarget() string {
	s.lock.Lock()
	defer s.lock.Unlock()

	return s.seriesTarget
}

func (s *MoveServer) GetMoviesTarget() string {
	s.lock.Lock()
	defer s.lock.Unlock()

	return s.moviesTarget
}

func (s *MoveServer) GetPathInfo() map[string]PathInfo {
	s.lock.Lock()
	defer s.lock.Unlock()

	return s.pathInfo
}

func (s *MoveServer) GetPathInfoHistory() []PathInfo {
	s.lock.Lock()
	defer s.lock.Unlock()

	return s.pathInfoHistory
}

func (s *MoveServer) GetCacheRefreshed() time.Time {
	s.lock.Lock()
	defer s.lock.Unlock()

	return s.cacheRefreshed
}

func (s *MoveServer) GetSeriesTargetListing() []string {
	s.lock.Lock()
	defer s.lock.Unlock()

	return s.seriesTargetListing
}

func (s *MoveServer) GetPathInfoAndPathInfoHistory() (map[string]PathInfo, []PathInfo) {
	s.lock.Lock()
	defer s.lock.Unlock()

	return s.pathInfo, s.pathInfoHistory
}

func (s *MoveServer) GetMvBuffSizeAndElems() (int, int) {
	s.lock.Lock()
	defer s.lock.Unlock()

	return cap(s.moveChannel), len(s.moveChannel)
}

func (s *MoveServer) GetDiskStats() []DiskStats {
	s.lock.Lock()
	defer s.lock.Unlock()

	return s.diskStats
}

func (s *MoveServer) getTargetForSeriesMove(path, to string) (string, error) {
	if filepath.IsAbs(to) {
		return "", fmt.Errorf("Target path is an absolute path, expected relative: %s", to)
	}
	target, err := filepath.Abs(filepath.Join(s.seriesTarget, to, filepath.Base(path)))
	if err != nil {
		return "", err
	}
	if target[:len(s.seriesTarget)] != s.seriesTarget {
		return "", fmt.Errorf("Tried to move path outside series dir, to %s", target)
	}
	return target, nil
}

func (s *MoveServer) getTargetForMovieMove(path string) (string, error) {
	target, err := filepath.Abs(filepath.Join(s.moviesTarget, filepath.Base(path)))
	if err != nil {
		return "", err
	}
	if filepath.Dir(target) != s.moviesTarget {
		return "", fmt.Errorf("Tried to move outside movies dir: %s", target)
	}
	return target, nil
}

func (s *MoveServer) Move(path string, to string) error {
	s.lock.Lock()
	defer s.lock.Unlock()

	// ------------ Path verification ---------------
	if filepath.Dir(path) != s.sourceDir {
		return fmt.Errorf("Trying to move file outside of %s", s.sourceDir)
	}
	pi, ok := s.pathInfo[path]
	if !ok {
		return fmt.Errorf("Requested move path %s not found in our data bank", path)
	}
	if pi.MoveInfo.Moving {
		return fmt.Errorf("Requested move path %s is currently in move to %s", pi.MoveInfo.Target)
	}
	if _, err := os.Stat(path); err != nil && os.IsNotExist(err) {
		pi.MoveInfo.LastError = err
		s.pathInfo[path] = pi
		return err
	}

	// ------------ Target verification ---------------
	var target string
	var err error
	if to == "movies" {
		target, err = s.getTargetForMovieMove(path)
	} else {
		target, err = s.getTargetForSeriesMove(path, to)
	}
	if err != nil {
		pi.MoveInfo.LastError = err
		s.pathInfo[path] = pi
		return err
	}
	if len(s.moveChannel) == cap(s.moveChannel) {
		return fmt.Errorf("Mv requests buffer buffer is full")
	}

	// Actually making a move.
	pi.MoveInfo.Moving = true
	pi.MoveInfo.Target = target

	s.moveChannel <- MoveListenerRequest{
		Request: MoveRequest{
			Path: path,
			To:   target,
		},
	}
	s.pathInfo[path] = pi
	return nil
}

func (s *MoveServer) SetPathMoveResult(path string, err error, output string) error {
	s.lock.Lock()
	defer s.lock.Unlock()

	pi, ok := s.pathInfo[path]
	if !ok {
		return fmt.Errorf("path not found")
	}

	pi.AllowMove = false
	pi.MoveInfo = PathMoveInfo{
		Moving:          false,
		Target:          pi.MoveInfo.Target,
		LastError:       err,
		LastErrorOutput: output,
	}
	if err == nil {
		// Successful move.
		delete(s.pathInfo, path)
		s.pathInfoHistory = append(s.pathInfoHistory, pi)
	} else {
		// Unsuccessful move.
		pi.MoveInfo.Target = ""
		s.pathInfo[path] = pi
	}
	return nil
}

func (s *MoveServer) loadTorrentsInfo() ([]transmission.TorrentInfo, error) {
	// In the future I need more to ask both - transmission and scan local disk.
	return s.transmissionCli.GetTorrents(s.sourceDir)
}

func getSeriesTargetListing(seriesTarget string) ([]string, error) {
	listing, err := directoryListing(seriesTarget, 2, true)
	if err != nil {
		return nil, err
	}
	return listing, nil
}

func (s *MoveServer) setCachedInfo(paths []string, ntis []transmission.TorrentInfo, nstl []string) {
	s.lock.Lock()
	defer s.lock.Unlock()

	// Set new info.
	oldPathInfo := s.pathInfo
	newPathInfo := map[string]PathInfo{}
	for _, path := range paths {
		newPathInfo[path] = PathInfo{
			Name:      filepath.Base(path),
			Path:      path,
			AllowMove: true,
			MoveInfo:  PathMoveInfo{},
		}
	}
	// Copy MoveInfo and TorrentInfo from old data.
	for _, opi := range oldPathInfo {
		pi, ok := newPathInfo[opi.Path]
		if ok {
			pi.MoveInfo = opi.MoveInfo

			// Copy old torrent info only if the new torrent info is nil (there was an
			// error while getting torrents info) or that path is currently moved
			// (torrent may have been deleted from transmission.
			var ti transmission.TorrentInfo
			if ntis == nil || pi.MoveInfo.Moving {
				ti = pi.TorrentInfo
			}
			pi.TorrentInfo = ti

			newPathInfo[opi.Path] = pi
		}
	}
	// Update path info with torrents info.
	for _, nti := range ntis {
		path := filepath.Join(nti.DownloadDir, nti.Name)
		pi, ok := newPathInfo[path]
		if ok {
			pi.TorrentInfo = nti
			newPathInfo[path] = pi
		}
	}
	// Update AllowMove
	for _, path := range paths {
		pi := newPathInfo[path]
		pi.AllowMove = pi.TorrentInfo.IsFinished && !pi.MoveInfo.Moving
		newPathInfo[path] = pi
	}

	s.cacheRefreshed = time.Now()
	s.pathInfo = newPathInfo
}

func (s *MoveServer) setDiskStats(nds []DiskStats) {
	s.lock.Lock()
	defer s.lock.Unlock()

	diskStats := []DiskStats{}
	for _, ds := range nds {
		if strings.HasPrefix(s.seriesTarget, ds.Path) || strings.HasPrefix(s.moviesTarget, ds.Path) {
			diskStats = append(diskStats, ds)
		}
	}
	s.diskStats = diskStats
}
<commit_msg>Get combined output of the command<commit_after>package moveserver

import (
	"fmt"
	"kodi_automation/transmission"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	//"strconv"
	"strings"
	"sync"
	"time"
)

func directoryListing(dirname string, levels int, dirs_only bool) ([]string, error) {
	res := []string{}
	err := filepath.Walk(dirname, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if dirs_only && !info.IsDir() {
			return nil
		}
		tmp := path
		for i := 0; i < levels; i++ {
			tmp = filepath.Dir(tmp)
			if tmp == dirname {
				res = append(res, path)
				break
			}
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	log.Printf("directoryListing %s %v", dirname, res)
	return res, nil
}

type MoveRequest struct {
	Path string
	To   string
}

type MoveListenerRequest struct {
	Request MoveRequest
}

func moveListener(s *MoveServer, ch chan MoveListenerRequest) {
	for {
		req := <-ch
		log.Printf("Received move requests %v", req)

		bytes, err := exec.Command("mv", req.Request.Path, req.Request.To).CombinedOutput()
		log.Printf("Move result: err: %v; output: %s", err, string(bytes))
		s.SetPathMoveResult(req.Request.Path, err, string(bytes))
	}
}

func updateDiskStats(s *MoveServer) {
	regex := regexp.MustCompile(`([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)%\s+(.*)`)
	bytes, err := exec.Command("df", "-h").Output()
	if err != nil {
		log.Printf("df -h: %v", err)
		s.setDiskStats(nil)
	} else {
		ds := []DiskStats{}
		output := string(bytes)
		for _, line := range strings.Split(string(output), "\n")[1:] {
			grs := regex.FindStringSubmatch(line)
			if grs == nil {
				continue
			}
			//size, err1 := strconv.ParseInt(grs[2], 10, 64)
			//used, err2 := strconv.ParseInt(grs[3], 10, 64)
			//avail, err3 := strconv.ParseInt(grs[4], 10, 64)
			//if err1 != nil || err2 != nil || err3 != nil {
			//	log.Printf("Atoi errors: %v, %v, %v", err1, err2, err3)
			//	continue
			//}
			if grs != nil {
				ds = append(ds, DiskStats{
					Path:       grs[6],
					Filesystem: grs[1],
					Size:       grs[2], //size,
					Used:       grs[3], // used,
					Avail:      grs[4], //avail,
				})
			}
		}
		log.Print("Disk stats:")
		for _, x := range ds {
			log.Printf("   %#v", x)
		}
		s.setDiskStats(ds)
	}
}

func diskStatsUpdater(s *MoveServer, d time.Duration) {
	// regex := regexp.MustCompile(`([^\s]+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)%\s+(.*)`)
	for {
		updateDiskStats(s)
		time.Sleep(d)
	}
}

func updateCache(s *MoveServer) {
	log.Printf("Updating cached info.")
	paths, err := directoryListing(s.sourceDir, 1, false)
	sl, slErr := getSeriesTargetListing(s.seriesTarget)
	tis, _ := s.transmissionCli.GetTorrents(s.sourceDir)

	if err != nil || slErr != nil {
		log.Printf(fmt.Sprintf("Update cached info errors: %v; %v", err, slErr))
	} else {
		s.setCachedInfo(paths, tis, sl)
	}
}

func cacheUpdater(s *MoveServer, d time.Duration) {
	for {
		updateCache(s)
		time.Sleep(d)
	}
}

func (s *MoveServer) UpdateCacheAsync() {
	go func() {
		updateCache(s)
	}()
}

func (s *MoveServer) UpdateDiskStatsAsync() {
	go func() {
		updateDiskStats(s)
	}()
}

// PathInfo has all the information kept on Server about paths in the Download directory.
type PathMoveInfo struct {
	Moving          bool
	Target          string
	LastError       error
	LastErrorOutput string
}

type PathInfo struct {
	Name        string
	Path        string
	AllowMove   bool
	MoveInfo    PathMoveInfo
	TorrentInfo transmission.TorrentInfo
}

type DiskStats struct {
	Path       string
	Filesystem string
	Size       string
	Used       string
	Avail      string
}

type MoveServer struct {
	transmissionCli *transmission.Transmission

	// Directory to scan
	sourceDir string

	// Targets for movies
	moviesTarget string

	// Target for series and directory listing.
	seriesTarget        string
	seriesTargetListing []string

	// old path info kept for history - sorted by when things were moved
	pathInfoHistory []PathInfo

	// Path info is kept separately.
	pathInfo        map[string]PathInfo
	cacheRefreshed  time.Time
	refreshDuration time.Duration

	// Disk stats
	diskStats []DiskStats

	// chennels
	moveChannel chan MoveListenerRequest

	lock sync.Mutex
}

func New(sourceDir string, moviesTarget string, seriesTarget string, maxMvComands int, mvBufferSize int) (*MoveServer, error) {
	s := &MoveServer{
		transmissionCli: transmission.New("", 0, "", ""),
		sourceDir:       sourceDir,
		moviesTarget:    moviesTarget,
		seriesTarget:    seriesTarget,
		pathInfoHistory: []PathInfo{},
		pathInfo:        map[string]PathInfo{},
		refreshDuration: 5 * time.Minute,
		cacheRefreshed:  time.Now(),
		moveChannel:     make(chan MoveListenerRequest, mvBufferSize),
		lock:            sync.Mutex{},
	}

	for i := 0; i < maxMvComands; i++ {
		go moveListener(s, s.moveChannel)
	}
	go cacheUpdater(s, 5*time.Minute)
	go diskStatsUpdater(s, 5*time.Minute)

	return s, nil
}

func (s *MoveServer) GetSeriesTarget() string {
	s.lock.Lock()
	defer s.lock.Unlock()

	return s.seriesTarget
}

func (s *MoveServer) GetMoviesTarget() string {
	s.lock.Lock()
	defer s.lock.Unlock()

	return s.moviesTarget
}

func (s *MoveServer) GetPathInfo() map[string]PathInfo {
	s.lock.Lock()
	defer s.lock.Unlock()

	return s.pathInfo
}

func (s *MoveServer) GetPathInfoHistory() []PathInfo {
	s.lock.Lock()
	defer s.lock.Unlock()

	return s.pathInfoHistory
}

func (s *MoveServer) GetCacheRefreshed() time.Time {
	s.lock.Lock()
	defer s.lock.Unlock()

	return s.cacheRefreshed
}

func (s *MoveServer) GetSeriesTargetListing() []string {
	s.lock.Lock()
	defer s.lock.Unlock()

	return s.seriesTargetListing
}

func (s *MoveServer) GetPathInfoAndPathInfoHistory() (map[string]PathInfo, []PathInfo) {
	s.lock.Lock()
	defer s.lock.Unlock()

	return s.pathInfo, s.pathInfoHistory
}

func (s *MoveServer) GetMvBuffSizeAndElems() (int, int) {
	s.lock.Lock()
	defer s.lock.Unlock()

	return cap(s.moveChannel), len(s.moveChannel)
}

func (s *MoveServer) GetDiskStats() []DiskStats {
	s.lock.Lock()
	defer s.lock.Unlock()

	return s.diskStats
}

func (s *MoveServer) getTargetForSeriesMove(path, to string) (string, error) {
	if filepath.IsAbs(to) {
		return "", fmt.Errorf("Target path is an absolute path, expected relative: %s", to)
	}
	target, err := filepath.Abs(filepath.Join(s.seriesTarget, to, filepath.Base(path)))
	if err != nil {
		return "", err
	}
	if target[:len(s.seriesTarget)] != s.seriesTarget {
		return "", fmt.Errorf("Tried to move path outside series dir, to %s", target)
	}
	return target, nil
}

func (s *MoveServer) getTargetForMovieMove(path string) (string, error) {
	target, err := filepath.Abs(filepath.Join(s.moviesTarget, filepath.Base(path)))
	if err != nil {
		return "", err
	}
	if filepath.Dir(target) != s.moviesTarget {
		return "", fmt.Errorf("Tried to move outside movies dir: %s", target)
	}
	return target, nil
}

func (s *MoveServer) Move(path string, to string) error {
	s.lock.Lock()
	defer s.lock.Unlock()

	// ------------ Path verification ---------------
	if filepath.Dir(path) != s.sourceDir {
		return fmt.Errorf("Trying to move file outside of %s", s.sourceDir)
	}
	pi, ok := s.pathInfo[path]
	if !ok {
		return fmt.Errorf("Requested move path %s not found in our data bank", path)
	}
	if pi.MoveInfo.Moving {
		return fmt.Errorf("Requested move path %s is currently in move to %s", pi.MoveInfo.Target)
	}
	if _, err := os.Stat(path); err != nil && os.IsNotExist(err) {
		pi.MoveInfo.LastError = err
		s.pathInfo[path] = pi
		return err
	}

	// ------------ Target verification ---------------
	var target string
	var err error
	if to == "movies" {
		target, err = s.getTargetForMovieMove(path)
	} else {
		target, err = s.getTargetForSeriesMove(path, to)
	}
	if err != nil {
		pi.MoveInfo.LastError = err
		s.pathInfo[path] = pi
		return err
	}
	if len(s.moveChannel) == cap(s.moveChannel) {
		return fmt.Errorf("Mv requests buffer buffer is full")
	}

	// Actually making a move.
	pi.MoveInfo.Moving = true
	pi.MoveInfo.Target = target

	s.moveChannel <- MoveListenerRequest{
		Request: MoveRequest{
			Path: path,
			To:   target,
		},
	}
	s.pathInfo[path] = pi
	return nil
}

func (s *MoveServer) SetPathMoveResult(path string, err error, output string) error {
	s.lock.Lock()
	defer s.lock.Unlock()

	pi, ok := s.pathInfo[path]
	if !ok {
		return fmt.Errorf("path not found")
	}

	pi.AllowMove = false
	pi.MoveInfo = PathMoveInfo{
		Moving:          false,
		Target:          pi.MoveInfo.Target,
		LastError:       err,
		LastErrorOutput: output,
	}
	if err == nil {
		// Successful move.
		delete(s.pathInfo, path)
		s.pathInfoHistory = append(s.pathInfoHistory, pi)
	} else {
		// Unsuccessful move.
		pi.MoveInfo.Target = ""
		s.pathInfo[path] = pi
	}
	return nil
}

func (s *MoveServer) loadTorrentsInfo() ([]transmission.TorrentInfo, error) {
	// In the future I need more to ask both - transmission and scan local disk.
	return s.transmissionCli.GetTorrents(s.sourceDir)
}

func getSeriesTargetListing(seriesTarget string) ([]string, error) {
	listing, err := directoryListing(seriesTarget, 2, true)
	if err != nil {
		return nil, err
	}
	return listing, nil
}

func (s *MoveServer) setCachedInfo(paths []string, ntis []transmission.TorrentInfo, nstl []string) {
	s.lock.Lock()
	defer s.lock.Unlock()

	// Set new info.
	oldPathInfo := s.pathInfo
	newPathInfo := map[string]PathInfo{}
	for _, path := range paths {
		newPathInfo[path] = PathInfo{
			Name:      filepath.Base(path),
			Path:      path,
			AllowMove: true,
			MoveInfo:  PathMoveInfo{},
		}
	}
	// Copy MoveInfo and TorrentInfo from old data.
	for _, opi := range oldPathInfo {
		pi, ok := newPathInfo[opi.Path]
		if ok {
			pi.MoveInfo = opi.MoveInfo

			// Copy old torrent info only if the new torrent info is nil (there was an
			// error while getting torrents info) or that path is currently moved
			// (torrent may have been deleted from transmission.
			var ti transmission.TorrentInfo
			if ntis == nil || pi.MoveInfo.Moving {
				ti = pi.TorrentInfo
			}
			pi.TorrentInfo = ti

			newPathInfo[opi.Path] = pi
		}
	}
	// Update path info with torrents info.
	for _, nti := range ntis {
		path := filepath.Join(nti.DownloadDir, nti.Name)
		pi, ok := newPathInfo[path]
		if ok {
			pi.TorrentInfo = nti
			newPathInfo[path] = pi
		}
	}
	// Update AllowMove
	for _, path := range paths {
		pi := newPathInfo[path]
		pi.AllowMove = pi.TorrentInfo.IsFinished && !pi.MoveInfo.Moving
		newPathInfo[path] = pi
	}

	s.cacheRefreshed = time.Now()
	s.pathInfo = newPathInfo
}

func (s *MoveServer) setDiskStats(nds []DiskStats) {
	s.lock.Lock()
	defer s.lock.Unlock()

	diskStats := []DiskStats{}
	for _, ds := range nds {
		if strings.HasPrefix(s.seriesTarget, ds.Path) || strings.HasPrefix(s.moviesTarget, ds.Path) {
			diskStats = append(diskStats, ds)
		}
	}
	s.diskStats = diskStats
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"strings"
)

func ExtractColumn(logContents string, column uint8) string {
	var buffer bytes.Buffer
	scanner := bufio.NewScanner(strings.NewReader(logContents))

	for scanner.Scan() {
		line := scanner.Text()

		if line == "" {
			continue
		}

		cols := strings.Split(line, " ")

		switch column {
		case 0:
			buffer.WriteString(cols[0])
			buffer.WriteString(" ")
			buffer.WriteString(cols[1])
		case 1:
			buffer.WriteString(cols[2])
		case 2:
			buffer.WriteString(cols[3])

			for j := 4; j < len(cols); j++ {
				buffer.WriteString(" ")
				buffer.WriteString(cols[j])
			}
		}

		buffer.WriteString("\n")
	}

	return buffer.String()
}
<commit_msg>fix: took `len' function call out of the for loop condition<commit_after>package main

import (
	"bufio"
	"bytes"
	"strings"
)

func ExtractColumn(logContents string, column uint8) string {
	var buffer bytes.Buffer
	scanner := bufio.NewScanner(strings.NewReader(logContents))

	for scanner.Scan() {
		line := scanner.Text()

		if line == "" {
			continue
		}

		cols := strings.Split(line, " ")

		switch column {
		case 0:
			buffer.WriteString(cols[0])
			buffer.WriteString(" ")
			buffer.WriteString(cols[1])
		case 1:
			buffer.WriteString(cols[2])
		case 2:
			buffer.WriteString(cols[3])
            colCount := len(cols)

			for j := 4; j < colCount; j++ {
				buffer.WriteString(" ")
				buffer.WriteString(cols[j])
			}
		}

		buffer.WriteString("\n")
	}

	return buffer.String()
}
<|endoftext|>


<commit_before>package onvif

import (
	"errors"
	"net"
	"regexp"
	"strings"
	"time"

	"github.com/clbanning/mxj"
	"github.com/satori/go.uuid"
)

var errWrongDiscoveryResponse = errors.New("Response is not related to discovery request")

// StartDiscovery send a WS-Discovery message and wait for all matching device to respond
func StartDiscovery(duration time.Duration) ([]Device, error) {
	// Create initial discovery results
	discoveryResults := []Device{}

	// Create WS-Discovery request
	requestID := "uuid:" + uuid.NewV4().String()
	request := `<?xml version="1.0" encoding="UTF-8"?>
		<s:Envelope
			xmlns:s="http://www.w3.org/2003/05/soap-envelope"
			xmlns:a="http://schemas.xmlsoap.org/ws/2004/08/addressing">
			<s:Header>
				<a:Action s:mustUnderstand="1">http://schemas.xmlsoap.org/ws/2005/04/discovery/Probe</a:Action>
				<a:MessageID>` + requestID + `</a:MessageID>
				<a:ReplyTo>
					<a:Address>http://schemas.xmlsoap.org/ws/2004/08/addressing/role/anonymous</a:Address>
				</a:ReplyTo>
				<a:To s:mustUnderstand="1">urn:schemas-xmlsoap-org:ws:2005:04:discovery</a:To>
			</s:Header>
			<s:Body>
				<Probe
					xmlns="http://schemas.xmlsoap.org/ws/2005/04/discovery">
					<d:Types
						xmlns:d="http://schemas.xmlsoap.org/ws/2005/04/discovery"
						xmlns:dp0="http://www.onvif.org/ver10/network/wsdl">dp0:NetworkVideoTransmitter
					</d:Types>
				</Probe>
			</s:Body>
		</s:Envelope>`

	// Clean WS-Discovery message
	request = regexp.MustCompile(`\>\s+\<`).ReplaceAllString(request, "><")
	request = regexp.MustCompile(`\s+`).ReplaceAllString(request, " ")

	// Create UDP address for local and multicast address
	localAddress, err := net.ResolveUDPAddr("udp4", ":0")
	if err != nil {
		return discoveryResults, err
	}

	multicastAddress, err := net.ResolveUDPAddr("udp4", "239.255.255.250:3702")
	if err != nil {
		return discoveryResults, err
	}

	// Create UDP connection to listen for respond from matching device
	conn, err := net.ListenUDP("udp", localAddress)
	if err != nil {
		return discoveryResults, err
	}
	defer conn.Close()

	// Set connection's timeout
	err = conn.SetDeadline(time.Now().Add(duration))
	if err != nil {
		return discoveryResults, err
	}

	// Send WS-Discovery request to multicast address
	_, err = conn.WriteToUDP([]byte(request), multicastAddress)
	if err != nil {
		return discoveryResults, err
	}

	// Keep reading UDP message until timeout
	for {
		// Create buffer and receive UDP response
		buffer := make([]byte, 10*1024)
		_, _, err = conn.ReadFromUDP(buffer)

		// Check if connection timeout
		if err != nil {
			if udpErr, ok := err.(net.Error); ok && udpErr.Timeout() {
				break
			} else {
				return discoveryResults, err
			}
		}

		// Read and parse WS-Discovery response
		device, err := readDiscoveryResponse(requestID, buffer)
		if err != nil && err != errWrongDiscoveryResponse {
			return discoveryResults, err
		}

		// Push device to results
		discoveryResults = append(discoveryResults, device)
	}

	return discoveryResults, nil
}

// readDiscoveryResponse reads and parses WS-Discovery response
func readDiscoveryResponse(messageID string, buffer []byte) (Device, error) {
	// Inital result
	result := Device{}

	// Parse XML to map
	mapXML, err := mxj.NewMapXml(buffer)
	if err != nil {
		return result, err
	}

	// Check if this response is for our request
	responseMessageID, _ := mapXML.ValueForPathString("Envelope.Header.RelatesTo")
	if responseMessageID != messageID {
		return result, errWrongDiscoveryResponse
	}

	// Get device's ID and clean it
	deviceID, _ := mapXML.ValueForPathString("Envelope.Body.ProbeMatches.ProbeMatch.EndpointReference.Address")
	deviceID = strings.Replace(deviceID, "urn:uuid", "", 1)

	// Get device's name
	deviceName := ""
	scopes, _ := mapXML.ValueForPathString("Envelope.Body.ProbeMatches.ProbeMatch.Scopes")
	for _, scope := range strings.Split(scopes, " ") {
		if strings.HasPrefix(scope, "onvif://www.onvif.org/name/") {
			deviceName = strings.Replace(scope, "onvif://www.onvif.org/name/", "", 1)
			deviceName = strings.Replace(deviceName, "_", " ", -1)
			break
		}
	}

	// Get device's xAddrs
	xAddrs, _ := mapXML.ValueForPathString("Envelope.Body.ProbeMatches.ProbeMatch.XAddrs")
	listXAddr := strings.Split(xAddrs, " ")
	if len(listXAddr) == 0 {
		return result, errors.New("Device does not have any xAddr")
	}

	// Finalize result
	result.ID = deviceID
	result.Name = deviceName
	result.XAddr = listXAddr[0]

	return result, nil
}
<commit_msg>Update soap message following guide from Onvif<commit_after>package onvif

import (
	"errors"
	"net"
	"regexp"
	"strings"
	"time"

	"github.com/clbanning/mxj"
	"github.com/satori/go.uuid"
)

var errWrongDiscoveryResponse = errors.New("Response is not related to discovery request")

// StartDiscovery send a WS-Discovery message and wait for all matching device to respond
func StartDiscovery(duration time.Duration) ([]Device, error) {
	// Create initial discovery results
	discoveryResults := []Device{}

	// Create WS-Discovery request
	requestID := "uuid:" + uuid.NewV4().String()
	request := `
		<?xml version="1.0" encoding="UTF-8"?>
		<e:Envelope
		    xmlns:e="http://www.w3.org/2003/05/soap-envelope"
		    xmlns:w="http://schemas.xmlsoap.org/ws/2004/08/addressing"
		    xmlns:d="http://schemas.xmlsoap.org/ws/2005/04/discovery"
		    xmlns:dn="http://www.onvif.org/ver10/network/wsdl">
		    <e:Header>
		        <w:MessageID>` + requestID + `</w:MessageID>
		        <w:To e:mustUnderstand="true">urn:schemas-xmlsoap-org:ws:2005:04:discovery</w:To>
		        <w:Action a:mustUnderstand="true">http://schemas.xmlsoap.org/ws/2005/04/discovery/Probe
		        </w:Action>
		    </e:Header>
		    <e:Body>
		        <d:Probe>
		            <d:Types>dn:NetworkVideoTransmitter</d:Types>
		        </d:Probe>
		    </e:Body>
		</e:Envelope>`

	// Clean WS-Discovery message
	request = regexp.MustCompile(`\>\s+\<`).ReplaceAllString(request, "><")
	request = regexp.MustCompile(`\s+`).ReplaceAllString(request, " ")

	// Create UDP address for local and multicast address
	localAddress, err := net.ResolveUDPAddr("udp4", ":0")
	if err != nil {
		return discoveryResults, err
	}

	multicastAddress, err := net.ResolveUDPAddr("udp4", "239.255.255.250:3702")
	if err != nil {
		return discoveryResults, err
	}

	// Create UDP connection to listen for respond from matching device
	conn, err := net.ListenUDP("udp", localAddress)
	if err != nil {
		return discoveryResults, err
	}
	defer conn.Close()

	// Set connection's timeout
	err = conn.SetDeadline(time.Now().Add(duration))
	if err != nil {
		return discoveryResults, err
	}

	// Send WS-Discovery request to multicast address
	_, err = conn.WriteToUDP([]byte(request), multicastAddress)
	if err != nil {
		return discoveryResults, err
	}

	// Keep reading UDP message until timeout
	for {
		// Create buffer and receive UDP response
		buffer := make([]byte, 10*1024)
		_, _, err = conn.ReadFromUDP(buffer)

		// Check if connection timeout
		if err != nil {
			if udpErr, ok := err.(net.Error); ok && udpErr.Timeout() {
				break
			} else {
				return discoveryResults, err
			}
		}

		// Read and parse WS-Discovery response
		device, err := readDiscoveryResponse(requestID, buffer)
		if err != nil && err != errWrongDiscoveryResponse {
			return discoveryResults, err
		}

		// Push device to results
		discoveryResults = append(discoveryResults, device)
	}

	return discoveryResults, nil
}

// readDiscoveryResponse reads and parses WS-Discovery response
func readDiscoveryResponse(messageID string, buffer []byte) (Device, error) {
	// Inital result
	result := Device{}

	// Parse XML to map
	mapXML, err := mxj.NewMapXml(buffer)
	if err != nil {
		return result, err
	}

	// Check if this response is for our request
	responseMessageID, _ := mapXML.ValueForPathString("Envelope.Header.RelatesTo")
	if responseMessageID != messageID {
		return result, errWrongDiscoveryResponse
	}

	// Get device's ID and clean it
	deviceID, _ := mapXML.ValueForPathString("Envelope.Body.ProbeMatches.ProbeMatch.EndpointReference.Address")
	deviceID = strings.Replace(deviceID, "urn:uuid", "", 1)

	// Get device's name
	deviceName := ""
	scopes, _ := mapXML.ValueForPathString("Envelope.Body.ProbeMatches.ProbeMatch.Scopes")
	for _, scope := range strings.Split(scopes, " ") {
		if strings.HasPrefix(scope, "onvif://www.onvif.org/name/") {
			deviceName = strings.Replace(scope, "onvif://www.onvif.org/name/", "", 1)
			deviceName = strings.Replace(deviceName, "_", " ", -1)
			break
		}
	}

	// Get device's xAddrs
	xAddrs, _ := mapXML.ValueForPathString("Envelope.Body.ProbeMatches.ProbeMatch.XAddrs")
	listXAddr := strings.Split(xAddrs, " ")
	if len(listXAddr) == 0 {
		return result, errors.New("Device does not have any xAddr")
	}

	// Finalize result
	result.ID = deviceID
	result.Name = deviceName
	result.XAddr = listXAddr[0]

	return result, nil
}
<|endoftext|>


<commit_before>// +build freebsd linux darwin

package gopsutil

import "syscall"

func DiskUsage(path string) (*DiskUsageStat, error) {
	stat := syscall.Statfs_t{}
	err := syscall.Statfs(path, &stat)
	if err != nil {
		return nil, err
	}

	bsize := stat.Bsize / 512

	ret := &DiskUsageStat{
		Path:        path,
		Total:       (uint64(stat.Blocks) * uint64(bsize)) >> 1,
		Free:        (uint64(stat.Bfree) * uint64(bsize)) >> 1,
		InodesTotal: (uint64(stat.Files)),
		InodesFree:  (uint64(stat.Ffree)),
	}

	ret.InodesUsed = (ret.InodesTotal - ret.InodesFree)
	ret.InodesUsedPercent = (float64(ret.InodesUsed) / float64(ret.InodesTotal)) * 100.0
	ret.Used = (ret.Total - ret.Free)
	ret.UsedPercent = (float64(ret.Used) / float64(ret.Total)) * 100.0

	return ret, nil
}
<commit_msg>unix disk usage in bytes<commit_after>// +build freebsd linux darwin

package gopsutil

import "syscall"

func DiskUsage(path string) (*DiskUsageStat, error) {
	stat := syscall.Statfs_t{}
	err := syscall.Statfs(path, &stat)
	if err != nil {
		return nil, err
	}

	bsize := stat.Bsize

	ret := &DiskUsageStat{
		Path:        path,
		Total:       (uint64(stat.Blocks) * uint64(bsize)),
		Free:        (uint64(stat.Bfree) * uint64(bsize)),
		InodesTotal: (uint64(stat.Files)),
		InodesFree:  (uint64(stat.Ffree)),
	}

	ret.InodesUsed = (ret.InodesTotal - ret.InodesFree)
	ret.InodesUsedPercent = (float64(ret.InodesUsed) / float64(ret.InodesTotal)) * 100.0
	ret.Used = (ret.Total - ret.Free)
	ret.UsedPercent = (float64(ret.Used) / float64(ret.Total)) * 100.0

	return ret, nil
}
<|endoftext|>


<commit_before>// +build linux
// +build cgo

package main

/*
#define _GNU_SOURCE
#define _FILE_OFFSET_BITS 64
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <linux/loop.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/mount.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

#ifndef LO_FLAGS_AUTOCLEAR
#define LO_FLAGS_AUTOCLEAR 4
#endif

static int get_unused_loop_dev_legacy(char *loop_name)
{
	struct dirent *dp;
	struct loop_info64 lo64;
	DIR *dir;
	int dfd = -1, fd = -1, ret = -1;

	dir = opendir("/dev");
	if (!dir)
		return -1;

	while ((dp = readdir(dir))) {
		if (!dp)
			break;

		if (strncmp(dp->d_name, "loop", 4) != 0)
			continue;

		dfd = dirfd(dir);
		if (dfd < 0)
			continue;

		fd = openat(dfd, dp->d_name, O_RDWR);
		if (fd < 0)
			continue;

		ret = ioctl(fd, LOOP_GET_STATUS64, &lo64);
		if (ret < 0)

			if (ioctl(fd, LOOP_GET_STATUS64, &lo64) == 0 ||
			    errno != ENXIO) {
				close(fd);
				fd = -1;
				continue;
			}

		ret = snprintf(loop_name, LO_NAME_SIZE, "/dev/%s", dp->d_name);
		if (ret < 0 || ret >= LO_NAME_SIZE) {
			close(fd);
			fd = -1;
			continue;
		}

		break;
	}

	closedir(dir);

	if (fd < 0)
		return -1;

	return fd;
}

static int get_unused_loop_dev(char *name_loop)
{
	int loop_nr, ret;
	int fd_ctl = -1, fd_tmp = -1;

	fd_ctl = open("/dev/loop-control", O_RDWR | O_CLOEXEC);
	if (fd_ctl < 0)
		return -ENODEV;

	loop_nr = ioctl(fd_ctl, LOOP_CTL_GET_FREE);
	if (loop_nr < 0)
		goto on_error;

	ret = snprintf(name_loop, LO_NAME_SIZE, "/dev/loop%d", loop_nr);
	if (ret < 0 || ret >= LO_NAME_SIZE)
		goto on_error;

	fd_tmp = open(name_loop, O_RDWR | O_CLOEXEC);
	if (fd_tmp < 0)
		goto on_error;

on_error:
	close(fd_ctl);
	return fd_tmp;
}

int prepare_loop_dev(const char *source, char *loop_dev)
{
	int ret;
	struct loop_info64 lo64;
	int fd_img = -1, fret = -1, fd_loop = -1;

	fd_loop = get_unused_loop_dev(loop_dev);
	if (fd_loop < 0) {
		if (fd_loop == -ENODEV)
			fd_loop = get_unused_loop_dev_legacy(loop_dev);
		else
			goto on_error;
	}

	fd_img = open(source, O_RDWR | O_CLOEXEC);
	if (fd_img < 0)
		goto on_error;

	ret = ioctl(fd_loop, LOOP_SET_FD, fd_img);
	if (ret < 0)
		goto on_error;

	memset(&lo64, 0, sizeof(lo64));
	lo64.lo_flags = LO_FLAGS_AUTOCLEAR;

	ret = ioctl(fd_loop, LOOP_SET_STATUS64, &lo64);
	if (ret < 0)
		goto on_error;

	fret = 0;

on_error:
	if (fd_img >= 0)
		close(fd_img);

	if (fret < 0 && fd_loop >= 0) {
		close(fd_loop);
		fd_loop = -1;
	}

	return fd_loop;
}
*/
import "C"

import (
	"fmt"
	"os"
	"unsafe"
)

// prepareLoopDev() detects and sets up a loop device for source. It returns an
// open file descriptor to the free loop device and the path of the free loop
// device. It's the callers responsibility to close the open file descriptor.
func prepareLoopDev(source string) (*os.File, error) {
	cLoopDev := C.malloc(C.size_t(C.LO_NAME_SIZE))
	if cLoopDev == nil {
		return nil, fmt.Errorf("Failed to allocate memory in C.")
	}
	defer C.free(cLoopDev)

	cSource := C.CString(source)
	defer C.free(unsafe.Pointer(cSource))
	loopFd := int(C.prepare_loop_dev(cSource, (*C.char)(cLoopDev)))
	if loopFd < 0 {
		return nil, fmt.Errorf("Failed to prepare loop device.")
	}

	return os.NewFile(uintptr(loopFd), C.GoString((*C.char)(cLoopDev))), nil
}
<commit_msg>storage_cgo: detect if loop file is already in use<commit_after>// +build linux
// +build cgo

package main

/*
#define _GNU_SOURCE
#define _FILE_OFFSET_BITS 64
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <linux/loop.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/mount.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

#ifndef LO_FLAGS_AUTOCLEAR
#define LO_FLAGS_AUTOCLEAR 4
#endif

#define LXD_MAXPATH 4096
#define LXD_NUMSTRLEN64 21
#define LXD_MAX_LOOP_PATHLEN (2 * sizeof("loop/")) + LXD_NUMSTRLEN64 + sizeof("backing_file") + 1

// If a loop file is already associated with a loop device, find it.
static int find_associated_loop_device(const char *loop_file,
				       char *loop_dev_name)
{
	char looppath[LXD_MAX_LOOP_PATHLEN];
	char buf[LXD_MAXPATH];
	struct dirent *dp;
	DIR *dir;
	int dfd = -1, fd = -1;

	dir = opendir("/sys/block");
	if (!dir)
		return -1;

	while ((dp = readdir(dir))) {
		int ret = -1;
		size_t totlen;
		struct stat fstatbuf;
		char *delsuffix = " (deleted)";
		size_t dellen = sizeof(delsuffix);

		if (!dp)
			break;

		if (strncmp(dp->d_name, "loop", 4) != 0)
			continue;

		dfd = dirfd(dir);
		if (dfd < 0)
			continue;

		ret = snprintf(looppath, sizeof(looppath),
			       "%s/loop/backing_file", dp->d_name);
		if (ret < 0 || (size_t)ret >= sizeof(looppath))
			continue;

		ret = fstatat(dfd, looppath, &fstatbuf, 0);
		if (ret < 0)
			continue;

		fd = openat(dfd, looppath, O_RDONLY | O_CLOEXEC, 0);
		if (ret < 0)
			continue;

		// Clear buffer.
		memset(buf, 0, sizeof(buf));
		ret = read(fd, buf, sizeof(buf));
		if (ret < 0)
			continue;

		totlen = strlen(buf);
		// Trim newline.
		if (buf[totlen - 1] == '\n') {
			buf[totlen - 1] = '\0';
			totlen--;
		}

		if (totlen > dellen) {
			char *deleted = &buf[totlen - dellen];

			// Skip deleted loop files.
			if (!strcmp(deleted, delsuffix))
				continue;
		}

		if (strcmp(buf, loop_file)) {
			close(fd);
			fd = -1;
			continue;
		}

		ret = snprintf(loop_dev_name, LO_NAME_SIZE, "/dev/%s",
			       dp->d_name);
		if (ret < 0 || ret >= LO_NAME_SIZE) {
			close(fd);
			fd = -1;
			continue;
		}

		break;
	}

	closedir(dir);

	if (fd < 0)
		return -1;

	return fd;
}

static int get_unused_loop_dev_legacy(char *loop_name)
{
	struct dirent *dp;
	struct loop_info64 lo64;
	DIR *dir;
	int dfd = -1, fd = -1, ret = -1;

	dir = opendir("/dev");
	if (!dir)
		return -1;

	while ((dp = readdir(dir))) {
		if (!dp)
			break;

		if (strncmp(dp->d_name, "loop", 4) != 0)
			continue;

		dfd = dirfd(dir);
		if (dfd < 0)
			continue;

		fd = openat(dfd, dp->d_name, O_RDWR);
		if (fd < 0)
			continue;

		ret = ioctl(fd, LOOP_GET_STATUS64, &lo64);
		if (ret < 0) {
			if (ioctl(fd, LOOP_GET_STATUS64, &lo64) == 0 ||
			    errno != ENXIO) {
				close(fd);
				fd = -1;
				continue;
			}
		}

		ret = snprintf(loop_name, LO_NAME_SIZE, "/dev/%s", dp->d_name);
		if (ret < 0 || ret >= LO_NAME_SIZE) {
			close(fd);
			fd = -1;
			continue;
		}

		break;
	}

	closedir(dir);

	if (fd < 0)
		return -1;

	return fd;
}

static int get_unused_loop_dev(char *name_loop)
{
	int loop_nr, ret;
	int fd_ctl = -1, fd_tmp = -1;

	fd_ctl = open("/dev/loop-control", O_RDWR | O_CLOEXEC);
	if (fd_ctl < 0)
		return -ENODEV;

	loop_nr = ioctl(fd_ctl, LOOP_CTL_GET_FREE);
	if (loop_nr < 0)
		goto on_error;

	ret = snprintf(name_loop, LO_NAME_SIZE, "/dev/loop%d", loop_nr);
	if (ret < 0 || ret >= LO_NAME_SIZE)
		goto on_error;

	fd_tmp = open(name_loop, O_RDWR | O_CLOEXEC);
	if (fd_tmp < 0)
		goto on_error;

on_error:
	close(fd_ctl);
	return fd_tmp;
}

int prepare_loop_dev(const char *source, char *loop_dev)
{
	int ret;
	struct loop_info64 lo64;
	int fd_img = -1, fret = -1, fd_loop = -1;

	fd_loop = get_unused_loop_dev(loop_dev);
	if (fd_loop < 0) {
		if (fd_loop == -ENODEV)
			fd_loop = get_unused_loop_dev_legacy(loop_dev);
		else
			goto on_error;
	}

	fd_img = open(source, O_RDWR | O_CLOEXEC);
	if (fd_img < 0)
		goto on_error;

	ret = ioctl(fd_loop, LOOP_SET_FD, fd_img);
	if (ret < 0)
		goto on_error;

	memset(&lo64, 0, sizeof(lo64));
	lo64.lo_flags = LO_FLAGS_AUTOCLEAR;

	ret = ioctl(fd_loop, LOOP_SET_STATUS64, &lo64);
	if (ret < 0)
		goto on_error;

	fret = 0;

on_error:
	if (fd_img >= 0)
		close(fd_img);

	if (fret < 0 && fd_loop >= 0) {
		close(fd_loop);
		fd_loop = -1;
	}

	return fd_loop;
}
*/
import "C"

import (
	"fmt"
	"os"
	"unsafe"
)

// prepareLoopDev() detects and sets up a loop device for source. It returns an
// open file descriptor to the free loop device and the path of the free loop
// device. It's the callers responsibility to close the open file descriptor.
func prepareLoopDev(source string) (*os.File, error) {
	cLoopDev := C.malloc(C.size_t(C.LO_NAME_SIZE))
	if cLoopDev == nil {
		return nil, fmt.Errorf("Failed to allocate memory in C.")
	}
	defer C.free(cLoopDev)

	cSource := C.CString(source)
	defer C.free(unsafe.Pointer(cSource))
	loopFd, _ := C.find_associated_loop_device(cSource, (*C.char)(cLoopDev))
	if loopFd >= 0 {
		return os.NewFile(uintptr(loopFd), C.GoString((*C.char)(cLoopDev))), nil
	}

	loopFd, err := C.prepare_loop_dev(cSource, (*C.char)(cLoopDev))
	if loopFd < 0 {
		if err != nil {
			return nil, fmt.Errorf("Failed to prepare loop device: %s.", err)
		}
		return nil, fmt.Errorf("Failed to prepare loop device.")
	}

	return os.NewFile(uintptr(loopFd), C.GoString((*C.char)(cLoopDev))), nil
}
<|endoftext|>