<commit_before>package objectstore

import (
	"code.google.com/p/go-uuid/uuid"
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/rancher/convoy/convoydriver"
	"github.com/rancher/convoy/metadata"
	"github.com/rancher/convoy/util"
	"io"
	"os"
	"path/filepath"

	. "github.com/rancher/convoy/logging"
)

type BlockMapping struct {
	Offset        int64
	BlockChecksum string
}

type DeltaBlockBackupOperations interface {
	GetVolumeDevice(id string) (string, error)
	HasSnapshot(id, volumeID string) bool
	CompareSnapshot(id, compareID, volumeID string) (*metadata.Mappings, error)
	OpenSnapshot(id, volumeID string) error
	ReadSnapshot(id, volumeID string, start int64, data []byte) error
	CloseSnapshot(id, volumeID string) error
}

const (
	DEFAULT_BLOCK_SIZE = 2097152

	BLOCKS_DIRECTORY      = "blocks"
	BLOCK_SEPARATE_LAYER1 = 2
	BLOCK_SEPARATE_LAYER2 = 4
)

func CreateDeltaBlockBackup(volume *Volume, snapshot *Snapshot, destURL string, sDriver convoydriver.ConvoyDriver) (string, error) {
	deltaOps, ok := sDriver.(DeltaBlockBackupOperations)
	if !ok {
		return "", fmt.Errorf("Driver %s doesn't implemented DeltaBlockBackupOperations interface", sDriver.Name())
	}

	bsDriver, err := GetObjectStoreDriver(destURL)
	if err != nil {
		return "", err
	}

	if err := addVolume(volume, bsDriver); err != nil {
		return "", err
	}

	// Update volume from objectstore
	volume, err = loadVolume(volume.UUID, bsDriver)
	if err != nil {
		return "", err
	}

	lastBackupUUID := volume.LastBackupUUID

	var lastSnapshotUUID string
	var lastBackup *Backup
	if lastBackupUUID != "" {
		lastBackup, err = loadBackup(lastBackupUUID, volume.UUID, bsDriver)
		if err != nil {
			return "", err
		}

		lastSnapshotUUID = lastBackup.SnapshotUUID
		if lastSnapshotUUID == snapshot.UUID {
			//Generate full snapshot if the snapshot has been backed up last time
			lastSnapshotUUID = ""
			log.Debug("Would create full snapshot metadata")
		} else if !deltaOps.HasSnapshot(lastSnapshotUUID, volume.UUID) {
			// It's possible that the snapshot in objectstore doesn't exist
			// in local storage
			lastSnapshotUUID = ""
			log.WithFields(logrus.Fields{
				LOG_FIELD_REASON:   LOG_REASON_FALLBACK,
				LOG_FIELD_OBJECT:   LOG_OBJECT_SNAPSHOT,
				LOG_FIELD_SNAPSHOT: lastSnapshotUUID,
				LOG_FIELD_VOLUME:   volume.UUID,
			}).Debug("Cannot find last snapshot in local storage, would process with full backup")
		}
	}

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:        LOG_REASON_START,
		LOG_FIELD_OBJECT:        LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_EVENT:         LOG_EVENT_COMPARE,
		LOG_FIELD_SNAPSHOT:      snapshot.UUID,
		LOG_FIELD_LAST_SNAPSHOT: lastSnapshotUUID,
	}).Debug("Generating snapshot changed blocks metadata")
	delta, err := deltaOps.CompareSnapshot(snapshot.UUID, lastSnapshotUUID, volume.UUID)
	if err != nil {
		return "", err
	}
	if delta.BlockSize != DEFAULT_BLOCK_SIZE {
		return "", fmt.Errorf("Currently doesn't support different block sizes driver other than %v", DEFAULT_BLOCK_SIZE)
	}
	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:        LOG_REASON_COMPLETE,
		LOG_FIELD_OBJECT:        LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_EVENT:         LOG_EVENT_COMPARE,
		LOG_FIELD_SNAPSHOT:      snapshot.UUID,
		LOG_FIELD_LAST_SNAPSHOT: lastSnapshotUUID,
	}).Debug("Generated snapshot changed blocks metadata")

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:   LOG_REASON_START,
		LOG_FIELD_EVENT:    LOG_EVENT_BACKUP,
		LOG_FIELD_OBJECT:   LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_SNAPSHOT: snapshot.UUID,
	}).Debug("Creating backup")

	deltaBackup := &Backup{
		UUID:         uuid.New(),
		VolumeUUID:   volume.UUID,
		SnapshotUUID: snapshot.UUID,
		Blocks:       []BlockMapping{},
	}
	if err := deltaOps.OpenSnapshot(snapshot.UUID, volume.UUID); err != nil {
		return "", err
	}
	defer deltaOps.CloseSnapshot(snapshot.UUID, volume.UUID)
	for _, d := range delta.Mappings {
		block := make([]byte, DEFAULT_BLOCK_SIZE)
		for i := int64(0); i < d.Size/delta.BlockSize; i++ {
			offset := d.Offset + i*delta.BlockSize
			err := deltaOps.ReadSnapshot(snapshot.UUID, volume.UUID, offset, block)
			if err != nil {
				return "", err
			}
			checksum := util.GetChecksum(block)
			blkFile := getBlockFilePath(volume.UUID, checksum)
			if bsDriver.FileSize(blkFile) >= 0 {
				blockMapping := BlockMapping{
					Offset:        offset,
					BlockChecksum: checksum,
				}
				deltaBackup.Blocks = append(deltaBackup.Blocks, blockMapping)
				log.Debugf("Found existed block match at %v", blkFile)
				continue
			}

			rs, err := util.CompressData(block)
			if err != nil {
				return "", err
			}

			if err := bsDriver.Write(blkFile, rs); err != nil {
				return "", err
			}
			log.Debugf("Created new block file at %v", blkFile)

			blockMapping := BlockMapping{
				Offset:        offset,
				BlockChecksum: checksum,
			}
			deltaBackup.Blocks = append(deltaBackup.Blocks, blockMapping)
		}
	}

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:   LOG_REASON_COMPLETE,
		LOG_FIELD_EVENT:    LOG_EVENT_BACKUP,
		LOG_FIELD_OBJECT:   LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_SNAPSHOT: snapshot.UUID,
	}).Debug("Created snapshot changed blocks")

	backup := mergeSnapshotMap(deltaBackup, lastBackup)
	backup.SnapshotName = snapshot.Name
	backup.SnapshotCreatedAt = snapshot.CreatedTime
	backup.CreatedTime = util.Now()

	if err := saveBackup(backup, bsDriver); err != nil {
		return "", err
	}

	volume.LastBackupUUID = backup.UUID
	if err := saveVolume(volume, bsDriver); err != nil {
		return "", err
	}

	return encodeBackupURL(backup.UUID, volume.UUID, destURL), nil
}

func mergeSnapshotMap(deltaBackup, lastBackup *Backup) *Backup {
	if lastBackup == nil {
		return deltaBackup
	}
	backup := &Backup{
		UUID:         deltaBackup.UUID,
		VolumeUUID:   deltaBackup.VolumeUUID,
		SnapshotUUID: deltaBackup.SnapshotUUID,
		Blocks:       []BlockMapping{},
	}
	var d, l int
	for d, l = 0, 0; d < len(deltaBackup.Blocks) && l < len(lastBackup.Blocks); {
		dB := deltaBackup.Blocks[d]
		lB := lastBackup.Blocks[l]
		if dB.Offset == lB.Offset {
			backup.Blocks = append(backup.Blocks, dB)
			d++
			l++
		} else if dB.Offset < lB.Offset {
			backup.Blocks = append(backup.Blocks, dB)
			d++
		} else {
			//dB.Offset > lB.offset
			backup.Blocks = append(backup.Blocks, lB)
			l++
		}
	}

	if d == len(deltaBackup.Blocks) {
		backup.Blocks = append(backup.Blocks, lastBackup.Blocks[l:]...)
	} else {
		backup.Blocks = append(backup.Blocks, deltaBackup.Blocks[d:]...)
	}

	return backup
}

func RestoreDeltaBlockBackup(backupURL, volDevName string) error {
	bsDriver, err := GetObjectStoreDriver(backupURL)
	if err != nil {
		return err
	}

	srcBackupUUID, srcVolumeUUID, err := decodeBackupURL(backupURL)
	if err != nil {
		return err
	}

	if _, err := loadVolume(srcVolumeUUID, bsDriver); err != nil {
		return generateError(logrus.Fields{
			LOG_FIELD_VOLUME:     srcVolumeUUID,
			LOG_FIELD_BACKUP_URL: backupURL,
		}, "Volume doesn't exist in objectstore: %v", err)
	}

	volDev, err := os.Create(volDevName)
	if err != nil {
		return err
	}
	defer volDev.Close()

	backup, err := loadBackup(srcBackupUUID, srcVolumeUUID, bsDriver)
	if err != nil {
		return err
	}

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:      LOG_REASON_START,
		LOG_FIELD_EVENT:       LOG_EVENT_RESTORE,
		LOG_FIELD_OBJECT:      LOG_FIELD_SNAPSHOT,
		LOG_FIELD_SNAPSHOT:    srcBackupUUID,
		LOG_FIELD_ORIN_VOLUME: srcVolumeUUID,
		LOG_FIELD_VOLUME_DEV:  volDevName,
		LOG_FIELD_BACKUP_URL:  backupURL,
	}).Debug()
	blkCounts := len(backup.Blocks)
	for i, block := range backup.Blocks {
		log.Debugf("Restore for %v: block %v, %v/%v", volDevName, block.BlockChecksum, i+1, blkCounts)
		blkFile := getBlockFilePath(srcVolumeUUID, block.BlockChecksum)
		rc, err := bsDriver.Read(blkFile)
		if err != nil {
			return err
		}
		r, err := util.DecompressAndVerify(rc, block.BlockChecksum)
		rc.Close()
		if err != nil {
			return err
		}
		if _, err := volDev.Seek(block.Offset, 0); err != nil {
			return err
		}
		if _, err := io.CopyN(volDev, r, DEFAULT_BLOCK_SIZE); err != nil {
			return err
		}
	}

	return nil
}

func DeleteDeltaBlockBackup(backupURL string) error {
	bsDriver, err := GetObjectStoreDriver(backupURL)
	if err != nil {
		return err
	}

	backupUUID, volumeUUID, err := decodeBackupURL(backupURL)
	if err != nil {
		return err
	}

	v, err := loadVolume(volumeUUID, bsDriver)
	if err != nil {
		return fmt.Errorf("Cannot find volume %v in objectstore", volumeUUID, err)
	}

	backup, err := loadBackup(backupUUID, volumeUUID, bsDriver)
	if err != nil {
		return err
	}
	discardBlockSet := make(map[string]bool)
	for _, blk := range backup.Blocks {
		discardBlockSet[blk.BlockChecksum] = true
	}
	discardBlockCounts := len(discardBlockSet)

	if err := removeBackup(backup, bsDriver); err != nil {
		return err
	}

	if backup.UUID == v.LastBackupUUID {
		v.LastBackupUUID = ""
		if err := saveVolume(v, bsDriver); err != nil {
			return err
		}
	}

	backupUUIDs, err := getBackupUUIDsForVolume(volumeUUID, bsDriver)
	if err != nil {
		return err
	}
	if len(backupUUIDs) == 0 {
		log.Debugf("No snapshot existed for the volume %v, removing volume", volumeUUID)
		if err := removeVolume(volumeUUID, bsDriver); err != nil {
			log.Warningf("Failed to remove volume %v due to: %v", volumeUUID, err.Error())
		}
		return nil
	}

	log.Debug("GC started")
	for _, backupUUID := range backupUUIDs {
		backup, err := loadBackup(backupUUID, volumeUUID, bsDriver)
		if err != nil {
			return err
		}
		for _, blk := range backup.Blocks {
			if _, exists := discardBlockSet[blk.BlockChecksum]; exists {
				delete(discardBlockSet, blk.BlockChecksum)
				discardBlockCounts--
				if discardBlockCounts == 0 {
					break
				}
			}
		}
		if discardBlockCounts == 0 {
			break
		}
	}

	var blkFileList []string
	for blk := range discardBlockSet {
		blkFileList = append(blkFileList, getBlockFilePath(volumeUUID, blk))
		log.Debugf("Found unused blocks %v for volume %v", blk, volumeUUID)
	}
	if err := bsDriver.Remove(blkFileList...); err != nil {
		return err
	}
	log.Debug("Removed unused blocks for volume ", volumeUUID)

	log.Debug("GC completed")
	log.Debug("Removed objectstore backup ", backupUUID)

	return nil
}

func getBlockPath(volumeUUID string) string {
	return filepath.Join(getVolumePath(volumeUUID), BLOCKS_DIRECTORY) + "/"
}

func getBlockFilePath(volumeUUID, checksum string) string {
	blockSubDirLayer1 := checksum[0:BLOCK_SEPARATE_LAYER1]
	blockSubDirLayer2 := checksum[BLOCK_SEPARATE_LAYER1:BLOCK_SEPARATE_LAYER2]
	path := filepath.Join(getBlockPath(volumeUUID), blockSubDirLayer1, blockSubDirLayer2)
	fileName := checksum + ".blk"

	return filepath.Join(path, fileName)
}
<commit_msg>backup: Add progress info in log for backup<commit_after>package objectstore

import (
	"code.google.com/p/go-uuid/uuid"
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/rancher/convoy/convoydriver"
	"github.com/rancher/convoy/metadata"
	"github.com/rancher/convoy/util"
	"io"
	"os"
	"path/filepath"

	. "github.com/rancher/convoy/logging"
)

type BlockMapping struct {
	Offset        int64
	BlockChecksum string
}

type DeltaBlockBackupOperations interface {
	GetVolumeDevice(id string) (string, error)
	HasSnapshot(id, volumeID string) bool
	CompareSnapshot(id, compareID, volumeID string) (*metadata.Mappings, error)
	OpenSnapshot(id, volumeID string) error
	ReadSnapshot(id, volumeID string, start int64, data []byte) error
	CloseSnapshot(id, volumeID string) error
}

const (
	DEFAULT_BLOCK_SIZE = 2097152

	BLOCKS_DIRECTORY      = "blocks"
	BLOCK_SEPARATE_LAYER1 = 2
	BLOCK_SEPARATE_LAYER2 = 4
)

func CreateDeltaBlockBackup(volume *Volume, snapshot *Snapshot, destURL string, sDriver convoydriver.ConvoyDriver) (string, error) {
	deltaOps, ok := sDriver.(DeltaBlockBackupOperations)
	if !ok {
		return "", fmt.Errorf("Driver %s doesn't implemented DeltaBlockBackupOperations interface", sDriver.Name())
	}

	bsDriver, err := GetObjectStoreDriver(destURL)
	if err != nil {
		return "", err
	}

	if err := addVolume(volume, bsDriver); err != nil {
		return "", err
	}

	// Update volume from objectstore
	volume, err = loadVolume(volume.UUID, bsDriver)
	if err != nil {
		return "", err
	}

	lastBackupUUID := volume.LastBackupUUID

	var lastSnapshotUUID string
	var lastBackup *Backup
	if lastBackupUUID != "" {
		lastBackup, err = loadBackup(lastBackupUUID, volume.UUID, bsDriver)
		if err != nil {
			return "", err
		}

		lastSnapshotUUID = lastBackup.SnapshotUUID
		if lastSnapshotUUID == snapshot.UUID {
			//Generate full snapshot if the snapshot has been backed up last time
			lastSnapshotUUID = ""
			log.Debug("Would create full snapshot metadata")
		} else if !deltaOps.HasSnapshot(lastSnapshotUUID, volume.UUID) {
			// It's possible that the snapshot in objectstore doesn't exist
			// in local storage
			lastSnapshotUUID = ""
			log.WithFields(logrus.Fields{
				LOG_FIELD_REASON:   LOG_REASON_FALLBACK,
				LOG_FIELD_OBJECT:   LOG_OBJECT_SNAPSHOT,
				LOG_FIELD_SNAPSHOT: lastSnapshotUUID,
				LOG_FIELD_VOLUME:   volume.UUID,
			}).Debug("Cannot find last snapshot in local storage, would process with full backup")
		}
	}

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:        LOG_REASON_START,
		LOG_FIELD_OBJECT:        LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_EVENT:         LOG_EVENT_COMPARE,
		LOG_FIELD_SNAPSHOT:      snapshot.UUID,
		LOG_FIELD_LAST_SNAPSHOT: lastSnapshotUUID,
	}).Debug("Generating snapshot changed blocks metadata")
	delta, err := deltaOps.CompareSnapshot(snapshot.UUID, lastSnapshotUUID, volume.UUID)
	if err != nil {
		return "", err
	}
	if delta.BlockSize != DEFAULT_BLOCK_SIZE {
		return "", fmt.Errorf("Currently doesn't support different block sizes driver other than %v", DEFAULT_BLOCK_SIZE)
	}
	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:        LOG_REASON_COMPLETE,
		LOG_FIELD_OBJECT:        LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_EVENT:         LOG_EVENT_COMPARE,
		LOG_FIELD_SNAPSHOT:      snapshot.UUID,
		LOG_FIELD_LAST_SNAPSHOT: lastSnapshotUUID,
	}).Debug("Generated snapshot changed blocks metadata")

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:   LOG_REASON_START,
		LOG_FIELD_EVENT:    LOG_EVENT_BACKUP,
		LOG_FIELD_OBJECT:   LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_SNAPSHOT: snapshot.UUID,
	}).Debug("Creating backup")

	deltaBackup := &Backup{
		UUID:         uuid.New(),
		VolumeUUID:   volume.UUID,
		SnapshotUUID: snapshot.UUID,
		Blocks:       []BlockMapping{},
	}
	if err := deltaOps.OpenSnapshot(snapshot.UUID, volume.UUID); err != nil {
		return "", err
	}
	defer deltaOps.CloseSnapshot(snapshot.UUID, volume.UUID)
	mCounts := len(delta.Mappings)
	for m, d := range delta.Mappings {
		block := make([]byte, DEFAULT_BLOCK_SIZE)
		blkCounts := d.Size / delta.BlockSize
		for i := int64(0); i < blkCounts; i++ {
			offset := d.Offset + i*delta.BlockSize
			log.Debugf("Backup for %v: segment %v/%v, blocks %v/%v", snapshot.UUID, m+1, mCounts, i+1, blkCounts)
			err := deltaOps.ReadSnapshot(snapshot.UUID, volume.UUID, offset, block)
			if err != nil {
				return "", err
			}
			checksum := util.GetChecksum(block)
			blkFile := getBlockFilePath(volume.UUID, checksum)
			if bsDriver.FileSize(blkFile) >= 0 {
				blockMapping := BlockMapping{
					Offset:        offset,
					BlockChecksum: checksum,
				}
				deltaBackup.Blocks = append(deltaBackup.Blocks, blockMapping)
				log.Debugf("Found existed block match at %v", blkFile)
				continue
			}

			rs, err := util.CompressData(block)
			if err != nil {
				return "", err
			}

			if err := bsDriver.Write(blkFile, rs); err != nil {
				return "", err
			}
			log.Debugf("Created new block file at %v", blkFile)

			blockMapping := BlockMapping{
				Offset:        offset,
				BlockChecksum: checksum,
			}
			deltaBackup.Blocks = append(deltaBackup.Blocks, blockMapping)
		}
	}

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:   LOG_REASON_COMPLETE,
		LOG_FIELD_EVENT:    LOG_EVENT_BACKUP,
		LOG_FIELD_OBJECT:   LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_SNAPSHOT: snapshot.UUID,
	}).Debug("Created snapshot changed blocks")

	backup := mergeSnapshotMap(deltaBackup, lastBackup)
	backup.SnapshotName = snapshot.Name
	backup.SnapshotCreatedAt = snapshot.CreatedTime
	backup.CreatedTime = util.Now()

	if err := saveBackup(backup, bsDriver); err != nil {
		return "", err
	}

	volume.LastBackupUUID = backup.UUID
	if err := saveVolume(volume, bsDriver); err != nil {
		return "", err
	}

	return encodeBackupURL(backup.UUID, volume.UUID, destURL), nil
}

func mergeSnapshotMap(deltaBackup, lastBackup *Backup) *Backup {
	if lastBackup == nil {
		return deltaBackup
	}
	backup := &Backup{
		UUID:         deltaBackup.UUID,
		VolumeUUID:   deltaBackup.VolumeUUID,
		SnapshotUUID: deltaBackup.SnapshotUUID,
		Blocks:       []BlockMapping{},
	}
	var d, l int
	for d, l = 0, 0; d < len(deltaBackup.Blocks) && l < len(lastBackup.Blocks); {
		dB := deltaBackup.Blocks[d]
		lB := lastBackup.Blocks[l]
		if dB.Offset == lB.Offset {
			backup.Blocks = append(backup.Blocks, dB)
			d++
			l++
		} else if dB.Offset < lB.Offset {
			backup.Blocks = append(backup.Blocks, dB)
			d++
		} else {
			//dB.Offset > lB.offset
			backup.Blocks = append(backup.Blocks, lB)
			l++
		}
	}

	if d == len(deltaBackup.Blocks) {
		backup.Blocks = append(backup.Blocks, lastBackup.Blocks[l:]...)
	} else {
		backup.Blocks = append(backup.Blocks, deltaBackup.Blocks[d:]...)
	}

	return backup
}

func RestoreDeltaBlockBackup(backupURL, volDevName string) error {
	bsDriver, err := GetObjectStoreDriver(backupURL)
	if err != nil {
		return err
	}

	srcBackupUUID, srcVolumeUUID, err := decodeBackupURL(backupURL)
	if err != nil {
		return err
	}

	if _, err := loadVolume(srcVolumeUUID, bsDriver); err != nil {
		return generateError(logrus.Fields{
			LOG_FIELD_VOLUME:     srcVolumeUUID,
			LOG_FIELD_BACKUP_URL: backupURL,
		}, "Volume doesn't exist in objectstore: %v", err)
	}

	volDev, err := os.Create(volDevName)
	if err != nil {
		return err
	}
	defer volDev.Close()

	backup, err := loadBackup(srcBackupUUID, srcVolumeUUID, bsDriver)
	if err != nil {
		return err
	}

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:      LOG_REASON_START,
		LOG_FIELD_EVENT:       LOG_EVENT_RESTORE,
		LOG_FIELD_OBJECT:      LOG_FIELD_SNAPSHOT,
		LOG_FIELD_SNAPSHOT:    srcBackupUUID,
		LOG_FIELD_ORIN_VOLUME: srcVolumeUUID,
		LOG_FIELD_VOLUME_DEV:  volDevName,
		LOG_FIELD_BACKUP_URL:  backupURL,
	}).Debug()
	blkCounts := len(backup.Blocks)
	for i, block := range backup.Blocks {
		log.Debugf("Restore for %v: block %v, %v/%v", volDevName, block.BlockChecksum, i+1, blkCounts)
		blkFile := getBlockFilePath(srcVolumeUUID, block.BlockChecksum)
		rc, err := bsDriver.Read(blkFile)
		if err != nil {
			return err
		}
		r, err := util.DecompressAndVerify(rc, block.BlockChecksum)
		rc.Close()
		if err != nil {
			return err
		}
		if _, err := volDev.Seek(block.Offset, 0); err != nil {
			return err
		}
		if _, err := io.CopyN(volDev, r, DEFAULT_BLOCK_SIZE); err != nil {
			return err
		}
	}

	return nil
}

func DeleteDeltaBlockBackup(backupURL string) error {
	bsDriver, err := GetObjectStoreDriver(backupURL)
	if err != nil {
		return err
	}

	backupUUID, volumeUUID, err := decodeBackupURL(backupURL)
	if err != nil {
		return err
	}

	v, err := loadVolume(volumeUUID, bsDriver)
	if err != nil {
		return fmt.Errorf("Cannot find volume %v in objectstore", volumeUUID, err)
	}

	backup, err := loadBackup(backupUUID, volumeUUID, bsDriver)
	if err != nil {
		return err
	}
	discardBlockSet := make(map[string]bool)
	for _, blk := range backup.Blocks {
		discardBlockSet[blk.BlockChecksum] = true
	}
	discardBlockCounts := len(discardBlockSet)

	if err := removeBackup(backup, bsDriver); err != nil {
		return err
	}

	if backup.UUID == v.LastBackupUUID {
		v.LastBackupUUID = ""
		if err := saveVolume(v, bsDriver); err != nil {
			return err
		}
	}

	backupUUIDs, err := getBackupUUIDsForVolume(volumeUUID, bsDriver)
	if err != nil {
		return err
	}
	if len(backupUUIDs) == 0 {
		log.Debugf("No snapshot existed for the volume %v, removing volume", volumeUUID)
		if err := removeVolume(volumeUUID, bsDriver); err != nil {
			log.Warningf("Failed to remove volume %v due to: %v", volumeUUID, err.Error())
		}
		return nil
	}

	log.Debug("GC started")
	for _, backupUUID := range backupUUIDs {
		backup, err := loadBackup(backupUUID, volumeUUID, bsDriver)
		if err != nil {
			return err
		}
		for _, blk := range backup.Blocks {
			if _, exists := discardBlockSet[blk.BlockChecksum]; exists {
				delete(discardBlockSet, blk.BlockChecksum)
				discardBlockCounts--
				if discardBlockCounts == 0 {
					break
				}
			}
		}
		if discardBlockCounts == 0 {
			break
		}
	}

	var blkFileList []string
	for blk := range discardBlockSet {
		blkFileList = append(blkFileList, getBlockFilePath(volumeUUID, blk))
		log.Debugf("Found unused blocks %v for volume %v", blk, volumeUUID)
	}
	if err := bsDriver.Remove(blkFileList...); err != nil {
		return err
	}
	log.Debug("Removed unused blocks for volume ", volumeUUID)

	log.Debug("GC completed")
	log.Debug("Removed objectstore backup ", backupUUID)

	return nil
}

func getBlockPath(volumeUUID string) string {
	return filepath.Join(getVolumePath(volumeUUID), BLOCKS_DIRECTORY) + "/"
}

func getBlockFilePath(volumeUUID, checksum string) string {
	blockSubDirLayer1 := checksum[0:BLOCK_SEPARATE_LAYER1]
	blockSubDirLayer2 := checksum[BLOCK_SEPARATE_LAYER1:BLOCK_SEPARATE_LAYER2]
	path := filepath.Join(getBlockPath(volumeUUID), blockSubDirLayer1, blockSubDirLayer2)
	fileName := checksum + ".blk"

	return filepath.Join(path, fileName)
}
<|endoftext|>


<commit_before>package docindexing

import (
	"log"
	"runtime"
	"sync"
)

// Highest level inverted index structure
type InvertedIndex struct {
	Terms     map[string]*TermEntry
	TermCount int64
	IndexLock *sync.Mutex
}

// Stores terms and their respective document entries
type TermEntry struct {
	Frequency int
	Documents []*DocumentEntry
	EntryLock *sync.Mutex
}

// Stores document metadata as well as additional term information
type DocumentEntry struct {
	Path      string
	ID        int64
	Frequency int
	Positions []int
}

// Constructs new inverted index
func NewIndex() *InvertedIndex {
	return &InvertedIndex{Terms: make(map[string]*TermEntry), TermCount: int64(0), IndexLock: &sync.Mutex{}}
}

// Adds new term entry to the index
func (ind *InvertedIndex) AddTerm(term string, verbose bool) {
	ind.IndexLock.Lock()
	// Make sure we aren't overwriting existing term
	if _, found := ind.Terms[term]; !found {
		if verbose {
			log.Printf("Adding new term %s to index", term)
		}
		ind.Terms[term] = &TermEntry{Frequency: 0, Documents: make([]*DocumentEntry, 0), EntryLock: &sync.Mutex{}}
		ind.TermCount++
	}
	ind.IndexLock.Unlock()
	runtime.Gosched()
}

// Adds new document entry to given term entry
func (ind *InvertedIndex) AddDocument(term string, document *DocumentEntry, verbose bool) {
	// Make sure term is in index
	ind.AddTerm(term, verbose)
	if verbose {
		log.Printf("Adding new document %s to term %s's document list", document.Path, term)
	}
	// Get index and term locks
	ind.IndexLock.Lock();
	ind.Terms[term].EntryLock.Lock()

	// Safely update values
	ind.Terms[term].Frequency += document.Frequency
	ind.Terms[term].Documents = append(ind.Terms[term].Documents, document)

	// Release locks
	ind.Terms[term].EntryLock.Unlock()
	ind.IndexLock.Unlock()
	runtime.Gosched()
}
<commit_msg>Removed superfluous entry lock<commit_after>package docindexing

import (
	"log"
	"runtime"
	"sync"
)

// Highest level inverted index structure
type InvertedIndex struct {
	Terms     map[string]*TermEntry
	TermCount int64
	IndexLock *sync.Mutex
}

// Stores terms and their respective document entries
type TermEntry struct {
	Frequency int
	Documents []*DocumentEntry
}

// Stores document metadata as well as additional term information
type DocumentEntry struct {
	Path      string
	ID        int64
	Frequency int
	Positions []int
}

// Constructs new inverted index
func NewIndex() *InvertedIndex {
	return &InvertedIndex{Terms: make(map[string]*TermEntry), TermCount: int64(0), IndexLock: &sync.Mutex{}}
}

// Adds new term entry to the index
func (ind *InvertedIndex) AddTerm(term string, verbose bool) {
	ind.IndexLock.Lock()
	// Make sure we aren't overwriting existing term
	if _, found := ind.Terms[term]; !found {
		if verbose {
			log.Printf("Adding new term %s to index", term)
		}
		ind.Terms[term] = &TermEntry{Frequency: 0, Documents: make([]*DocumentEntry, 0)}
		ind.TermCount++
	}
	ind.IndexLock.Unlock()
	runtime.Gosched()
}

// Adds new document entry to given term entry
func (ind *InvertedIndex) AddDocument(term string, document *DocumentEntry, verbose bool) {
	// Make sure term is in index
	ind.AddTerm(term, verbose)
	if verbose {
		log.Printf("Adding new document %s to term %s's document list", document.Path, term)
	}
	// Get index lock
	ind.IndexLock.Lock()

	// Safely update values
	ind.Terms[term].Frequency += document.Frequency
	ind.Terms[term].Documents = append(ind.Terms[term].Documents, document)

	// Release lock
	ind.IndexLock.Unlock()
	runtime.Gosched()
}
<|endoftext|>


<commit_before>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tabletserver

import (
	"fmt"
	"net/url"
	"strings"
	"sync"
	"time"

	log "github.com/golang/glog"
	"github.com/youtube/vitess/go/pools"
	"github.com/youtube/vitess/go/sqldb"
	"github.com/youtube/vitess/go/sqltypes"
	"github.com/youtube/vitess/go/stats"
	"github.com/youtube/vitess/go/streamlog"
	"github.com/youtube/vitess/go/sync2"
	"github.com/youtube/vitess/go/timer"
	"github.com/youtube/vitess/go/vt/callerid"
	querypb "github.com/youtube/vitess/go/vt/proto/query"
	vtrpcpb "github.com/youtube/vitess/go/vt/proto/vtrpc"
	"golang.org/x/net/context"
)

// TxLogger can be used to enable logging of transactions.
// Call TxLogger.ServeLogs in your main program to enable logging.
// The log format can be inferred by looking at TxConnection.Format.
var TxLogger = streamlog.New("TxLog", 10)

// These consts identify how a transaction was resolved.
const (
	TxClose    = "close"
	TxCommit   = "commit"
	TxRollback = "rollback"
	TxPrepare  = "prepare"
	TxKill     = "kill"
)

const txLogInterval = time.Duration(1 * time.Minute)

// TxPool is the transaction pool for the query service.
type TxPool struct {
	pool              *ConnPool
	activePool        *pools.Numbered
	lastID            sync2.AtomicInt64
	timeout           sync2.AtomicDuration
	ticks             *timer.Timer
	txStats           *stats.Timings
	queryServiceStats *QueryServiceStats
	checker           MySQLChecker
	// Tracking culprits that cause tx pool full errors.
	logMu   sync.Mutex
	lastLog time.Time
}

// NewTxPool creates a new TxPool. It's not operational until it's Open'd.
func NewTxPool(
	name string,
	txStatsPrefix string,
	capacity int,
	timeout time.Duration,
	idleTimeout time.Duration,
	enablePublishStats bool,
	qStats *QueryServiceStats,
	checker MySQLChecker) *TxPool {

	txStatsName := ""
	if enablePublishStats {
		txStatsName = txStatsPrefix + "Transactions"
	}

	axp := &TxPool{
		pool:              NewConnPool(name, capacity, idleTimeout, enablePublishStats, qStats, checker),
		activePool:        pools.NewNumbered(),
		lastID:            sync2.NewAtomicInt64(time.Now().UnixNano()),
		timeout:           sync2.NewAtomicDuration(timeout),
		ticks:             timer.NewTimer(timeout / 10),
		txStats:           stats.NewTimings(txStatsName),
		checker:           checker,
		queryServiceStats: qStats,
	}
	// Careful: pool also exports name+"xxx" vars,
	// but we know it doesn't export Timeout.
	if enablePublishStats {
		stats.Publish(name+"Timeout", stats.DurationFunc(axp.timeout.Get))
	}
	return axp
}

// Open makes the TxPool operational. This also starts the transaction killer
// that will kill long-running transactions.
func (axp *TxPool) Open(appParams, dbaParams *sqldb.ConnParams) {
	log.Infof("Starting transaction id: %d", axp.lastID)
	axp.pool.Open(appParams, dbaParams)
	axp.ticks.Start(func() { axp.transactionKiller() })
}

// Close closes the TxPool. A closed pool can be reopened.
func (axp *TxPool) Close() {
	axp.ticks.Stop()
	for _, v := range axp.activePool.GetOutdated(time.Duration(0), "for closing") {
		conn := v.(*TxConnection)
		log.Warningf("killing transaction for shutdown: %s", conn.Format(nil))
		axp.queryServiceStats.InternalErrors.Add("StrayTransactions", 1)
		conn.Close()
		conn.conclude(TxClose)
	}
	axp.pool.Close()
}

// RollbackNonBusy rolls back all transactions that are not in use.
// Transactions can be in use for situations like executing statements
// or in prepared state.
func (axp *TxPool) RollbackNonBusy(ctx context.Context) {
	for _, v := range axp.activePool.GetOutdated(time.Duration(0), "for transition") {
		conn := v.(*TxConnection)
		log.Warningf("rolling back transaction for transition: %s", conn.Format(nil))
		axp.queryServiceStats.InternalErrors.Add("StrayTransactions", 1)
		axp.LocalConclude(ctx, conn)
	}
}

func (axp *TxPool) transactionKiller() {
	defer logError(axp.queryServiceStats)
	for _, v := range axp.activePool.GetOutdated(time.Duration(axp.Timeout()), "for rollback") {
		conn := v.(*TxConnection)
		log.Warningf("killing transaction (exceeded timeout: %v): %s", axp.Timeout(), conn.Format(nil))
		axp.queryServiceStats.KillStats.Add("Transactions", 1)
		conn.Close()
		conn.conclude(TxKill)
	}
}

// WaitForEmpty waits until all active transactions are completed.
func (axp *TxPool) WaitForEmpty() {
	axp.activePool.WaitForEmpty()
}

// Begin begins a transaction, and returns the associated transaction id.
// Subsequent statements can access the connection through the transaction id.
func (axp *TxPool) Begin(ctx context.Context) (int64, error) {
	poolCtx := ctx
	if deadline, ok := ctx.Deadline(); ok {
		var cancel func()
		poolCtx, cancel = context.WithDeadline(ctx, deadline.Add(-10*time.Millisecond))
		defer cancel()
	}
	conn, err := axp.pool.Get(poolCtx)
	if err != nil {
		switch err {
		case ErrConnPoolClosed:
			return 0, err
		case pools.ErrTimeout:
			axp.LogActive()
			return 0, NewTabletError(vtrpcpb.ErrorCode_RESOURCE_EXHAUSTED, "Transaction pool connection limit exceeded")
		}
		return 0, NewTabletErrorSQL(vtrpcpb.ErrorCode_INTERNAL_ERROR, err)
	}
	if _, err := conn.Exec(ctx, "begin", 1, false); err != nil {
		conn.Recycle()
		return 0, NewTabletErrorSQL(vtrpcpb.ErrorCode_UNKNOWN_ERROR, err)
	}
	transactionID := axp.lastID.Add(1)
	axp.activePool.Register(
		transactionID,
		newTxConnection(
			conn,
			transactionID,
			axp,
			callerid.ImmediateCallerIDFromContext(ctx),
			callerid.EffectiveCallerIDFromContext(ctx),
		),
	)
	return transactionID, nil
}

// Commit commits the specified transaction.
func (axp *TxPool) Commit(ctx context.Context, transactionID int64) error {
	conn, err := axp.Get(transactionID, "for commit")
	if err != nil {
		return err
	}
	return axp.LocalCommit(ctx, conn)
}

// Rollback rolls back the specified transaction.
func (axp *TxPool) Rollback(ctx context.Context, transactionID int64) error {
	conn, err := axp.Get(transactionID, "for rollback")
	if err != nil {
		return err
	}
	return axp.localRollback(ctx, conn)
}

// Get fetches the connection associated to the transactionID.
// You must call Recycle on TxConnection once done.
func (axp *TxPool) Get(transactionID int64, reason string) (*TxConnection, error) {
	v, err := axp.activePool.Get(transactionID, reason)
	if err != nil {
		return nil, NewTabletError(vtrpcpb.ErrorCode_NOT_IN_TX, "Transaction %d: %v", transactionID, err)
	}
	return v.(*TxConnection), nil
}

// LocalBegin is equivalent to Begin->Get.
// It's used for executing transactions within a request. It's safe
// to always call LocalConclude at the end.
func (axp *TxPool) LocalBegin(ctx context.Context) (*TxConnection, error) {
	transactionID, err := axp.Begin(ctx)
	if err != nil {
		return nil, err
	}
	return axp.Get(transactionID, "for local query")
}

// LocalCommit is the commit function for LocalBegin.
func (axp *TxPool) LocalCommit(ctx context.Context, conn *TxConnection) error {
	defer conn.conclude(TxCommit)
	axp.txStats.Add("Completed", time.Now().Sub(conn.StartTime))
	if _, err := conn.Exec(ctx, "commit", 1, false); err != nil {
		conn.Close()
		return NewTabletErrorSQL(vtrpcpb.ErrorCode_UNKNOWN_ERROR, err)
	}
	return nil
}

// LocalConclude concludes a transaction started by LocalBegin.
// If the transaction was not previously concluded, it's rolled back.
func (axp *TxPool) LocalConclude(ctx context.Context, conn *TxConnection) {
	if conn.DBConn != nil {
		_ = axp.localRollback(ctx, conn)
	}
}

func (axp *TxPool) localRollback(ctx context.Context, conn *TxConnection) error {
	defer conn.conclude(TxRollback)
	axp.txStats.Add("Aborted", time.Now().Sub(conn.StartTime))
	if _, err := conn.Exec(ctx, "rollback", 1, false); err != nil {
		conn.Close()
		return NewTabletErrorSQL(vtrpcpb.ErrorCode_UNKNOWN_ERROR, err)
	}
	return nil
}

// LogActive causes all existing transactions to be logged when they complete.
// The logging is throttled to no more than once every txLogInterval.
func (axp *TxPool) LogActive() {
	axp.logMu.Lock()
	defer axp.logMu.Unlock()
	if time.Now().Sub(axp.lastLog) < txLogInterval {
		return
	}
	axp.lastLog = time.Now()
	conns := axp.activePool.GetAll()
	for _, c := range conns {
		c.(*TxConnection).LogToFile.Set(1)
	}
}

// Timeout returns the transaction timeout.
func (axp *TxPool) Timeout() time.Duration {
	return axp.timeout.Get()
}

// SetTimeout sets the transaction timeout.
func (axp *TxPool) SetTimeout(timeout time.Duration) {
	axp.timeout.Set(timeout)
	axp.ticks.SetInterval(timeout / 10)
}

// TxConnection is meant for executing transactions. It can return itself to
// the tx pool correctly. It also does not retry statements if there
// are failures.
type TxConnection struct {
	*DBConn
	TransactionID     int64
	pool              *TxPool
	StartTime         time.Time
	EndTime           time.Time
	Queries           []string
	Conclusion        string
	LogToFile         sync2.AtomicInt32
	ImmediateCallerID *querypb.VTGateCallerID
	EffectiveCallerID *vtrpcpb.CallerID
}

func newTxConnection(conn *DBConn, transactionID int64, pool *TxPool, immediate *querypb.VTGateCallerID, effective *vtrpcpb.CallerID) *TxConnection {
	return &TxConnection{
		DBConn:            conn,
		TransactionID:     transactionID,
		pool:              pool,
		StartTime:         time.Now(),
		Queries:           make([]string, 0, 8),
		ImmediateCallerID: immediate,
		EffectiveCallerID: effective,
	}
}

// Exec executes the statement for the current transaction.
func (txc *TxConnection) Exec(ctx context.Context, query string, maxrows int, wantfields bool) (*sqltypes.Result, error) {
	r, err := txc.DBConn.ExecOnce(ctx, query, maxrows, wantfields)
	if err != nil {
		if IsConnErr(err) {
			txc.pool.checker.CheckMySQL()
			return nil, NewTabletErrorSQL(vtrpcpb.ErrorCode_INTERNAL_ERROR, err)
		}
		return nil, NewTabletErrorSQL(vtrpcpb.ErrorCode_UNKNOWN_ERROR, err)
	}
	return r, nil
}

// Recycle returns the connection to the pool. The transaction remains
// active.
func (txc *TxConnection) Recycle() {
	if txc.IsClosed() {
		txc.conclude(TxClose)
	} else {
		txc.pool.activePool.Put(txc.TransactionID)
	}
}

// RecordQuery records the query against this transaction.
func (txc *TxConnection) RecordQuery(query string) {
	txc.Queries = append(txc.Queries, query)
}

func (txc *TxConnection) conclude(conclusion string) {
	txc.pool.activePool.Unregister(txc.TransactionID)
	txc.DBConn.Recycle()
	txc.DBConn = nil
	txc.log(conclusion)
}

func (txc *TxConnection) log(conclusion string) {
	txc.Conclusion = conclusion
	txc.EndTime = time.Now()

	username := callerid.GetPrincipal(txc.EffectiveCallerID)
	if username == "" {
		username = callerid.GetUsername(txc.ImmediateCallerID)
	}
	duration := txc.EndTime.Sub(txc.StartTime)
	txc.pool.queryServiceStats.UserTransactionCount.Add([]string{username, conclusion}, 1)
	txc.pool.queryServiceStats.UserTransactionTimesNs.Add([]string{username, conclusion}, int64(duration))
	if txc.LogToFile.Get() != 0 {
		log.Infof("Logged transaction: %s", txc.Format(nil))
	}
	TxLogger.Send(txc)
}

// EventTime returns the time the event was created.
func (txc *TxConnection) EventTime() time.Time {
	return txc.EndTime
}

// Format returns a printable version of the connection info.
func (txc *TxConnection) Format(params url.Values) string {
	return fmt.Sprintf(
		"%v\t'%v'\t'%v'\t%v\t%v\t%.6f\t%v\t%v\t\n",
		txc.TransactionID,
		callerid.GetPrincipal(txc.EffectiveCallerID),
		callerid.GetUsername(txc.ImmediateCallerID),
		txc.StartTime.Format(time.StampMicro),
		txc.EndTime.Format(time.StampMicro),
		txc.EndTime.Sub(txc.StartTime).Seconds(),
		txc.Conclusion,
		strings.Join(txc.Queries, ";"),
	)
}
<commit_msg>b/32560113: Remove false alarm (#2220)<commit_after>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tabletserver

import (
	"fmt"
	"net/url"
	"strings"
	"sync"
	"time"

	log "github.com/golang/glog"
	"github.com/youtube/vitess/go/pools"
	"github.com/youtube/vitess/go/sqldb"
	"github.com/youtube/vitess/go/sqltypes"
	"github.com/youtube/vitess/go/stats"
	"github.com/youtube/vitess/go/streamlog"
	"github.com/youtube/vitess/go/sync2"
	"github.com/youtube/vitess/go/timer"
	"github.com/youtube/vitess/go/vt/callerid"
	querypb "github.com/youtube/vitess/go/vt/proto/query"
	vtrpcpb "github.com/youtube/vitess/go/vt/proto/vtrpc"
	"golang.org/x/net/context"
)

// TxLogger can be used to enable logging of transactions.
// Call TxLogger.ServeLogs in your main program to enable logging.
// The log format can be inferred by looking at TxConnection.Format.
var TxLogger = streamlog.New("TxLog", 10)

// These consts identify how a transaction was resolved.
const (
	TxClose    = "close"
	TxCommit   = "commit"
	TxRollback = "rollback"
	TxPrepare  = "prepare"
	TxKill     = "kill"
)

const txLogInterval = time.Duration(1 * time.Minute)

// TxPool is the transaction pool for the query service.
type TxPool struct {
	pool              *ConnPool
	activePool        *pools.Numbered
	lastID            sync2.AtomicInt64
	timeout           sync2.AtomicDuration
	ticks             *timer.Timer
	txStats           *stats.Timings
	queryServiceStats *QueryServiceStats
	checker           MySQLChecker
	// Tracking culprits that cause tx pool full errors.
	logMu   sync.Mutex
	lastLog time.Time
}

// NewTxPool creates a new TxPool. It's not operational until it's Open'd.
func NewTxPool(
	name string,
	txStatsPrefix string,
	capacity int,
	timeout time.Duration,
	idleTimeout time.Duration,
	enablePublishStats bool,
	qStats *QueryServiceStats,
	checker MySQLChecker) *TxPool {

	txStatsName := ""
	if enablePublishStats {
		txStatsName = txStatsPrefix + "Transactions"
	}

	axp := &TxPool{
		pool:              NewConnPool(name, capacity, idleTimeout, enablePublishStats, qStats, checker),
		activePool:        pools.NewNumbered(),
		lastID:            sync2.NewAtomicInt64(time.Now().UnixNano()),
		timeout:           sync2.NewAtomicDuration(timeout),
		ticks:             timer.NewTimer(timeout / 10),
		txStats:           stats.NewTimings(txStatsName),
		checker:           checker,
		queryServiceStats: qStats,
	}
	// Careful: pool also exports name+"xxx" vars,
	// but we know it doesn't export Timeout.
	if enablePublishStats {
		stats.Publish(name+"Timeout", stats.DurationFunc(axp.timeout.Get))
	}
	return axp
}

// Open makes the TxPool operational. This also starts the transaction killer
// that will kill long-running transactions.
func (axp *TxPool) Open(appParams, dbaParams *sqldb.ConnParams) {
	log.Infof("Starting transaction id: %d", axp.lastID)
	axp.pool.Open(appParams, dbaParams)
	axp.ticks.Start(func() { axp.transactionKiller() })
}

// Close closes the TxPool. A closed pool can be reopened.
func (axp *TxPool) Close() {
	axp.ticks.Stop()
	for _, v := range axp.activePool.GetOutdated(time.Duration(0), "for closing") {
		conn := v.(*TxConnection)
		log.Warningf("killing transaction for shutdown: %s", conn.Format(nil))
		axp.queryServiceStats.InternalErrors.Add("StrayTransactions", 1)
		conn.Close()
		conn.conclude(TxClose)
	}
	axp.pool.Close()
}

// RollbackNonBusy rolls back all transactions that are not in use.
// Transactions can be in use for situations like executing statements
// or in prepared state.
func (axp *TxPool) RollbackNonBusy(ctx context.Context) {
	for _, v := range axp.activePool.GetOutdated(time.Duration(0), "for transition") {
		axp.LocalConclude(ctx, v.(*TxConnection))
	}
}

func (axp *TxPool) transactionKiller() {
	defer logError(axp.queryServiceStats)
	for _, v := range axp.activePool.GetOutdated(time.Duration(axp.Timeout()), "for rollback") {
		conn := v.(*TxConnection)
		log.Warningf("killing transaction (exceeded timeout: %v): %s", axp.Timeout(), conn.Format(nil))
		axp.queryServiceStats.KillStats.Add("Transactions", 1)
		conn.Close()
		conn.conclude(TxKill)
	}
}

// WaitForEmpty waits until all active transactions are completed.
func (axp *TxPool) WaitForEmpty() {
	axp.activePool.WaitForEmpty()
}

// Begin begins a transaction, and returns the associated transaction id.
// Subsequent statements can access the connection through the transaction id.
func (axp *TxPool) Begin(ctx context.Context) (int64, error) {
	poolCtx := ctx
	if deadline, ok := ctx.Deadline(); ok {
		var cancel func()
		poolCtx, cancel = context.WithDeadline(ctx, deadline.Add(-10*time.Millisecond))
		defer cancel()
	}
	conn, err := axp.pool.Get(poolCtx)
	if err != nil {
		switch err {
		case ErrConnPoolClosed:
			return 0, err
		case pools.ErrTimeout:
			axp.LogActive()
			return 0, NewTabletError(vtrpcpb.ErrorCode_RESOURCE_EXHAUSTED, "Transaction pool connection limit exceeded")
		}
		return 0, NewTabletErrorSQL(vtrpcpb.ErrorCode_INTERNAL_ERROR, err)
	}
	if _, err := conn.Exec(ctx, "begin", 1, false); err != nil {
		conn.Recycle()
		return 0, NewTabletErrorSQL(vtrpcpb.ErrorCode_UNKNOWN_ERROR, err)
	}
	transactionID := axp.lastID.Add(1)
	axp.activePool.Register(
		transactionID,
		newTxConnection(
			conn,
			transactionID,
			axp,
			callerid.ImmediateCallerIDFromContext(ctx),
			callerid.EffectiveCallerIDFromContext(ctx),
		),
	)
	return transactionID, nil
}

// Commit commits the specified transaction.
func (axp *TxPool) Commit(ctx context.Context, transactionID int64) error {
	conn, err := axp.Get(transactionID, "for commit")
	if err != nil {
		return err
	}
	return axp.LocalCommit(ctx, conn)
}

// Rollback rolls back the specified transaction.
func (axp *TxPool) Rollback(ctx context.Context, transactionID int64) error {
	conn, err := axp.Get(transactionID, "for rollback")
	if err != nil {
		return err
	}
	return axp.localRollback(ctx, conn)
}

// Get fetches the connection associated to the transactionID.
// You must call Recycle on TxConnection once done.
func (axp *TxPool) Get(transactionID int64, reason string) (*TxConnection, error) {
	v, err := axp.activePool.Get(transactionID, reason)
	if err != nil {
		return nil, NewTabletError(vtrpcpb.ErrorCode_NOT_IN_TX, "Transaction %d: %v", transactionID, err)
	}
	return v.(*TxConnection), nil
}

// LocalBegin is equivalent to Begin->Get.
// It's used for executing transactions within a request. It's safe
// to always call LocalConclude at the end.
func (axp *TxPool) LocalBegin(ctx context.Context) (*TxConnection, error) {
	transactionID, err := axp.Begin(ctx)
	if err != nil {
		return nil, err
	}
	return axp.Get(transactionID, "for local query")
}

// LocalCommit is the commit function for LocalBegin.
func (axp *TxPool) LocalCommit(ctx context.Context, conn *TxConnection) error {
	defer conn.conclude(TxCommit)
	axp.txStats.Add("Completed", time.Now().Sub(conn.StartTime))
	if _, err := conn.Exec(ctx, "commit", 1, false); err != nil {
		conn.Close()
		return NewTabletErrorSQL(vtrpcpb.ErrorCode_UNKNOWN_ERROR, err)
	}
	return nil
}

// LocalConclude concludes a transaction started by LocalBegin.
// If the transaction was not previously concluded, it's rolled back.
func (axp *TxPool) LocalConclude(ctx context.Context, conn *TxConnection) {
	if conn.DBConn != nil {
		_ = axp.localRollback(ctx, conn)
	}
}

func (axp *TxPool) localRollback(ctx context.Context, conn *TxConnection) error {
	defer conn.conclude(TxRollback)
	axp.txStats.Add("Aborted", time.Now().Sub(conn.StartTime))
	if _, err := conn.Exec(ctx, "rollback", 1, false); err != nil {
		conn.Close()
		return NewTabletErrorSQL(vtrpcpb.ErrorCode_UNKNOWN_ERROR, err)
	}
	return nil
}

// LogActive causes all existing transactions to be logged when they complete.
// The logging is throttled to no more than once every txLogInterval.
func (axp *TxPool) LogActive() {
	axp.logMu.Lock()
	defer axp.logMu.Unlock()
	if time.Now().Sub(axp.lastLog) < txLogInterval {
		return
	}
	axp.lastLog = time.Now()
	conns := axp.activePool.GetAll()
	for _, c := range conns {
		c.(*TxConnection).LogToFile.Set(1)
	}
}

// Timeout returns the transaction timeout.
func (axp *TxPool) Timeout() time.Duration {
	return axp.timeout.Get()
}

// SetTimeout sets the transaction timeout.
func (axp *TxPool) SetTimeout(timeout time.Duration) {
	axp.timeout.Set(timeout)
	axp.ticks.SetInterval(timeout / 10)
}

// TxConnection is meant for executing transactions. It can return itself to
// the tx pool correctly. It also does not retry statements if there
// are failures.
type TxConnection struct {
	*DBConn
	TransactionID     int64
	pool              *TxPool
	StartTime         time.Time
	EndTime           time.Time
	Queries           []string
	Conclusion        string
	LogToFile         sync2.AtomicInt32
	ImmediateCallerID *querypb.VTGateCallerID
	EffectiveCallerID *vtrpcpb.CallerID
}

func newTxConnection(conn *DBConn, transactionID int64, pool *TxPool, immediate *querypb.VTGateCallerID, effective *vtrpcpb.CallerID) *TxConnection {
	return &TxConnection{
		DBConn:            conn,
		TransactionID:     transactionID,
		pool:              pool,
		StartTime:         time.Now(),
		Queries:           make([]string, 0, 8),
		ImmediateCallerID: immediate,
		EffectiveCallerID: effective,
	}
}

// Exec executes the statement for the current transaction.
func (txc *TxConnection) Exec(ctx context.Context, query string, maxrows int, wantfields bool) (*sqltypes.Result, error) {
	r, err := txc.DBConn.ExecOnce(ctx, query, maxrows, wantfields)
	if err != nil {
		if IsConnErr(err) {
			txc.pool.checker.CheckMySQL()
			return nil, NewTabletErrorSQL(vtrpcpb.ErrorCode_INTERNAL_ERROR, err)
		}
		return nil, NewTabletErrorSQL(vtrpcpb.ErrorCode_UNKNOWN_ERROR, err)
	}
	return r, nil
}

// Recycle returns the connection to the pool. The transaction remains
// active.
func (txc *TxConnection) Recycle() {
	if txc.IsClosed() {
		txc.conclude(TxClose)
	} else {
		txc.pool.activePool.Put(txc.TransactionID)
	}
}

// RecordQuery records the query against this transaction.
func (txc *TxConnection) RecordQuery(query string) {
	txc.Queries = append(txc.Queries, query)
}

func (txc *TxConnection) conclude(conclusion string) {
	txc.pool.activePool.Unregister(txc.TransactionID)
	txc.DBConn.Recycle()
	txc.DBConn = nil
	txc.log(conclusion)
}

func (txc *TxConnection) log(conclusion string) {
	txc.Conclusion = conclusion
	txc.EndTime = time.Now()

	username := callerid.GetPrincipal(txc.EffectiveCallerID)
	if username == "" {
		username = callerid.GetUsername(txc.ImmediateCallerID)
	}
	duration := txc.EndTime.Sub(txc.StartTime)
	txc.pool.queryServiceStats.UserTransactionCount.Add([]string{username, conclusion}, 1)
	txc.pool.queryServiceStats.UserTransactionTimesNs.Add([]string{username, conclusion}, int64(duration))
	if txc.LogToFile.Get() != 0 {
		log.Infof("Logged transaction: %s", txc.Format(nil))
	}
	TxLogger.Send(txc)
}

// EventTime returns the time the event was created.
func (txc *TxConnection) EventTime() time.Time {
	return txc.EndTime
}

// Format returns a printable version of the connection info.
func (txc *TxConnection) Format(params url.Values) string {
	return fmt.Sprintf(
		"%v\t'%v'\t'%v'\t%v\t%v\t%.6f\t%v\t%v\t\n",
		txc.TransactionID,
		callerid.GetPrincipal(txc.EffectiveCallerID),
		callerid.GetUsername(txc.ImmediateCallerID),
		txc.StartTime.Format(time.StampMicro),
		txc.EndTime.Format(time.StampMicro),
		txc.EndTime.Sub(txc.StartTime).Seconds(),
		txc.Conclusion,
		strings.Join(txc.Queries, ";"),
	)
}
<|endoftext|>


<commit_before>//  Copyright 2020 Istio Authors
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package cni

import (
	"testing"

	"istio.io/istio/pkg/test/framework"
	"istio.io/istio/pkg/test/framework/components/echo"
	"istio.io/istio/pkg/test/framework/components/environment"
	"istio.io/istio/pkg/test/framework/components/galley"
	"istio.io/istio/pkg/test/framework/components/istio"
	"istio.io/istio/pkg/test/framework/components/namespace"
	"istio.io/istio/pkg/test/framework/components/pilot"
	"istio.io/istio/tests/integration/security/util/reachability"
)

func TestMain(m *testing.M) {
	framework.
		NewSuite("cni", m).
		Skip("https://github.com/istio/istio/issues/20487").
		SetupOnEnv(environment.Kube, istio.Setup(nil, func(cfg *istio.Config) {
			cfg.ControlPlaneValues = `
components:
  cni:
     enabled: true
     hub: gcr.io/istio-testing
     tag: latest
`
		})).
		Run()
}

// This test verifies reachability under different authN scenario:
// - app A to app B using mTLS.
// In each test, the steps are:
// - Configure authn policy.
// - Wait for config propagation.
// - Send HTTP/gRPC requests between apps.
func TestCNIReachability(t *testing.T) {
	framework.NewTest(t).
		Run(func(ctx framework.TestContext) {
			g, err := galley.New(ctx, galley.Config{})
			if err != nil {
				ctx.Fatal(err)
			}
			p, err := pilot.New(ctx, pilot.Config{
				Galley: g,
			})
			if err != nil {
				ctx.Fatal(err)
			}
			rctx := reachability.CreateContext(ctx, g, p)
			systemNM := namespace.ClaimSystemNamespaceOrFail(ctx, ctx)

			testCases := []reachability.TestCase{
				{
					ConfigFile:          "global-mtls-on.yaml",
					Namespace:           systemNM,
					RequiredEnvironment: environment.Kube,
					Include: func(src echo.Instance, opts echo.CallOptions) bool {
						// Exclude calls to the headless TCP port.
						if opts.Target == rctx.Headless && opts.PortName == "tcp" {
							return false
						}
						return true
					},
					ExpectSuccess: func(src echo.Instance, opts echo.CallOptions) bool {
						if src == rctx.Naked && opts.Target == rctx.Naked {
							// naked->naked should always succeed.
							return true
						}

						// If one of the two endpoints is naked, expect failure.
						return src != rctx.Naked && opts.Target != rctx.Naked
					},
				},
			}
			rctx.Run(testCases)
		})
}
<commit_msg>Re-enable CNI test (#20826)<commit_after>//  Copyright 2020 Istio Authors
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package cni

import (
	"testing"

	"istio.io/istio/pkg/test/framework"
	"istio.io/istio/pkg/test/framework/components/echo"
	"istio.io/istio/pkg/test/framework/components/environment"
	"istio.io/istio/pkg/test/framework/components/environment/kube"
	"istio.io/istio/pkg/test/framework/components/galley"
	"istio.io/istio/pkg/test/framework/components/istio"
	"istio.io/istio/pkg/test/framework/components/namespace"
	"istio.io/istio/pkg/test/framework/components/pilot"
	"istio.io/istio/tests/integration/security/util/reachability"
)

func TestMain(m *testing.M) {
	framework.
		NewSuite("cni", m).
		RequireEnvironment(environment.Kube).
		SetupOnEnv(environment.Kube, istio.Setup(nil, func(cfg *istio.Config) {
			cfg.ControlPlaneValues = `
components:
  cni:
     enabled: true
     hub: gcr.io/istio-testing
     tag: latest
     namespace: kube-system
`
		})).
		Run()
}

// This test verifies reachability under different authN scenario:
// - app A to app B using mTLS.
// In each test, the steps are:
// - Configure authn policy.
// - Wait for config propagation.
// - Send HTTP/gRPC requests between apps.
func TestCNIReachability(t *testing.T) {
	framework.NewTest(t).
		Run(func(ctx framework.TestContext) {
			g, err := galley.New(ctx, galley.Config{})
			if err != nil {
				ctx.Fatal(err)
			}
			p, err := pilot.New(ctx, pilot.Config{
				Galley: g,
			})
			if err != nil {
				ctx.Fatal(err)
			}
			kenv := ctx.Environment().(*kube.Environment)
			_, err = kenv.WaitUntilPodsAreReady(kenv.NewSinglePodFetch("kube-system", "k8s-app=istio-cni-node"))
			if err != nil {
				ctx.Fatal(err)
			}
			rctx := reachability.CreateContext(ctx, g, p)
			systemNM := namespace.ClaimSystemNamespaceOrFail(ctx, ctx)

			testCases := []reachability.TestCase{
				{
					ConfigFile:          "global-mtls-on.yaml",
					Namespace:           systemNM,
					RequiredEnvironment: environment.Kube,
					Include: func(src echo.Instance, opts echo.CallOptions) bool {
						// Exclude calls to the headless TCP port.
						if opts.Target == rctx.Headless && opts.PortName == "tcp" {
							return false
						}
						return true
					},
					ExpectSuccess: func(src echo.Instance, opts echo.CallOptions) bool {
						if src == rctx.Naked && opts.Target == rctx.Naked {
							// naked->naked should always succeed.
							return true
						}

						// If one of the two endpoints is naked, expect failure.
						return src != rctx.Naked && opts.Target != rctx.Naked
					},
				},
			}
			rctx.Run(testCases)
		})
}
<|endoftext|>


<commit_before>// Same copyright and license as the rest of the files in this project
// This file contains style related functions and structures

package gtk

// #include <gtk/gtk.h>
// #include "gtk.go.h"
import "C"
import (
	"unsafe"

	"github.com/gotk3/gotk3/gdk"
	"github.com/gotk3/gotk3/glib"
)

type StyleProviderPriority int

const (
	STYLE_PROVIDER_PRIORITY_FALLBACK    StyleProviderPriority = C.GTK_STYLE_PROVIDER_PRIORITY_FALLBACK
	STYLE_PROVIDER_PRIORITY_THEME                             = C.GTK_STYLE_PROVIDER_PRIORITY_THEME
	STYLE_PROVIDER_PRIORITY_SETTINGS                          = C.GTK_STYLE_PROVIDER_PRIORITY_SETTINGS
	STYLE_PROVIDER_PRIORITY_APPLICATION                       = C.GTK_STYLE_PROVIDER_PRIORITY_APPLICATION
	STYLE_PROVIDER_PRIORITY_USER                              = C.GTK_STYLE_PROVIDER_PRIORITY_USER
)

/*
 * GtkStyleContext
 */

// StyleContext is a representation of GTK's GtkStyleContext.
type StyleContext struct {
	*glib.Object
}

// native returns a pointer to the underlying GtkStyleContext.
func (v *StyleContext) native() *C.GtkStyleContext {
	if v == nil || v.Object == nil {
		return nil
	}
	p := unsafe.Pointer(v.GObject)
	return C.toGtkStyleContext(p)
}

func wrapStyleContext(obj *glib.Object) *StyleContext {
	if obj == nil {
		return nil
	}

	return &StyleContext{obj}
}

func (v *StyleContext) AddClass(class_name string) {
	cstr := C.CString(class_name)
	defer C.free(unsafe.Pointer(cstr))

	C.gtk_style_context_add_class(v.native(), (*C.gchar)(cstr))
}

func (v *StyleContext) RemoveClass(class_name string) {
	cstr := C.CString(class_name)
	defer C.free(unsafe.Pointer(cstr))

	C.gtk_style_context_remove_class(v.native(), (*C.gchar)(cstr))
}

func fromNativeStyleContext(c *C.GtkStyleContext) (*StyleContext, error) {
	if c == nil {
		return nil, nilPtrErr
	}

	obj := glib.Take(unsafe.Pointer(c))
	return wrapStyleContext(obj), nil
}

// GetStyleContext is a wrapper around gtk_widget_get_style_context().
func (v *Widget) GetStyleContext() (*StyleContext, error) {
	return fromNativeStyleContext(C.gtk_widget_get_style_context(v.native()))
}

// GetParent is a wrapper around gtk_style_context_get_parent().
func (v *StyleContext) GetParent() (*StyleContext, error) {
	return fromNativeStyleContext(C.gtk_style_context_get_parent(v.native()))
}

// GetProperty is a wrapper around gtk_style_context_get_property().
func (v *StyleContext) GetProperty(property string, state StateFlags) (interface{}, error) {
	cstr := (*C.gchar)(C.CString(property))
	defer C.free(unsafe.Pointer(cstr))

	var gval C.GValue
	C.gtk_style_context_get_property(v.native(), cstr, C.GtkStateFlags(state), &gval)
	val := glib.ValueFromNative(unsafe.Pointer(&gval))
	return val.GoValue()
}

// GetStyleProperty is a wrapper around gtk_style_context_get_style_property().
func (v *StyleContext) GetStyleProperty(property string) (interface{}, error) {
	cstr := (*C.gchar)(C.CString(property))
	defer C.free(unsafe.Pointer(cstr))

	var gval C.GValue
	C.gtk_style_context_get_style_property(v.native(), cstr, &gval)
	val := glib.ValueFromNative(unsafe.Pointer(&gval))
	return val.GoValue()
}

// GetScreen is a wrapper around gtk_style_context_get_screen().
func (v *StyleContext) GetScreen() (*gdk.Screen, error) {
	c := C.gtk_style_context_get_screen(v.native())
	if c == nil {
		return nil, nilPtrErr
	}

	d := &gdk.Screen{glib.Take(unsafe.Pointer(c))}
	return d, nil
}

// GetState is a wrapper around gtk_style_context_get_state().
func (v *StyleContext) GetState() StateFlags {
	return StateFlags(C.gtk_style_context_get_state(v.native()))
}

// GetColor is a wrapper around gtk_style_context_get_color().
func (v *StyleContext) GetColor(state StateFlags) *gdk.RGBA {
	gdkColor := gdk.NewRGBA()
	C.gtk_style_context_get_color(v.native(), C.GtkStateFlags(state), (*C.GdkRGBA)(unsafe.Pointer(gdkColor.Native())))
	return gdkColor
}

// LookupColor is a wrapper around gtk_style_context_lookup_color().
func (v *StyleContext) LookupColor(colorName string) (*gdk.RGBA, bool) {
	cstr := (*C.gchar)(C.CString(colorName))
	defer C.free(unsafe.Pointer(cstr))
	gdkColor := gdk.NewRGBA()
	ret := C.gtk_style_context_lookup_color(v.native(), cstr, (*C.GdkRGBA)(unsafe.Pointer(gdkColor.Native())))
	return gdkColor, gobool(ret)
}

// StyleContextResetWidgets is a wrapper around gtk_style_context_reset_widgets().
func StyleContextResetWidgets(v *gdk.Screen) {
	C.gtk_style_context_reset_widgets((*C.GdkScreen)(unsafe.Pointer(v.Native())))
}

// Restore is a wrapper around gtk_style_context_restore().
func (v *StyleContext) Restore() {
	C.gtk_style_context_restore(v.native())
}

// Save is a wrapper around gtk_style_context_save().
func (v *StyleContext) Save() {
	C.gtk_style_context_save(v.native())
}

// SetParent is a wrapper around gtk_style_context_set_parent().
func (v *StyleContext) SetParent(p *StyleContext) {
	C.gtk_style_context_set_parent(v.native(), p.native())
}

// HasClass is a wrapper around gtk_style_context_has_class().
func (v *StyleContext) HasClass(className string) bool {
	cstr := C.CString(className)
	defer C.free(unsafe.Pointer(cstr))

	return gobool(C.gtk_style_context_has_class(v.native(), (*C.gchar)(cstr)))
}

// SetScreen is a wrapper around gtk_style_context_set_screen().
func (v *StyleContext) SetScreen(s *gdk.Screen) {
	C.gtk_style_context_set_screen(v.native(), (*C.GdkScreen)(unsafe.Pointer(s.Native())))
}

// SetState is a wrapper around gtk_style_context_set_state().
func (v *StyleContext) SetState(state StateFlags) {
	C.gtk_style_context_set_state(v.native(), C.GtkStateFlags(state))
}

type IStyleProvider interface {
	toStyleProvider() *C.GtkStyleProvider
}

// AddProvider is a wrapper around gtk_style_context_add_provider().
func (v *StyleContext) AddProvider(provider IStyleProvider, prio uint) {
	C.gtk_style_context_add_provider(v.native(), provider.toStyleProvider(), C.guint(prio))
}

// AddProviderForScreen is a wrapper around gtk_style_context_add_provider_for_screen().
func AddProviderForScreen(s *gdk.Screen, provider IStyleProvider, prio uint) {
	C.gtk_style_context_add_provider_for_screen((*C.GdkScreen)(unsafe.Pointer(s.Native())), provider.toStyleProvider(), C.guint(prio))
}

// RemoveProvider is a wrapper around gtk_style_context_remove_provider().
func (v *StyleContext) RemoveProvider(provider IStyleProvider) {
	C.gtk_style_context_remove_provider(v.native(), provider.toStyleProvider())
}

// RemoveProviderForScreen is a wrapper around gtk_style_context_remove_provider_for_screen().
func RemoveProviderForScreen(s *gdk.Screen, provider IStyleProvider) {
	C.gtk_style_context_remove_provider_for_screen((*C.GdkScreen)(unsafe.Pointer(s.Native())), provider.toStyleProvider())
}

// GtkStyleContext * 	gtk_style_context_new ()
// void 	gtk_style_context_get ()
// GtkTextDirection 	gtk_style_context_get_direction ()
// GtkJunctionSides 	gtk_style_context_get_junction_sides ()
// const GtkWidgetPath * 	gtk_style_context_get_path ()
// GdkFrameClock * 	gtk_style_context_get_frame_clock ()
// void 	gtk_style_context_get_style ()
// void 	gtk_style_context_get_style_valist ()
// void 	gtk_style_context_get_valist ()
// GtkCssSection * 	gtk_style_context_get_section ()
// void 	gtk_style_context_get_background_color ()
// void 	gtk_style_context_get_border_color ()
// void 	gtk_style_context_get_border ()
// void 	gtk_style_context_get_padding ()
// void 	gtk_style_context_get_margin ()
// const PangoFontDescription * 	gtk_style_context_get_font ()
// void 	gtk_style_context_invalidate ()
// gboolean 	gtk_style_context_state_is_running ()
// GtkIconSet * 	gtk_style_context_lookup_icon_set ()
// void 	gtk_style_context_cancel_animations ()
// void 	gtk_style_context_scroll_animations ()
// void 	gtk_style_context_notify_state_change ()
// void 	gtk_style_context_pop_animatable_region ()
// void 	gtk_style_context_push_animatable_region ()
// void 	gtk_style_context_set_background ()
// void 	gtk_style_context_set_direction ()
// void 	gtk_style_context_set_junction_sides ()
// void 	gtk_style_context_set_path ()
// void 	gtk_style_context_add_region ()
// void 	gtk_style_context_remove_region ()
// gboolean 	gtk_style_context_has_region ()
// GList * 	gtk_style_context_list_regions ()
// void 	gtk_style_context_set_frame_clock ()
// void 	gtk_style_context_set_scale ()
// gint 	gtk_style_context_get_scale ()
// GList * 	gtk_style_context_list_classes ()
<commit_msg>Add gtk.RenderBackground<commit_after>// Same copyright and license as the rest of the files in this project
// This file contains style related functions and structures

package gtk

// #include <gtk/gtk.h>
// #include "gtk.go.h"
import "C"
import (
	"unsafe"

	"github.com/gotk3/gotk3/cairo"
	"github.com/gotk3/gotk3/gdk"
	"github.com/gotk3/gotk3/glib"
)

type StyleProviderPriority int

const (
	STYLE_PROVIDER_PRIORITY_FALLBACK    StyleProviderPriority = C.GTK_STYLE_PROVIDER_PRIORITY_FALLBACK
	STYLE_PROVIDER_PRIORITY_THEME                             = C.GTK_STYLE_PROVIDER_PRIORITY_THEME
	STYLE_PROVIDER_PRIORITY_SETTINGS                          = C.GTK_STYLE_PROVIDER_PRIORITY_SETTINGS
	STYLE_PROVIDER_PRIORITY_APPLICATION                       = C.GTK_STYLE_PROVIDER_PRIORITY_APPLICATION
	STYLE_PROVIDER_PRIORITY_USER                              = C.GTK_STYLE_PROVIDER_PRIORITY_USER
)

// RenderBackground is a wrapper around gtk_render_background().
func RenderBackground(context *StyleContext, cr *cairo.Context, x, y, w, h float64) {
	C.gtk_render_background(
		context.native(),
		(*C.cairo_t)(unsafe.Pointer(cr.Native())),
		C.gdouble(x),
		C.gdouble(y),
		C.gdouble(w),
		C.gdouble(h),
	)
}

/*
 * GtkStyleContext
 */

// StyleContext is a representation of GTK's GtkStyleContext.
type StyleContext struct {
	*glib.Object
}

// native returns a pointer to the underlying GtkStyleContext.
func (v *StyleContext) native() *C.GtkStyleContext {
	if v == nil || v.Object == nil {
		return nil
	}
	p := unsafe.Pointer(v.GObject)
	return C.toGtkStyleContext(p)
}

func wrapStyleContext(obj *glib.Object) *StyleContext {
	if obj == nil {
		return nil
	}

	return &StyleContext{obj}
}

func (v *StyleContext) AddClass(class_name string) {
	cstr := C.CString(class_name)
	defer C.free(unsafe.Pointer(cstr))

	C.gtk_style_context_add_class(v.native(), (*C.gchar)(cstr))
}

func (v *StyleContext) RemoveClass(class_name string) {
	cstr := C.CString(class_name)
	defer C.free(unsafe.Pointer(cstr))

	C.gtk_style_context_remove_class(v.native(), (*C.gchar)(cstr))
}

func fromNativeStyleContext(c *C.GtkStyleContext) (*StyleContext, error) {
	if c == nil {
		return nil, nilPtrErr
	}

	obj := glib.Take(unsafe.Pointer(c))
	return wrapStyleContext(obj), nil
}

// GetStyleContext is a wrapper around gtk_widget_get_style_context().
func (v *Widget) GetStyleContext() (*StyleContext, error) {
	return fromNativeStyleContext(C.gtk_widget_get_style_context(v.native()))
}

// GetParent is a wrapper around gtk_style_context_get_parent().
func (v *StyleContext) GetParent() (*StyleContext, error) {
	return fromNativeStyleContext(C.gtk_style_context_get_parent(v.native()))
}

// GetProperty is a wrapper around gtk_style_context_get_property().
func (v *StyleContext) GetProperty(property string, state StateFlags) (interface{}, error) {
	cstr := (*C.gchar)(C.CString(property))
	defer C.free(unsafe.Pointer(cstr))

	var gval C.GValue
	C.gtk_style_context_get_property(v.native(), cstr, C.GtkStateFlags(state), &gval)
	val := glib.ValueFromNative(unsafe.Pointer(&gval))
	return val.GoValue()
}

// GetStyleProperty is a wrapper around gtk_style_context_get_style_property().
func (v *StyleContext) GetStyleProperty(property string) (interface{}, error) {
	cstr := (*C.gchar)(C.CString(property))
	defer C.free(unsafe.Pointer(cstr))

	var gval C.GValue
	C.gtk_style_context_get_style_property(v.native(), cstr, &gval)
	val := glib.ValueFromNative(unsafe.Pointer(&gval))
	return val.GoValue()
}

// GetScreen is a wrapper around gtk_style_context_get_screen().
func (v *StyleContext) GetScreen() (*gdk.Screen, error) {
	c := C.gtk_style_context_get_screen(v.native())
	if c == nil {
		return nil, nilPtrErr
	}

	d := &gdk.Screen{glib.Take(unsafe.Pointer(c))}
	return d, nil
}

// GetState is a wrapper around gtk_style_context_get_state().
func (v *StyleContext) GetState() StateFlags {
	return StateFlags(C.gtk_style_context_get_state(v.native()))
}

// GetColor is a wrapper around gtk_style_context_get_color().
func (v *StyleContext) GetColor(state StateFlags) *gdk.RGBA {
	gdkColor := gdk.NewRGBA()
	C.gtk_style_context_get_color(v.native(), C.GtkStateFlags(state), (*C.GdkRGBA)(unsafe.Pointer(gdkColor.Native())))
	return gdkColor
}

// LookupColor is a wrapper around gtk_style_context_lookup_color().
func (v *StyleContext) LookupColor(colorName string) (*gdk.RGBA, bool) {
	cstr := (*C.gchar)(C.CString(colorName))
	defer C.free(unsafe.Pointer(cstr))
	gdkColor := gdk.NewRGBA()
	ret := C.gtk_style_context_lookup_color(v.native(), cstr, (*C.GdkRGBA)(unsafe.Pointer(gdkColor.Native())))
	return gdkColor, gobool(ret)
}

// StyleContextResetWidgets is a wrapper around gtk_style_context_reset_widgets().
func StyleContextResetWidgets(v *gdk.Screen) {
	C.gtk_style_context_reset_widgets((*C.GdkScreen)(unsafe.Pointer(v.Native())))
}

// Restore is a wrapper around gtk_style_context_restore().
func (v *StyleContext) Restore() {
	C.gtk_style_context_restore(v.native())
}

// Save is a wrapper around gtk_style_context_save().
func (v *StyleContext) Save() {
	C.gtk_style_context_save(v.native())
}

// SetParent is a wrapper around gtk_style_context_set_parent().
func (v *StyleContext) SetParent(p *StyleContext) {
	C.gtk_style_context_set_parent(v.native(), p.native())
}

// HasClass is a wrapper around gtk_style_context_has_class().
func (v *StyleContext) HasClass(className string) bool {
	cstr := C.CString(className)
	defer C.free(unsafe.Pointer(cstr))

	return gobool(C.gtk_style_context_has_class(v.native(), (*C.gchar)(cstr)))
}

// SetScreen is a wrapper around gtk_style_context_set_screen().
func (v *StyleContext) SetScreen(s *gdk.Screen) {
	C.gtk_style_context_set_screen(v.native(), (*C.GdkScreen)(unsafe.Pointer(s.Native())))
}

// SetState is a wrapper around gtk_style_context_set_state().
func (v *StyleContext) SetState(state StateFlags) {
	C.gtk_style_context_set_state(v.native(), C.GtkStateFlags(state))
}

type IStyleProvider interface {
	toStyleProvider() *C.GtkStyleProvider
}

// AddProvider is a wrapper around gtk_style_context_add_provider().
func (v *StyleContext) AddProvider(provider IStyleProvider, prio uint) {
	C.gtk_style_context_add_provider(v.native(), provider.toStyleProvider(), C.guint(prio))
}

// AddProviderForScreen is a wrapper around gtk_style_context_add_provider_for_screen().
func AddProviderForScreen(s *gdk.Screen, provider IStyleProvider, prio uint) {
	C.gtk_style_context_add_provider_for_screen((*C.GdkScreen)(unsafe.Pointer(s.Native())), provider.toStyleProvider(), C.guint(prio))
}

// RemoveProvider is a wrapper around gtk_style_context_remove_provider().
func (v *StyleContext) RemoveProvider(provider IStyleProvider) {
	C.gtk_style_context_remove_provider(v.native(), provider.toStyleProvider())
}

// RemoveProviderForScreen is a wrapper around gtk_style_context_remove_provider_for_screen().
func RemoveProviderForScreen(s *gdk.Screen, provider IStyleProvider) {
	C.gtk_style_context_remove_provider_for_screen((*C.GdkScreen)(unsafe.Pointer(s.Native())), provider.toStyleProvider())
}

// GtkStyleContext * 	gtk_style_context_new ()
// void 	gtk_style_context_get ()
// GtkTextDirection 	gtk_style_context_get_direction ()
// GtkJunctionSides 	gtk_style_context_get_junction_sides ()
// const GtkWidgetPath * 	gtk_style_context_get_path ()
// GdkFrameClock * 	gtk_style_context_get_frame_clock ()
// void 	gtk_style_context_get_style ()
// void 	gtk_style_context_get_style_valist ()
// void 	gtk_style_context_get_valist ()
// GtkCssSection * 	gtk_style_context_get_section ()
// void 	gtk_style_context_get_background_color ()
// void 	gtk_style_context_get_border_color ()
// void 	gtk_style_context_get_border ()
// void 	gtk_style_context_get_padding ()
// void 	gtk_style_context_get_margin ()
// const PangoFontDescription * 	gtk_style_context_get_font ()
// void 	gtk_style_context_invalidate ()
// gboolean 	gtk_style_context_state_is_running ()
// GtkIconSet * 	gtk_style_context_lookup_icon_set ()
// void 	gtk_style_context_cancel_animations ()
// void 	gtk_style_context_scroll_animations ()
// void 	gtk_style_context_notify_state_change ()
// void 	gtk_style_context_pop_animatable_region ()
// void 	gtk_style_context_push_animatable_region ()
// void 	gtk_style_context_set_background ()
// void 	gtk_style_context_set_direction ()
// void 	gtk_style_context_set_junction_sides ()
// void 	gtk_style_context_set_path ()
// void 	gtk_style_context_add_region ()
// void 	gtk_style_context_remove_region ()
// gboolean 	gtk_style_context_has_region ()
// GList * 	gtk_style_context_list_regions ()
// void 	gtk_style_context_set_frame_clock ()
// void 	gtk_style_context_set_scale ()
// gint 	gtk_style_context_get_scale ()
// GList * 	gtk_style_context_list_classes ()
<|endoftext|>


<commit_before>// Copyright 2017 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gui

import (
	"bytes"
	"fmt"
	"net/url"
	"reflect"
	"strconv"
	"strings"

	"github.com/vdobler/ht/errorlist"
)

type Message struct {
	Type string
	Text string
}

// ----------------------------------------------------------------------------
// Value

// Value contains a value to be displayed and updated through a HTML GUI.
type Value struct {
	// Current is the current value.
	Current interface{}

	// Last contains the last values
	Last []interface{}

	// Path is the path prefix applied to this value.
	Path string

	// Messages contains messages to be rendered for paths. E.g.:
	//   Test.Request.Timeout => Message{typ:error, txt=wrong}
	Messages map[string][]Message

	buf *bytes.Buffer

	nextfieldinfo Fieldinfo
}

// NewValue creates a new Value from val.
func NewValue(val interface{}, path string) *Value {
	return &Value{
		Current:  val,
		Path:     path,
		Messages: make(map[string][]Message),
		buf:      &bytes.Buffer{},
	}
}

// Render v's Current value. The returned byte slice must neither be modified
// nor kept longer then up to the next call of Render.
func (v *Value) Render() ([]byte, error) {
	val := reflect.ValueOf(v.Current)
	v.buf.Reset()
	err := v.render(v.Path, 0, false, val) // TODO: type based readonly
	return v.buf.Bytes(), err
}

// Update v with data from the received HTML form. It returns the path of the
// most prominent field (TODO: explain better).
func (v *Value) Update(form url.Values) (string, errorlist.List) {
	val := reflect.ValueOf(v.Current)
	v.Messages = make(map[string][]Message) // clear errors // TODO: really automaticall here?
	firstErrorPath := ""

	updated, err := walk(form, v.Path, val)

	if err == nil {
		v.Last = append(v.Last, v.Current)
	} else {
		// Process validation errors
		for _, e := range err {
			fmt.Println(e)
			switch ve := e.(type) {
			case ValueError:
				if firstErrorPath == "" {
					firstErrorPath = ve.Path
				}
				v.Messages[ve.Path] = []Message{{
					Type: "error",
					Text: ve.Err.Error(),
				}}
			case addNoticeError:
				firstErrorPath = string(ve)
			}
		}
	}

	v.Current = updated.Interface()

	return firstErrorPath, err
}

// BinaryData returns the string or byte slice addressed by path.
func (v *Value) BinaryData(path string) ([]byte, error) {
	if !strings.HasPrefix(path, v.Path) {
		return nil, fmt.Errorf("gui: bad path") // TODO error message
	}

	path = path[len(v.Path)+1:]
	return binData(path, reflect.ValueOf(v.Current))
}

func binData(path string, val reflect.Value) ([]byte, error) {
	if path == "" {
		switch val.Kind() {
		case reflect.String:
			return []byte(val.String()), nil
		case reflect.Slice:
			elKind := val.Type().Elem().Kind()
			if elKind == reflect.Uint8 {
				return val.Bytes(), nil
			}
			return nil, fmt.Errorf("gui: bin data of %s slice", elKind.String())
		}
	}
	part := strings.SplitN(path, ".", 2)
	if len(part) == 1 {
		part = append(part, "")
	}
	switch val.Kind() {
	case reflect.Struct:
		field := val.FieldByName(part[0])
		if !field.IsValid() {
			return nil, fmt.Errorf("gui: no such field %s", part[0])
		}
		return binData(part[1], field)
	case reflect.Map:
		name := demangleKey(part[0])
		key := reflect.ValueOf(name)
		// TODO handel maps index by other stuff han strings
		v := val.MapIndex(key)
		if !v.IsValid() {
			return nil, fmt.Errorf("gui: no such key %s", part[0])
		}

		return binData(part[1], v)
	case reflect.Slice:
		i, err := strconv.Atoi(part[0])
		if err != nil {
			return nil, fmt.Errorf("gui: bad index: %s", err)
		}
		if i < 0 || i > val.Len() {
			return nil, fmt.Errorf("gui: index %d out of range", i)
		}
		return binData(part[1], val.Index(i))
	case reflect.Interface, reflect.Ptr:
		if val.IsNil() {
			return nil, fmt.Errorf("gui: cannot traverse nil ptr/interface")
		}
		return binData(part[1], val.Elem())
	}
	return nil, fmt.Errorf("gui: cannot traverse %s in %s", val.Kind().String(), path)
}

// ----------------------------------------------------------------------------

// printf to internal buf.
func (v *Value) printf(format string, val ...interface{}) {
	fmt.Fprintf(v.buf, format, val...)
}

// typeinfo returns name and tooltip for val's type.
func (v *Value) typeinfo(val interface{}) (string, string) {
	var typ reflect.Type
	if v, ok := val.(reflect.Value); ok {
		typ = v.Type()
	} else {
		typ = val.(reflect.Type)
	}
	name := typ.Name()
	if name == "" {
		name = "-anonymous-"
	}

	tooltip := "??"
	if info, ok := Typedata[typ]; ok {
		tooltip = info.Doc
	}

	return name, tooltip
}

// fieldinfo returns the name and optional data for field nr i of val.
func (v *Value) fieldinfo(val reflect.Value, i int) (string, Fieldinfo) {
	typ := val.Type()
	name := typ.Field(i).Name

	if tinfo, ok := Typedata[typ]; ok {
		if finfo, ok := tinfo.Field[name]; ok {
			return name, finfo
		}
	}

	return name, Fieldinfo{Doc: "-??-"}
}
<commit_msg>gui: remove debug statement<commit_after>// Copyright 2017 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gui

import (
	"bytes"
	"fmt"
	"net/url"
	"reflect"
	"strconv"
	"strings"

	"github.com/vdobler/ht/errorlist"
)

type Message struct {
	Type string
	Text string
}

// ----------------------------------------------------------------------------
// Value

// Value contains a value to be displayed and updated through a HTML GUI.
type Value struct {
	// Current is the current value.
	Current interface{}

	// Last contains the last values
	Last []interface{}

	// Path is the path prefix applied to this value.
	Path string

	// Messages contains messages to be rendered for paths. E.g.:
	//   Test.Request.Timeout => Message{typ:error, txt=wrong}
	Messages map[string][]Message

	buf *bytes.Buffer

	nextfieldinfo Fieldinfo
}

// NewValue creates a new Value from val.
func NewValue(val interface{}, path string) *Value {
	return &Value{
		Current:  val,
		Path:     path,
		Messages: make(map[string][]Message),
		buf:      &bytes.Buffer{},
	}
}

// Render v's Current value. The returned byte slice must neither be modified
// nor kept longer then up to the next call of Render.
func (v *Value) Render() ([]byte, error) {
	val := reflect.ValueOf(v.Current)
	v.buf.Reset()
	err := v.render(v.Path, 0, false, val) // TODO: type based readonly
	return v.buf.Bytes(), err
}

// Update v with data from the received HTML form. It returns the path of the
// most prominent field (TODO: explain better).
func (v *Value) Update(form url.Values) (string, errorlist.List) {
	val := reflect.ValueOf(v.Current)
	v.Messages = make(map[string][]Message) // clear errors // TODO: really automaticall here?
	firstErrorPath := ""

	updated, err := walk(form, v.Path, val)

	if err == nil {
		v.Last = append(v.Last, v.Current)
	} else {
		// Process validation errors
		for _, e := range err {
			switch ve := e.(type) {
			case ValueError:
				if firstErrorPath == "" {
					firstErrorPath = ve.Path
				}
				v.Messages[ve.Path] = []Message{{
					Type: "error",
					Text: ve.Err.Error(),
				}}
			case addNoticeError:
				firstErrorPath = string(ve)
			}
		}
	}

	v.Current = updated.Interface()

	return firstErrorPath, err
}

// BinaryData returns the string or byte slice addressed by path.
func (v *Value) BinaryData(path string) ([]byte, error) {
	if !strings.HasPrefix(path, v.Path) {
		return nil, fmt.Errorf("gui: bad path") // TODO error message
	}

	path = path[len(v.Path)+1:]
	return binData(path, reflect.ValueOf(v.Current))
}

func binData(path string, val reflect.Value) ([]byte, error) {
	if path == "" {
		switch val.Kind() {
		case reflect.String:
			return []byte(val.String()), nil
		case reflect.Slice:
			elKind := val.Type().Elem().Kind()
			if elKind == reflect.Uint8 {
				return val.Bytes(), nil
			}
			return nil, fmt.Errorf("gui: bin data of %s slice", elKind.String())
		}
	}
	part := strings.SplitN(path, ".", 2)
	if len(part) == 1 {
		part = append(part, "")
	}
	switch val.Kind() {
	case reflect.Struct:
		field := val.FieldByName(part[0])
		if !field.IsValid() {
			return nil, fmt.Errorf("gui: no such field %s", part[0])
		}
		return binData(part[1], field)
	case reflect.Map:
		name := demangleKey(part[0])
		key := reflect.ValueOf(name)
		// TODO handel maps index by other stuff han strings
		v := val.MapIndex(key)
		if !v.IsValid() {
			return nil, fmt.Errorf("gui: no such key %s", part[0])
		}

		return binData(part[1], v)
	case reflect.Slice:
		i, err := strconv.Atoi(part[0])
		if err != nil {
			return nil, fmt.Errorf("gui: bad index: %s", err)
		}
		if i < 0 || i > val.Len() {
			return nil, fmt.Errorf("gui: index %d out of range", i)
		}
		return binData(part[1], val.Index(i))
	case reflect.Interface, reflect.Ptr:
		if val.IsNil() {
			return nil, fmt.Errorf("gui: cannot traverse nil ptr/interface")
		}
		return binData(part[1], val.Elem())
	}
	return nil, fmt.Errorf("gui: cannot traverse %s in %s", val.Kind().String(), path)
}

// ----------------------------------------------------------------------------

// printf to internal buf.
func (v *Value) printf(format string, val ...interface{}) {
	fmt.Fprintf(v.buf, format, val...)
}

// typeinfo returns name and tooltip for val's type.
func (v *Value) typeinfo(val interface{}) (string, string) {
	var typ reflect.Type
	if v, ok := val.(reflect.Value); ok {
		typ = v.Type()
	} else {
		typ = val.(reflect.Type)
	}
	name := typ.Name()
	if name == "" {
		name = "-anonymous-"
	}

	tooltip := "??"
	if info, ok := Typedata[typ]; ok {
		tooltip = info.Doc
	}

	return name, tooltip
}

// fieldinfo returns the name and optional data for field nr i of val.
func (v *Value) fieldinfo(val reflect.Value, i int) (string, Fieldinfo) {
	typ := val.Type()
	name := typ.Field(i).Name

	if tinfo, ok := Typedata[typ]; ok {
		if finfo, ok := tinfo.Field[name]; ok {
			return name, finfo
		}
	}

	return name, Fieldinfo{Doc: "-??-"}
}
<|endoftext|>


<commit_before>package gxutil

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"path"
	"strings"

	mh "github.com/jbenet/go-multihash"
	ish "github.com/whyrusleeping/fallback-ipfs-shell"
	. "github.com/whyrusleeping/stump"
)

const PkgFileName = "package.json"

type PM struct {
	shell ish.Shell

	cfg *Config

	// hash of the 'empty' ipfs dir to avoid extra calls to object new
	blankDir string
}

func NewPM(cfg *Config) (*PM, error) {
	sh, err := ish.NewShell()
	if err != nil {
		return nil, err
	}

	return &PM{
		shell: sh,
		cfg:   cfg,
	}, nil
}

// InstallDeps recursively installs all dependencies for the given package
func (pm *PM) InstallDeps(pkg *Package, location string) error {
	Log("installing package: %s-%s", pkg.Name, pkg.Version)
	for _, dep := range pkg.Dependencies {

		// if its already local, skip it
		pkgdir := path.Join(location, dep.Hash)
		err := FindPackageInDir(&Package{}, pkgdir)
		if err == nil {
			continue
		}

		deppkg, err := pm.GetPackageLocalDaemon(dep.Hash, location)
		if err != nil {
			return fmt.Errorf("failed to fetch package: %s (%s):%s", dep.Name,
				dep.Hash, err)
		}

		err = RunReqCheckHook(deppkg.Language, dep.Hash)
		if err != nil {
			return err
		}

		err = pm.InstallDeps(deppkg, location)
		if err != nil {
			return err
		}
	}
	return nil
}

func (pm *PM) InitPkg(dir, name, lang string) error {
	// check for existing packagefile
	p := path.Join(dir, PkgFileName)
	_, err := os.Stat(p)
	if err == nil {
		return errors.New("package file already exists in working dir")
	}

	username := pm.cfg.User.Name
	if username == "" {
		u, err := user.Current()
		if err != nil {
			fmt.Errorf("error looking up current user: %s", err)
		}
		username = u.Username
	}

	pkg := new(Package)
	pkg.Name = name
	pkg.Language = lang
	pkg.Author = username
	pkg.Version = "1.0.0"

	// check if the user has a tool installed for the selected language
	CheckForHelperTools(lang)

	err = SavePackageFile(pkg, p)
	if err != nil {
		return err
	}

	err = TryRunHook("post-init", lang, dir)
	if err != nil {
		return err
	}
	return nil
}

func CheckForHelperTools(lang string) {
	_, err := exec.LookPath("gx-" + lang)
	if err == nil {
		return
	}

	if strings.Contains(err.Error(), "file not found") {
		Log("notice: no helper tool found for", lang)
		return
	}

	Error("checking for helper tool:", err)
}

func (pm *PM) ImportPackage(dir, dephash string) (*Dependency, error) {
	ndep, err := pm.GetPackage(dephash)
	if err != nil {
		return nil, err
	}

	for _, child := range ndep.Dependencies {
		_, err := pm.ImportPackage(dir, child.Hash)
		if err != nil {
			return nil, err
		}
	}

	err = RunPostImportHook(ndep.Language, dephash)
	if err != nil {
		Fatal(err)
	}

	return &Dependency{
		Name:    ndep.Name,
		Hash:    dephash,
		Version: ndep.Version,
	}, nil
}

func (pm *PM) ResolveDepName(name string) (string, error) {
	_, err := mh.FromB58String(name)
	if err == nil {
		return name, nil
	}

	if strings.Contains(name, "/") {
		parts := strings.Split(name, "/")
		rpath, ok := pm.cfg.GetRepos()[parts[0]]
		if !ok {
			return "", fmt.Errorf("unknown repo: '%s'", parts[0])
		}

		pkgs, err := pm.FetchRepo(rpath, true)
		if err != nil {
			return "", err
		}

		val, ok := pkgs[parts[1]]
		if !ok {
			return "", fmt.Errorf("package %s not found in repo %s", parts[1], parts[0])
		}

		return val, nil
	}

	out, err := pm.QueryRepos(name)
	if err != nil {
		return "", err
	}

	if len(out) == 0 {
		return "", fmt.Errorf("could not find package by name: %s", name)
	}

	if len(out) == 1 {
		for _, v := range out {
			return v, nil
		}
	}

	return "", fmt.Errorf("ambiguous ref, appears in multiple repos")
}

func RunPostImportHook(env, pkg string) error {
	return TryRunHook("post-import", env, pkg)
}

func RunReqCheckHook(env, pkg string) error {
	return TryRunHook("req-check", env, pkg)
}

func TryRunHook(hook, env string, args ...string) error {
	if env == "" {
		return nil
	}

	binname := "gx-" + env
	_, err := exec.LookPath(binname)
	if err != nil {
		if os.IsNotExist(err) {
			Log("No gx helper tool found for", env)
			return nil
		}
		return err
	}

	args = append([]string{"hook", hook}, args...)
	cmd := exec.Command(binname, args...)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Stdin = os.Stdin

	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("hook failed: %s", err)
	}

	return nil
}
<commit_msg>better support global installs and resuming installs<commit_after>package gxutil

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"path"
	"strings"

	mh "github.com/jbenet/go-multihash"
	ish "github.com/whyrusleeping/fallback-ipfs-shell"
	. "github.com/whyrusleeping/stump"
)

const PkgFileName = "package.json"

type PM struct {
	shell ish.Shell

	cfg *Config

	// hash of the 'empty' ipfs dir to avoid extra calls to object new
	blankDir string
}

func NewPM(cfg *Config) (*PM, error) {
	sh, err := ish.NewShell()
	if err != nil {
		return nil, err
	}

	return &PM{
		shell: sh,
		cfg:   cfg,
	}, nil
}

// InstallDeps recursively installs all dependencies for the given package
func (pm *PM) InstallDeps(pkg *Package, location string) error {
	Log("installing package: %s-%s", pkg.Name, pkg.Version)
	for _, dep := range pkg.Dependencies {

		// if its already local, skip it
		pkgdir := path.Join(location, dep.Hash)
		pkg := new(Package)
		err := FindPackageInDir(pkg, pkgdir)
		if err != nil {
			deppkg, err := pm.GetPackageLocalDaemon(dep.Hash, location)
			if err != nil {
				return fmt.Errorf("failed to fetch package: %s (%s):%s", dep.Name,
					dep.Hash, err)
			}
			pkg = deppkg
		}

		err = pm.InstallDeps(pkg, location)
		if err != nil {
			return err
		}
	}
	return nil
}

func (pm *PM) InitPkg(dir, name, lang string) error {
	// check for existing packagefile
	p := path.Join(dir, PkgFileName)
	_, err := os.Stat(p)
	if err == nil {
		return errors.New("package file already exists in working dir")
	}

	username := pm.cfg.User.Name
	if username == "" {
		u, err := user.Current()
		if err != nil {
			fmt.Errorf("error looking up current user: %s", err)
		}
		username = u.Username
	}

	pkg := new(Package)
	pkg.Name = name
	pkg.Language = lang
	pkg.Author = username
	pkg.Version = "1.0.0"

	// check if the user has a tool installed for the selected language
	CheckForHelperTools(lang)

	err = SavePackageFile(pkg, p)
	if err != nil {
		return err
	}

	err = TryRunHook("post-init", lang, dir)
	if err != nil {
		return err
	}
	return nil
}

func CheckForHelperTools(lang string) {
	_, err := exec.LookPath("gx-" + lang)
	if err == nil {
		return
	}

	if strings.Contains(err.Error(), "file not found") {
		Log("notice: no helper tool found for", lang)
		return
	}

	Error("checking for helper tool:", err)
}

func (pm *PM) ImportPackage(dir, dephash string) (*Dependency, error) {
	ndep, err := pm.GetPackage(dephash)
	if err != nil {
		return nil, err
	}

	for _, child := range ndep.Dependencies {
		_, err := pm.ImportPackage(dir, child.Hash)
		if err != nil {
			return nil, err
		}
	}

	err = RunPostImportHook(ndep.Language, dephash)
	if err != nil {
		Fatal(err)
	}

	return &Dependency{
		Name:    ndep.Name,
		Hash:    dephash,
		Version: ndep.Version,
	}, nil
}

func (pm *PM) ResolveDepName(name string) (string, error) {
	_, err := mh.FromB58String(name)
	if err == nil {
		return name, nil
	}

	if strings.Contains(name, "/") {
		parts := strings.Split(name, "/")
		rpath, ok := pm.cfg.GetRepos()[parts[0]]
		if !ok {
			return "", fmt.Errorf("unknown repo: '%s'", parts[0])
		}

		pkgs, err := pm.FetchRepo(rpath, true)
		if err != nil {
			return "", err
		}

		val, ok := pkgs[parts[1]]
		if !ok {
			return "", fmt.Errorf("package %s not found in repo %s", parts[1], parts[0])
		}

		return val, nil
	}

	out, err := pm.QueryRepos(name)
	if err != nil {
		return "", err
	}

	if len(out) == 0 {
		return "", fmt.Errorf("could not find package by name: %s", name)
	}

	if len(out) == 1 {
		for _, v := range out {
			return v, nil
		}
	}

	return "", fmt.Errorf("ambiguous ref, appears in multiple repos")
}

func RunPostImportHook(env, pkg string) error {
	return TryRunHook("post-import", env, pkg)
}

func RunReqCheckHook(env, pkg string) error {
	return TryRunHook("req-check", env, pkg)
}

func TryRunHook(hook, env string, args ...string) error {
	if env == "" {
		return nil
	}

	binname := "gx-" + env
	_, err := exec.LookPath(binname)
	if err != nil {
		if os.IsNotExist(err) {
			Log("No gx helper tool found for", env)
			return nil
		}
		return err
	}

	args = append([]string{"hook", hook}, args...)
	cmd := exec.Command(binname, args...)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Stdin = os.Stdin

	err = cmd.Run()
	if err != nil {
		return fmt.Errorf("hook failed: %s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package logrus

import (
	"fmt"
	"sort"
	"strings"
	"time"
)

const (
	nocolor = 0
	red     = 31
	green   = 32
	yellow  = 33
	blue    = 34
)

func init() {
	baseTimestamp = time.Now()
}

func miniTS() int {
	return int(time.Since(baseTimestamp) / time.Second)
}

type TextFormatter struct {
	// Set to true to bypass checking for a TTY before outputting colors.
	ForceColors bool
}

func (f *TextFormatter) Format(entry *Entry) ([]byte, error) {
	var serialized []byte

	if f.ForceColors || IsTerminal() {
		levelText := strings.ToUpper(entry.Data["level"].(string))[0:4]

		levelColor := blue

		if entry.Data["level"] == "warning" {
			levelColor = yellow
		} else if entry.Data["level"] == "error" ||
			entry.Data["level"] == "fatal" ||
			entry.Data["level"] == "panic" {
			levelColor = red
		}

		serialized = append(serialized, []byte(fmt.Sprintf("\x1b[%dm%s\x1b[0m[%04d] %-45s ", levelColor, levelText, miniTS(), entry.Data["msg"]))...)

		keys := make([]string, 0)
		for k, _ := range entry.Data {
			if k != "level" && k != "time" && k != "msg" {
				keys = append(keys, k)
			}
		}
		sort.Strings(keys)
		first := true
		for _, k := range keys {
			v := entry.Data[k]
			if first {
				first = false
			} else {
				serialized = append(serialized, ' ')
			}
			serialized = append(serialized, []byte(fmt.Sprintf("\x1b[%dm%s\x1b[0m=%v", levelColor, k, v))...)
		}
	} else {
		serialized = f.AppendKeyValue(serialized, "time", entry.Data["time"].(string))
		serialized = f.AppendKeyValue(serialized, "level", entry.Data["level"].(string))
		serialized = f.AppendKeyValue(serialized, "msg", entry.Data["msg"].(string))

		for key, value := range entry.Data {
			if key != "time" && key != "level" && key != "msg" {
				serialized = f.AppendKeyValue(serialized, key, value)
			}
		}
	}

	return append(serialized, '\n'), nil
}

func (f *TextFormatter) AppendKeyValue(serialized []byte, key, value interface{}) []byte {
	if _, ok := value.(string); ok {
		return append(serialized, []byte(fmt.Sprintf("%v=%q ", key, value))...)
	} else {
		return append(serialized, []byte(fmt.Sprintf("%v=%v ", key, value))...)
	}
}
<commit_msg>Clean up text_formatter code with a byte.Buffer<commit_after>package logrus

import (
	"bytes"
	"fmt"
	"sort"
	"strings"
	"time"
)

const (
	nocolor = 0
	red     = 31
	green   = 32
	yellow  = 33
	blue    = 34
)

func init() {
	baseTimestamp = time.Now()
}

func miniTS() int {
	return int(time.Since(baseTimestamp) / time.Second)
}

type TextFormatter struct {
	// Set to true to bypass checking for a TTY before outputting colors.
	ForceColors bool
}

func (f *TextFormatter) Format(entry *Entry) ([]byte, error) {
	b := &bytes.Buffer{}

	if f.ForceColors || IsTerminal() {
		levelText := strings.ToUpper(entry.Data["level"].(string))[0:4]

		levelColor := blue

		if entry.Data["level"] == "warning" {
			levelColor = yellow
		} else if entry.Data["level"] == "error" ||
			entry.Data["level"] == "fatal" ||
			entry.Data["level"] == "panic" {
			levelColor = red
		}

		fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%04d] %-44s ", levelColor, levelText, miniTS(), entry.Data["msg"])

		keys := make([]string, 0)
		for k, _ := range entry.Data {
			if k != "level" && k != "time" && k != "msg" {
				keys = append(keys, k)
			}
		}
		sort.Strings(keys)
		for _, k := range keys {
			v := entry.Data[k]
			fmt.Fprintf(b, " \x1b[%dm%s\x1b[0m=%v", levelColor, k, v)
		}
	} else {
		f.AppendKeyValue(b, "time", entry.Data["time"].(string))
		f.AppendKeyValue(b, "level", entry.Data["level"].(string))
		f.AppendKeyValue(b, "msg", entry.Data["msg"].(string))

		for key, value := range entry.Data {
			if key != "time" && key != "level" && key != "msg" {
				f.AppendKeyValue(b, key, value)
			}
		}
	}

	b.WriteByte('\n')
	return b.Bytes(), nil
}

func (f *TextFormatter) AppendKeyValue(b *bytes.Buffer, key, value interface{}) {
	if _, ok := value.(string); ok {
		fmt.Fprintf(b, "%v=%q ", key, value)
	} else {
		fmt.Fprintf(b, "%v=%v ", key, value)
	}
}
<|endoftext|>