<commit_before>package commands

import (
	"github.com/jfrogdev/jfrog-cli-go/bintray/utils"
	"github.com/jfrogdev/jfrog-cli-go/utils/cliutils"
	"github.com/jfrogdev/jfrog-cli-go/utils/config"
	"github.com/jfrogdev/jfrog-cli-go/utils/io/httputils"
	"errors"
	"github.com/jfrogdev/jfrog-cli-go/utils/cliutils/log"
	"fmt"
	"encoding/json"
	"strings"
)

func ShowAccessKeys(bintrayDetails *config.BintrayDetails, org string) error {
	path := GetAccessKeysPath(bintrayDetails, org)
	httpClientsDetails := utils.GetBintrayHttpClientDetails(bintrayDetails)
	log.Info("Getting access keys...")
	resp, body, _, _ := httputils.SendGet(path, true, httpClientsDetails)
	if resp.StatusCode != 200 {
		return cliutils.CheckError(errors.New("Bintray response: " + resp.Status + "\n" + cliutils.IndentJson(body)))
	}

	log.Debug("Bintray response:", resp.Status)
	log.Info("Access keys details:")
	fmt.Println(cliutils.IndentJson(body))
	return nil
}

func ShowAccessKey(flags *AccessKeyFlags, org string) (err error) {
	url := GetAccessKeyPath(flags.BintrayDetails, flags.Id, org)
	httpClientsDetails := utils.GetBintrayHttpClientDetails(flags.BintrayDetails)
	log.Info("Getting access key...")
	resp, body, _, _ := httputils.SendGet(url, true, httpClientsDetails)
	if resp.StatusCode != 200 {
		return cliutils.CheckError(errors.New("Bintray response: " + resp.Status + "\n" + cliutils.IndentJson(body)))
	}

	log.Debug("Bintray response:", resp.Status)
	log.Info("Access keys details:")
	fmt.Println(cliutils.IndentJson(body))
	return
}

func CreateAccessKey(flags *AccessKeyFlags, org string) (err error) {
	data, err := BuildAccessKeyJson(flags)
	if err != nil {
		return err
	}
	url := GetAccessKeysPath(flags.BintrayDetails, org)
	httpClientsDetails := utils.GetBintrayHttpClientDetails(flags.BintrayDetails)
	log.Info("Creating access key...")
	resp, body, err := httputils.SendPost(url, []byte(data), httpClientsDetails)
	if err != nil {
		return
	}
	if resp.StatusCode != 201 {
		return cliutils.CheckError(errors.New("Bintray response: " + resp.Status + "\n" + cliutils.IndentJson(body)))
	}

	log.Debug("Bintray response:", resp.Status)
	log.Info("Created access key, details:")
	fmt.Println(cliutils.IndentJson(body))
	return
}

func UpdateAccessKey(flags *AccessKeyFlags, org string) error {
	data, err := BuildAccessKeyJson(flags)
	if err != nil {
		return err
	}
	url := GetAccessKeyPath(flags.BintrayDetails, flags.Id, org)
	httpClientsDetails := utils.GetBintrayHttpClientDetails(flags.BintrayDetails)
	log.Info("Updating access key...")
	resp, body, err := httputils.SendPatch(url, []byte(data), httpClientsDetails)
	if err != nil {
		return err
	}
	if resp.StatusCode != 200 {
		return cliutils.CheckError(errors.New("Bintray response: " + resp.Status + "\n" + cliutils.IndentJson(body)))
	}

	log.Debug("Bintray response:", resp.Status)
	log.Info("Updated access key, details:")
	fmt.Println(cliutils.IndentJson(body))
	return nil
}

func BuildAccessKeyJson(flags *AccessKeyFlags) (string, error) {
	apiOnly, err := cliutils.StringToBool(flags.ApiOnly, true)
	if err != nil {
		return "", err
	}

	data := AccessKeyConfig{
		Id:             flags.Id,
		Expiry:         flags.Expiry,
		WhiteCidrs:     strings.Split(flags.WhiteCidrs, ","),
		BlackCidrs:     strings.Split(flags.BlackCidrs, ","),
		ApiOnly:        apiOnly,
		ExistenceCheck: ExistenceCheckConfig{
			Url:             flags.ExistenceCheckUrl,
			Cache_for_secs:  flags.ExistenceCheckCache},
	}
	requestContent, err := json.Marshal(data)
	if err != nil {
		return "", cliutils.CheckError(errors.New("Failed to execute request. " + cliutils.GetDocumentationMessage()))
	}
	return string(requestContent), nil
}

func DeleteAccessKey(flags *AccessKeyFlags, org string) error {
	url := GetAccessKeyPath(flags.BintrayDetails, flags.Id, org)
	httpClientsDetails := utils.GetBintrayHttpClientDetails(flags.BintrayDetails)
	log.Info("Deleting access key...")
	resp, body, err := httputils.SendDelete(url, nil, httpClientsDetails)
	if err != nil {
		return err
	}
	if resp.StatusCode != 200 {
		return cliutils.CheckError(errors.New("Bintray response: " + resp.Status + "\n" + cliutils.IndentJson(body)))
	}

	log.Debug("Bintray response:", resp.Status)
	log.Info("Deleted access key.")
	return nil
}

func GetAccessKeyPath(bintrayDetails *config.BintrayDetails, id, org string) string {
	return GetAccessKeysPath(bintrayDetails, org) + "/" + id
}

func GetAccessKeysPath(bintrayDetails *config.BintrayDetails, org string) string {
	if org == "" {
		return bintrayDetails.ApiUrl + "users/" + bintrayDetails.User + "/download_keys"
	}
	return bintrayDetails.ApiUrl + "orgs/" + org + "/download_keys"
}

type AccessKeyFlags struct {
	BintrayDetails      *config.BintrayDetails
	Id                  string
	Password            string
	Expiry              int64
	ExistenceCheckUrl   string
	ExistenceCheckCache int
	WhiteCidrs          string
	BlackCidrs          string
	ApiOnly             string
}

type AccessKeyConfig struct {
	Id             string                `json:"id,omitempty"`
	Expiry         int64                 `json:"expiry,omitempty"`
	ExistenceCheck ExistenceCheckConfig  `json:"existence_check,omitempty"`
	WhiteCidrs     []string              `json:"white_cidrs,omitempty"`
	BlackCidrs     []string              `json:"black_cidrs,omitempty"`
	ApiOnly        bool                  `json:"api_only"`
}

type ExistenceCheckConfig struct {
	Url            string  `json:"url,omitempty"`
	Cache_for_secs int     `json:"cache_for_secs,omitempty"`
}<commit_msg>Invalid CIDR response to access key creation #75<commit_after>package commands

import (
	"github.com/jfrogdev/jfrog-cli-go/bintray/utils"
	"github.com/jfrogdev/jfrog-cli-go/utils/cliutils"
	"github.com/jfrogdev/jfrog-cli-go/utils/config"
	"github.com/jfrogdev/jfrog-cli-go/utils/io/httputils"
	"errors"
	"github.com/jfrogdev/jfrog-cli-go/utils/cliutils/log"
	"fmt"
	"encoding/json"
	"strings"
)

func ShowAccessKeys(bintrayDetails *config.BintrayDetails, org string) error {
	path := GetAccessKeysPath(bintrayDetails, org)
	httpClientsDetails := utils.GetBintrayHttpClientDetails(bintrayDetails)
	log.Info("Getting access keys...")
	resp, body, _, _ := httputils.SendGet(path, true, httpClientsDetails)
	if resp.StatusCode != 200 {
		return cliutils.CheckError(errors.New("Bintray response: " + resp.Status + "\n" + cliutils.IndentJson(body)))
	}

	log.Debug("Bintray response:", resp.Status)
	log.Info("Access keys details:")
	fmt.Println(cliutils.IndentJson(body))
	return nil
}

func ShowAccessKey(flags *AccessKeyFlags, org string) (err error) {
	url := GetAccessKeyPath(flags.BintrayDetails, flags.Id, org)
	httpClientsDetails := utils.GetBintrayHttpClientDetails(flags.BintrayDetails)
	log.Info("Getting access key...")
	resp, body, _, _ := httputils.SendGet(url, true, httpClientsDetails)
	if resp.StatusCode != 200 {
		return cliutils.CheckError(errors.New("Bintray response: " + resp.Status + "\n" + cliutils.IndentJson(body)))
	}

	log.Debug("Bintray response:", resp.Status)
	log.Info("Access keys details:")
	fmt.Println(cliutils.IndentJson(body))
	return
}

func CreateAccessKey(flags *AccessKeyFlags, org string) (err error) {
	data, err := BuildAccessKeyJson(flags)
	if err != nil {
		return err
	}
	url := GetAccessKeysPath(flags.BintrayDetails, org)
	httpClientsDetails := utils.GetBintrayHttpClientDetails(flags.BintrayDetails)
	log.Info("Creating access key...")
	resp, body, err := httputils.SendPost(url, []byte(data), httpClientsDetails)
	if err != nil {
		return
	}
	if resp.StatusCode != 201 {
		return cliutils.CheckError(errors.New("Bintray response: " + resp.Status + "\n" + cliutils.IndentJson(body)))
	}

	log.Debug("Bintray response:", resp.Status)
	log.Info("Created access key, details:")
	fmt.Println(cliutils.IndentJson(body))
	return
}

func UpdateAccessKey(flags *AccessKeyFlags, org string) error {
	data, err := BuildAccessKeyJson(flags)
	if err != nil {
		return err
	}
	url := GetAccessKeyPath(flags.BintrayDetails, flags.Id, org)
	httpClientsDetails := utils.GetBintrayHttpClientDetails(flags.BintrayDetails)
	log.Info("Updating access key...")
	resp, body, err := httputils.SendPatch(url, []byte(data), httpClientsDetails)
	if err != nil {
		return err
	}
	if resp.StatusCode != 200 {
		return cliutils.CheckError(errors.New("Bintray response: " + resp.Status + "\n" + cliutils.IndentJson(body)))
	}

	log.Debug("Bintray response:", resp.Status)
	log.Info("Updated access key, details:")
	fmt.Println(cliutils.IndentJson(body))
	return nil
}

func BuildAccessKeyJson(flags *AccessKeyFlags) (string, error) {
	apiOnly, err := cliutils.StringToBool(flags.ApiOnly, true)
	if err != nil {
		return "", err
	}

	var whiteCidrs []string
	if flags.WhiteCidrs != "" {
		whiteCidrs = strings.Split(flags.WhiteCidrs, ",")
	}

	var blackCidrs []string
	if flags.BlackCidrs != "" {
		blackCidrs = strings.Split(flags.BlackCidrs, ",")
	}

	data := AccessKeyConfig{
		Id:         flags.Id,
		Expiry:     flags.Expiry,
		WhiteCidrs: whiteCidrs,
		BlackCidrs: blackCidrs,
		ApiOnly:    apiOnly,
		ExistenceCheck: ExistenceCheckConfig{
			Url:             flags.ExistenceCheckUrl,
			Cache_for_secs:  flags.ExistenceCheckCache},
	}
	requestContent, err := json.Marshal(data)
	if err != nil {
		return "", cliutils.CheckError(errors.New("Failed to execute request. " + cliutils.GetDocumentationMessage()))
	}
	return string(requestContent), nil
}

func DeleteAccessKey(flags *AccessKeyFlags, org string) error {
	url := GetAccessKeyPath(flags.BintrayDetails, flags.Id, org)
	httpClientsDetails := utils.GetBintrayHttpClientDetails(flags.BintrayDetails)
	log.Info("Deleting access key...")
	resp, body, err := httputils.SendDelete(url, nil, httpClientsDetails)
	if err != nil {
		return err
	}
	if resp.StatusCode != 200 {
		return cliutils.CheckError(errors.New("Bintray response: " + resp.Status + "\n" + cliutils.IndentJson(body)))
	}

	log.Debug("Bintray response:", resp.Status)
	log.Info("Deleted access key.")
	return nil
}

func GetAccessKeyPath(bintrayDetails *config.BintrayDetails, id, org string) string {
	return GetAccessKeysPath(bintrayDetails, org) + "/" + id
}

func GetAccessKeysPath(bintrayDetails *config.BintrayDetails, org string) string {
	if org == "" {
		return bintrayDetails.ApiUrl + "users/" + bintrayDetails.User + "/download_keys"
	}
	return bintrayDetails.ApiUrl + "orgs/" + org + "/download_keys"
}

type AccessKeyFlags struct {
	BintrayDetails      *config.BintrayDetails
	Id                  string
	Password            string
	Expiry              int64
	ExistenceCheckUrl   string
	ExistenceCheckCache int
	WhiteCidrs          string
	BlackCidrs          string
	ApiOnly             string
}

type AccessKeyConfig struct {
	Id             string                `json:"id,omitempty"`
	Expiry         int64                 `json:"expiry,omitempty"`
	ExistenceCheck ExistenceCheckConfig  `json:"existence_check,omitempty"`
	WhiteCidrs     []string              `json:"white_cidrs,omitempty"`
	BlackCidrs     []string              `json:"black_cidrs,omitempty"`
	ApiOnly        bool                  `json:"api_only"`
}

type ExistenceCheckConfig struct {
	Url            string  `json:"url,omitempty"`
	Cache_for_secs int     `json:"cache_for_secs,omitempty"`
}<|endoftext|>


<commit_before>package frontend

import (
	"fmt"
	"github.com/rfjakob/gocryptfs/cryptfs"
	"github.com/rfjakob/cluefs/lib/cluefs"
	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
	"golang.org/x/net/context"
)

type Dir struct {
	*cluefs.Dir
	crfs *cryptfs.CryptFS
}

func NewDir(parent string, name string, fs *FS) *Dir {
	fmt.Printf("NewDir parent=%s name=%s\n", parent, name)
	return &Dir {
		Dir: cluefs.NewDir(parent, name, fs.ClueFS),
		crfs: fs.CryptFS,
	}
}

func (d *Dir) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fusefs.Handle, error) {
	fmt.Printf("Open\n")
	h, err := d.Dir.Open(ctx, req, resp)
	if err != nil {
		return nil, err
	}
	clueDir := h.(*cluefs.Dir)

	return Dir {
		Dir: clueDir,
		crfs: d.crfs,
	}, nil
}

func (d *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fusefs.Node, error) {
	fmt.Printf("Lookup %s\n", req.Name)
	req.Name = d.crfs.EncryptPath(req.Name)
	n, err := d.Dir.Lookup(ctx, req, resp)
	if err != nil {
		return nil, err
	}
	clueDir, ok := n.(*cluefs.Dir)
	if ok {
		return &Dir { Dir: clueDir }, nil
	} else {
		clueFile := n.(*cluefs.File)
		return &File { File: clueFile }, nil
	}
}

func (d *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
	fmt.Printf("ReadDirAll\n")
	entries, err := d.Dir.ReadDirAll(ctx)
	if err != nil {
		return nil, err
	}
	var decrypted []fuse.Dirent
	for _, e := range entries {
		if e.Name == "." || e.Name == ".." {
			decrypted = append(decrypted, e)
			continue
		}
		newName, err := d.crfs.DecryptPath(e.Name)
		if err != nil {
			fmt.Printf("ReadDirAll: Error decoding \"%s\": %s\n", e.Name, err.Error())
			continue
		}
		e.Name = newName
		decrypted = append(decrypted, e)
	}
	return decrypted, nil
}

func (d *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fusefs.Node, error) {
	fmt.Printf("Mkdir %s\n", req.Name)
	req.Name = d.crfs.EncryptPath(req.Name)
	n, err := d.Dir.Mkdir(ctx, req)
	if err != nil {
		return nil, err
	}
	clueDir := n.(*cluefs.Dir)
	return &Dir {
		Dir: clueDir,
		crfs: d.crfs,
	}, nil
}

func (d *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
	fmt.Printf("Remove\n")
	req.Name = d.crfs.EncryptPath(req.Name)
	return d.Dir.Remove(ctx, req)
}

func (d *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fusefs.Node, fusefs.Handle, error) {
	fmt.Printf("Create\n")
	req.Name = d.crfs.EncryptPath(req.Name)
	n, _, err := d.Dir.Create(ctx, req, resp)
	if err != nil {
		return nil, nil, err
	}
	clueFile := n.(*cluefs.File)
	cryptFile := &File {File: clueFile}
	return cryptFile, cryptFile, nil
}
<commit_msg>Filename encryption working<commit_after>package frontend

import (
	"fmt"
	"github.com/rfjakob/gocryptfs/cryptfs"
	"github.com/rfjakob/cluefs/lib/cluefs"
	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
	"golang.org/x/net/context"
)

type Dir struct {
	*cluefs.Dir
	crfs *cryptfs.CryptFS
}

func NewDir(parent string, name string, fs *FS) *Dir {
	fmt.Printf("NewDir parent=%s name=%s\n", parent, name)
	return &Dir {
		Dir: cluefs.NewDir(parent, name, fs.ClueFS),
		crfs: fs.CryptFS,
	}
}

func (d *Dir) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fusefs.Handle, error) {
	fmt.Printf("Open\n")
	h, err := d.Dir.Open(ctx, req, resp)
	if err != nil {
		return nil, err
	}
	clueDir := h.(*cluefs.Dir)

	return &Dir {
		Dir: clueDir,
		crfs: d.crfs,
	}, nil
}

func (d *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fusefs.Node, error) {
	fmt.Printf("Lookup %s\n", req.Name)
	req.Name = d.crfs.EncryptPath(req.Name)
	n, err := d.Dir.Lookup(ctx, req, resp)
	if err != nil {
		return nil, err
	}
	clueDir, ok := n.(*cluefs.Dir)
	if ok {
		return &Dir { Dir: clueDir }, nil
	} else {
		clueFile := n.(*cluefs.File)
		return &File { File: clueFile }, nil
	}
}

func (d *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
	fmt.Printf("ReadDirAll\n")
	entries, err := d.Dir.ReadDirAll(ctx)
	if err != nil {
		return nil, err
	}
	var decrypted []fuse.Dirent
	for _, e := range entries {
		if e.Name == "." || e.Name == ".." {
			decrypted = append(decrypted, e)
			continue
		}
		newName, err := d.crfs.DecryptPath(e.Name)
		if err != nil {
			fmt.Printf("ReadDirAll: Error decoding \"%s\": %s\n", e.Name, err.Error())
			continue
		}
		e.Name = newName
		decrypted = append(decrypted, e)
	}
	return decrypted, nil
}

func (d *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fusefs.Node, error) {
	fmt.Printf("Mkdir %s\n", req.Name)
	req.Name = d.crfs.EncryptPath(req.Name)
	n, err := d.Dir.Mkdir(ctx, req)
	if err != nil {
		return nil, err
	}
	clueDir := n.(*cluefs.Dir)
	return &Dir {
		Dir: clueDir,
		crfs: d.crfs,
	}, nil
}

func (d *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
	fmt.Printf("Remove\n")
	req.Name = d.crfs.EncryptPath(req.Name)
	return d.Dir.Remove(ctx, req)
}

func (d *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fusefs.Node, fusefs.Handle, error) {
	fmt.Printf("Create\n")
	req.Name = d.crfs.EncryptPath(req.Name)
	n, _, err := d.Dir.Create(ctx, req, resp)
	if err != nil {
		return nil, nil, err
	}
	clueFile := n.(*cluefs.File)
	cryptFile := &File {File: clueFile}
	return cryptFile, cryptFile, nil
}
<|endoftext|>


<commit_before>// Copyright 2014, Hǎiliàng Wáng. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package errors

import (
	"testing"
)

func TestGetPos(t *testing.T) {
	var check = func(int, int, int) *Pos {
		return GetPos(1)
	}
	pos := check(
		0,
		1,
		2,
	) // this is the line that should be restored.
	exp := &Pos{"format_test.go", 19}
	if pos.BasePath() != exp.File {
		t.Fatalf(`Expect "%v" to be "%v"`, pos.BasePath(), exp.File)
	}
	if pos.Line != exp.Line {
		t.Fatalf(`Expect "%v" to be "%v"`, pos.Line, exp.Line)
	}
}
<commit_msg>fix position since runtime has changed<commit_after>// Copyright 2014, Hǎiliàng Wáng. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package errors

import (
	"testing"
)

func TestGetPos(t *testing.T) {
	var check = func(int, int, int) *Pos {
		return GetPos(1)
	}
	pos := check( // this is the line that should be restored.
		0,
		1,
		2,
	)
	exp := &Pos{"format_test.go", 15}
	if pos.BasePath() != exp.File {
		t.Fatalf(`Expect "%v" to be "%v"`, pos.BasePath(), exp.File)
	}
	if pos.Line != exp.Line {
		t.Fatalf(`Expect "%v" to be "%v"`, pos.Line, exp.Line)
	}
}
<|endoftext|>


<commit_before>package w32

// nt!_ALPC_MESSAGE_ATTRIBUTES
//  +0x000 AllocatedAttributes : Uint4B
//  +0x004 ValidAttributes  : Uint4B
type ALPC_MESSAGE_ATTRIBUTES struct {
	AllocatedAttributes uint32
	ValidAttributes     uint32
}

// nt!_CLIENT_ID
//  +0x000 UniqueProcess    : Ptr64 Void
//  +0x008 UniqueThread     : Ptr64 Void
type CLIENT_ID struct {
	UniqueProcess uintptr
	UniqueThread  uintptr
}

// nt!_UNICODE_STRING
//  +0x000 Length           : Uint2B
//  +0x002 MaximumLength    : Uint2B
//  +0x008 Buffer           : Ptr64 Uint2B
type UNICODE_STRING struct {
	Length        uint16
	MaximumLength uint16
	Buffer        unsafe.Pointer
}

// nt!_OBJECT_ATTRIBUTES
//  +0x000 Length           : Uint4B
//  +0x008 RootDirectory    : Ptr64 Void
//  +0x010 ObjectName       : Ptr64 _UNICODE_STRING
//  +0x018 Attributes       : Uint4B
//  +0x020 SecurityDescriptor : Ptr64 Void
//  +0x028 SecurityQualityOfService : Ptr64 Void
type OBJECT_ATTRIBUTES struct {
	Length                   uint32
	padding1                 [4]byte
	RootDirectory            HANDLE
	ObjectName               *UNICODE_STRING
	Attributes               uint32
	padding2                 [4]byte
	SecurityDescriptor       *SECURITY_DESCRIPTOR
	SecurityQualityOfService *SECURITY_QUALITY_OF_SERVICE
}

// cf: http://j00ru.vexillium.org/?p=502
// nt!_PORT_MESSAGE
//    +0x000 u1               : <unnamed-tag>
//    +0x004 u2               : <unnamed-tag>
//    +0x008 ClientId         : _CLIENT_ID
//    +0x008 DoNotUseThisField : Float
//    +0x018 MessageId        : Uint4B
//    +0x020 ClientViewSize   : Uint8B
//    +0x020 CallbackId       : Uint4B
type PORT_MESSAGE struct {
	DataLength     uint16 // These are the two unnamed unions
	TotalLength    uint16 // without Length and ZeroInit
	Type           uint16
	DataInfoOffset uint16
	ClientId       CLIENT_ID
	MessageId      uint32
	padding        [4]byte
	ClientViewSize uint64
}

func (pm PORT_MESSAGE) CallbackId() uint32 {
	return uint32(pm.ClientViewSize >> 32)
}

func (pm PORT_MESSAGE) DoNotUseThisField() float {
	panic("WE TOLD YOU NOT TO USE THIS FIELD")
}

type SECURITY_IMPERSONATION_LEVEL int

const (
	SecurityAnonymous SECURITY_IMPERSONATION_LEVEL = iota
	SecurityIdentification
	SecurityImpersonation
	SecurityDelegation
)

// http://www.nirsoft.net/kernel_struct/vista/SECURITY_QUALITY_OF_SERVICE.html
// Added internal padding to make it 0xC bytes, as per the dt output below
type SECURITY_QUALITY_OF_SERVICE struct {
	Length              uint32
	ImpersonationLevel  SECURITY_IMPERSONATION_LEVEL
	ContextTrackingMode byte
	EffectiveOnly       byte
	padding             [2]byte
}

// nt!_ALPC_PORT_ATTRIBUTES
//  +0x000 Flags            : Uint4B
//  +0x004 SecurityQos      : _SECURITY_QUALITY_OF_SERVICE
//  +0x010 MaxMessageLength : Uint8B
//  +0x018 MemoryBandwidth  : Uint8B
//  +0x020 MaxPoolUsage     : Uint8B
//  +0x028 MaxSectionSize   : Uint8B
//  +0x030 MaxViewSize      : Uint8B
//  +0x038 MaxTotalSectionSize : Uint8B
//  +0x040 DupObjectTypes   : Uint4B
//  +0x044 Reserved         : Uint4B
type ALPC_PORT_ATTRIBUTES struct {
	Flags               uint32
	SecurityQos         SECURITY_QUALITY_OF_SERVICE
	MaxMessageLength    uint64
	MemoryBandwidth     uint64
	MaxPoolUsage        uint64
	MaxSectionSize      uint64
	MaxViewSize         uint64
	MaxTotalSectionSize uint64
	DupObjectTypes      uint32
	Reserved            uint32
}

//   typedef struct _TRANSFERRED_MESSAGE
// {
//     PORT_MESSAGE Header;
//     ULONG   Command;
//     WCHAR   MessageText[48];
// }
type TRANSFERRED_MESSAGE struct {
	Header      PORT_MESSAGE
	Command     uint32
	MessageText [48]uint16
}
<commit_msg>bugfix<commit_after>package w32

import (
	"unsafe"
)

// nt!_ALPC_MESSAGE_ATTRIBUTES
//  +0x000 AllocatedAttributes : Uint4B
//  +0x004 ValidAttributes  : Uint4B
type ALPC_MESSAGE_ATTRIBUTES struct {
	AllocatedAttributes uint32
	ValidAttributes     uint32
}

// nt!_CLIENT_ID
//  +0x000 UniqueProcess    : Ptr64 Void
//  +0x008 UniqueThread     : Ptr64 Void
type CLIENT_ID struct {
	UniqueProcess uintptr
	UniqueThread  uintptr
}

// nt!_UNICODE_STRING
//  +0x000 Length           : Uint2B
//  +0x002 MaximumLength    : Uint2B
//  +0x008 Buffer           : Ptr64 Uint2B
type UNICODE_STRING struct {
	Length        uint16
	MaximumLength uint16
	Buffer        unsafe.Pointer
}

// nt!_OBJECT_ATTRIBUTES
//  +0x000 Length           : Uint4B
//  +0x008 RootDirectory    : Ptr64 Void
//  +0x010 ObjectName       : Ptr64 _UNICODE_STRING
//  +0x018 Attributes       : Uint4B
//  +0x020 SecurityDescriptor : Ptr64 Void
//  +0x028 SecurityQualityOfService : Ptr64 Void
type OBJECT_ATTRIBUTES struct {
	Length                   uint32
	padding1                 [4]byte
	RootDirectory            HANDLE
	ObjectName               *UNICODE_STRING
	Attributes               uint32
	padding2                 [4]byte
	SecurityDescriptor       *SECURITY_DESCRIPTOR
	SecurityQualityOfService *SECURITY_QUALITY_OF_SERVICE
}

// cf: http://j00ru.vexillium.org/?p=502
// nt!_PORT_MESSAGE
//    +0x000 u1               : <unnamed-tag>
//    +0x004 u2               : <unnamed-tag>
//    +0x008 ClientId         : _CLIENT_ID
//    +0x008 DoNotUseThisField : Float
//    +0x018 MessageId        : Uint4B
//    +0x020 ClientViewSize   : Uint8B
//    +0x020 CallbackId       : Uint4B
type PORT_MESSAGE struct {
	DataLength     uint16 // These are the two unnamed unions
	TotalLength    uint16 // without Length and ZeroInit
	Type           uint16
	DataInfoOffset uint16
	ClientId       CLIENT_ID
	MessageId      uint32
	padding        [4]byte
	ClientViewSize uint64
}

func (pm PORT_MESSAGE) CallbackId() uint32 {
	return uint32(pm.ClientViewSize >> 32)
}

func (pm PORT_MESSAGE) DoNotUseThisField() float {
	panic("WE TOLD YOU NOT TO USE THIS FIELD")
}

type SECURITY_IMPERSONATION_LEVEL int

const (
	SecurityAnonymous SECURITY_IMPERSONATION_LEVEL = iota
	SecurityIdentification
	SecurityImpersonation
	SecurityDelegation
)

// http://www.nirsoft.net/kernel_struct/vista/SECURITY_QUALITY_OF_SERVICE.html
// Added internal padding to make it 0xC bytes, as per the dt output below
type SECURITY_QUALITY_OF_SERVICE struct {
	Length              uint32
	ImpersonationLevel  SECURITY_IMPERSONATION_LEVEL
	ContextTrackingMode byte
	EffectiveOnly       byte
	padding             [2]byte
}

// nt!_ALPC_PORT_ATTRIBUTES
//  +0x000 Flags            : Uint4B
//  +0x004 SecurityQos      : _SECURITY_QUALITY_OF_SERVICE
//  +0x010 MaxMessageLength : Uint8B
//  +0x018 MemoryBandwidth  : Uint8B
//  +0x020 MaxPoolUsage     : Uint8B
//  +0x028 MaxSectionSize   : Uint8B
//  +0x030 MaxViewSize      : Uint8B
//  +0x038 MaxTotalSectionSize : Uint8B
//  +0x040 DupObjectTypes   : Uint4B
//  +0x044 Reserved         : Uint4B
type ALPC_PORT_ATTRIBUTES struct {
	Flags               uint32
	SecurityQos         SECURITY_QUALITY_OF_SERVICE
	MaxMessageLength    uint64
	MemoryBandwidth     uint64
	MaxPoolUsage        uint64
	MaxSectionSize      uint64
	MaxViewSize         uint64
	MaxTotalSectionSize uint64
	DupObjectTypes      uint32
	Reserved            uint32
}

//   typedef struct _TRANSFERRED_MESSAGE
// {
//     PORT_MESSAGE Header;
//     ULONG   Command;
//     WCHAR   MessageText[48];
// }
type TRANSFERRED_MESSAGE struct {
	Header      PORT_MESSAGE
	Command     uint32
	MessageText [48]uint16
}
<|endoftext|>


<commit_before>package zeusclient

import (
	"os"
	"os/signal"
	"net"
	"regexp"
	"strings"
	"strconv"
	"syscall"

	"github.com/kr/pty"
	"github.com/burke/ttyutils"
	usock "github.com/burke/zeus/unixsocket"
)

const (
	zeusSockName = ".zeus.sock"
	sigIntStr = "\x03"
	sigQuitStr = "\x1C"
	sigTstpStr = "\x1A"
)

var signalRegex = regexp.MustCompile(sigIntStr + "|" + sigQuitStr + "|" + sigTstpStr)

func Run() {
	master, slave, err := pty.Open()
	if err != nil {
		panic(err)
	}
	defer master.Close()
	defer slave.Close()

	if ttyutils.IsTerminal(os.Stdout.Fd()) {
		oldState, err := ttyutils.MakeTerminalRaw(os.Stdout.Fd())
		if err != nil {
			panic(err)
		}
		defer ttyutils.RestoreTerminalState(os.Stdout.Fd(), oldState)
	}

	ttyutils.MirrorWinsize(os.Stdout, master)

	addr, err := net.ResolveUnixAddr("unixgram", zeusSockName)
	if err != nil {
		panic("Can't resolve server address")
	}

	// TODO: WINCH
	conn, err := net.DialUnix("unix", nil, addr)
	if err != nil {
		panic("Can't connect to Master")
	}

	msg := "Q:console:[]\n"
	conn.Write([]byte(msg))

	usock.SendFdOverUnixSocket(conn, int(slave.Fd()))
	slave.Close()

	msg, _, err = usock.ReadFromUnixSocket(conn)
	if err != nil {
		panic(err)
	}

	parts := strings.Split(msg, "\n")
	pid, err := strconv.Atoi(parts[0])
	if err != nil {
		panic(err)
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGWINCH, syscall.SIGCONT)
	go func() {
		for sig := range c {
			if sig == syscall.SIGCONT {
				syscall.Kill(pid, syscall.SIGCONT)
			} else if sig == syscall.SIGWINCH {
				ttyutils.MirrorWinsize(os.Stdout, master)
				syscall.Kill(pid, syscall.SIGWINCH)
			}
		}
	}()

	println("PID:", pid)
	var exitStatus int = -1
	if len(parts) > 2 {
		exitStatus, err = strconv.Atoi(parts[0])
		if err != nil {
			panic(err)
		}
	}

	eof := make(chan bool)
	go func() {
		for {
			buf := make([]byte,1024)
			n, err := master.Read(buf)
			if err != nil {
				eof <- true
				break
			}
			os.Stdout.Write(buf[:n])
		}
	}()

	go func() {
		for {
			buf := make([]byte,1024)
			n, err := os.Stdin.Read(buf)
			if err != nil {
				eof <- true
				break
			}
			// TODO: Since we have a byte array, this could actually just check integer values...
			matches := signalRegex.FindAll(buf, 9999)
			for _, match := range matches {
				if m := string(match); m == sigIntStr {
					syscall.Kill(pid, syscall.SIGINT)
				} else if m == sigQuitStr {
					syscall.Kill(pid, syscall.SIGQUIT)
				} else if m == sigTstpStr {
					syscall.Kill(pid, syscall.SIGTSTP)
					syscall.Kill(os.Getpid(), syscall.SIGTSTP)
				}
			}
			master.Write(buf[:n])
		}
	}()

	<- eof

	if exitStatus == -1 {
		msg, _, err = usock.ReadFromUnixSocket(conn)
		if err != nil {
			panic(err)
		}
		parts := strings.Split(msg, "\n")
		exitStatus, err = strconv.Atoi(parts[0])
		if err != nil {
			panic(err)
		}
	}

	os.Exit(exitStatus)

}

<commit_msg>Remove incorrect comment<commit_after>package zeusclient

import (
	"os"
	"os/signal"
	"net"
	"regexp"
	"strings"
	"strconv"
	"syscall"

	"github.com/kr/pty"
	"github.com/burke/ttyutils"
	usock "github.com/burke/zeus/unixsocket"
)

const (
	zeusSockName = ".zeus.sock"
	sigIntStr = "\x03"
	sigQuitStr = "\x1C"
	sigTstpStr = "\x1A"
)

var signalRegex = regexp.MustCompile(sigIntStr + "|" + sigQuitStr + "|" + sigTstpStr)

func Run() {
	master, slave, err := pty.Open()
	if err != nil {
		panic(err)
	}
	defer master.Close()
	defer slave.Close()

	if ttyutils.IsTerminal(os.Stdout.Fd()) {
		oldState, err := ttyutils.MakeTerminalRaw(os.Stdout.Fd())
		if err != nil {
			panic(err)
		}
		defer ttyutils.RestoreTerminalState(os.Stdout.Fd(), oldState)
	}

	ttyutils.MirrorWinsize(os.Stdout, master)

	addr, err := net.ResolveUnixAddr("unixgram", zeusSockName)
	if err != nil {
		panic("Can't resolve server address")
	}

	conn, err := net.DialUnix("unix", nil, addr)
	if err != nil {
		panic("Can't connect to Master")
	}

	msg := "Q:console:[]\n"
	conn.Write([]byte(msg))

	usock.SendFdOverUnixSocket(conn, int(slave.Fd()))
	slave.Close()

	msg, _, err = usock.ReadFromUnixSocket(conn)
	if err != nil {
		panic(err)
	}

	parts := strings.Split(msg, "\n")
	pid, err := strconv.Atoi(parts[0])
	if err != nil {
		panic(err)
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGWINCH, syscall.SIGCONT)
	go func() {
		for sig := range c {
			if sig == syscall.SIGCONT {
				syscall.Kill(pid, syscall.SIGCONT)
			} else if sig == syscall.SIGWINCH {
				ttyutils.MirrorWinsize(os.Stdout, master)
				syscall.Kill(pid, syscall.SIGWINCH)
			}
		}
	}()

	println("PID:", pid)
	var exitStatus int = -1
	if len(parts) > 2 {
		exitStatus, err = strconv.Atoi(parts[0])
		if err != nil {
			panic(err)
		}
	}

	eof := make(chan bool)
	go func() {
		for {
			buf := make([]byte,1024)
			n, err := master.Read(buf)
			if err != nil {
				eof <- true
				break
			}
			os.Stdout.Write(buf[:n])
		}
	}()

	go func() {
		for {
			buf := make([]byte,1024)
			n, err := os.Stdin.Read(buf)
			if err != nil {
				eof <- true
				break
			}
			// TODO: Since we have a byte array, this could actually just check integer values...
			matches := signalRegex.FindAll(buf, 9999)
			for _, match := range matches {
				if m := string(match); m == sigIntStr {
					syscall.Kill(pid, syscall.SIGINT)
				} else if m == sigQuitStr {
					syscall.Kill(pid, syscall.SIGQUIT)
				} else if m == sigTstpStr {
					syscall.Kill(pid, syscall.SIGTSTP)
					syscall.Kill(os.Getpid(), syscall.SIGTSTP)
				}
			}
			master.Write(buf[:n])
		}
	}()

	<- eof

	if exitStatus == -1 {
		msg, _, err = usock.ReadFromUnixSocket(conn)
		if err != nil {
			panic(err)
		}
		parts := strings.Split(msg, "\n")
		exitStatus, err = strconv.Atoi(parts[0])
		if err != nil {
			panic(err)
		}
	}

	os.Exit(exitStatus)

}

<|endoftext|>


<commit_before>package bootstrapping

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/url"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
	"time"

	log "github.com/Sirupsen/logrus"
	singleinstance "github.com/allan-simon/go-singleinstance"
	"github.com/codegangsta/cli"
	"github.com/ingrammicro/concerto/api/blueprint"
	"github.com/ingrammicro/concerto/api/types"
	"github.com/ingrammicro/concerto/cmd"
	"github.com/ingrammicro/concerto/utils"
	"github.com/ingrammicro/concerto/utils/format"
)

const (
	//DefaultTimingInterval Default period for looping
	DefaultTimingInterval = 600 // 600 seconds = 10 minutes
	DefaultTimingSplay    = 360 // seconds
	DefaultThresholdLines = 10
	ProcessLockFile       = "imco-bootstrapping.lock"
	RetriesNumber         = 5
)

type bootstrappingProcess struct {
	startedAt                    time.Time
	finishedAt                   time.Time
	policyfiles                  []policyfile
	attributes                   attributes
	thresholdLines               int
	directoryPath                string
	appliedPolicyfileRevisionIDs map[string]string
}
type attributes struct {
	revisionID string
	rawData    *json.RawMessage
}

var allPolicyfilesSuccessfullyApplied bool

type policyfile types.BootstrappingPolicyfile

func (pf policyfile) Name() string {
	return strings.Join([]string{pf.ID, "-", pf.RevisionID}, "")
}

func (pf *policyfile) FileName() string {
	return strings.Join([]string{pf.Name(), "tgz"}, ".")
}

func (pf *policyfile) QueryURL() (string, error) {
	if pf.DownloadURL == "" {
		return "", fmt.Errorf("obtaining URL query: empty download URL")
	}
	url, err := url.Parse(pf.DownloadURL)
	if err != nil {
		return "", fmt.Errorf("parsing URL to extract query: %v", err)
	}
	return fmt.Sprintf("%s?%s", url.Path, url.RawQuery), nil
}

func (pf *policyfile) TarballPath(dir string) string {
	return filepath.Join(dir, pf.FileName())
}

func (pf *policyfile) Path(dir string) string {
	return filepath.Join(dir, pf.Name())
}

func (a *attributes) FileName() string {
	return fmt.Sprintf("attrs-%s.json", a.revisionID)
}

func (a *attributes) FilePath(dir string) string {
	return filepath.Join(dir, a.FileName())
}

// Handle signals
func handleSysSignals(cancelFunc context.CancelFunc) {
	log.Debug("handleSysSignals")

	gracefulStop := make(chan os.Signal, 1)
	signal.Notify(gracefulStop, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL)
	log.Debug("Ending, signal detected:", <-gracefulStop)
	cancelFunc()
}

// Returns the full path to the tmp directory joined with lock management file name
func getProcessLockFilePath() string {
	return filepath.Join(os.TempDir(), string(os.PathSeparator), ProcessLockFile)
}

// Returns the full path to the tmp directory
func generateWorkspaceDir() (string, error) {
	dir := filepath.Join(os.TempDir(), "imco")
	dirInfo, err := os.Stat(dir)
	if err != nil {
		err := os.Mkdir(dir, 0777)
		if err != nil {
			return "", err
		}
	} else {
		if !dirInfo.Mode().IsDir() {
			return "", fmt.Errorf("%s exists but is not a directory", dir)
		}
	}
	return dir, nil
}

// Start the bootstrapping process
func start(c *cli.Context) error {
	log.Debug("start")

	lockFile, err := singleinstance.CreateLockFile(getProcessLockFilePath())
	if err != nil {
		return err
	}
	defer lockFile.Close()

	formatter := format.GetFormatter()

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	go handleSysSignals(cancel)

	timingInterval := c.Int64("interval")
	if !(timingInterval > 0) {
		timingInterval = DefaultTimingInterval
	}

	timingSplay := c.Int64("splay")
	if !(timingSplay > 0) {
		timingSplay = DefaultTimingSplay
	}

	thresholdLines := c.Int("lines")
	if !(thresholdLines > 0) {
		thresholdLines = DefaultThresholdLines
	}
	log.Debug("routine lines threshold: ", thresholdLines)

	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	bootstrappingSvc, formatter := cmd.WireUpBootstrapping(c)
	for {
		applyPolicyfiles(bootstrappingSvc, formatter, thresholdLines)

		// Sleep for a configured amount of time plus a random amount of time (10 minutes plus 0 to 5 minutes, for instance)
		ticker := time.NewTicker(time.Duration(timingInterval+int64(r.Intn(int(timingSplay)))) * time.Second)

		select {
		case <-ticker.C:
			log.Debug("ticker")
		case <-ctx.Done():
			log.Debug(ctx.Err())
			log.Debug("closing bootstrapping")
		}
		ticker.Stop()
		if ctx.Err() != nil {
			break
		}
	}

	return nil
}

// Stop the bootstrapping process
func stop(c *cli.Context) error {
	log.Debug("cmdStop")

	formatter := format.GetFormatter()
	if err := utils.StopProcess(getProcessLockFilePath()); err != nil {
		formatter.PrintFatal("cannot stop the bootstrapping process", err)
	}

	log.Info("Bootstrapping routine successfully stopped")
	return nil
}

// Subsidiary routine for commands processing
func applyPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, formatter format.Formatter, thresholdLines int) error {
	log.Debug("applyPolicyfiles")

	// Inquire about desired configuration changes to be applied by querying the `GET /blueprint/configuration` endpoint. This will provide a JSON response with the desired configuration changes
	bsConfiguration, status, err := bootstrappingSvc.GetBootstrappingConfiguration()
	if err == nil && status != 200 {
		err = fmt.Errorf("received non-ok %d response", status)
	}
	if err != nil {
		formatter.PrintError("couldn't receive bootstrapping data", err)
		return err
	}
	dir, err := generateWorkspaceDir()
	if err != nil {
		return err
	}
	bsProcess := &bootstrappingProcess{
		startedAt:                    time.Now().UTC(),
		thresholdLines:               thresholdLines,
		directoryPath:                dir,
		appliedPolicyfileRevisionIDs: make(map[string]string),
	}

	// proto structures
	err = initializePrototype(bsConfiguration, bsProcess)
	if err != nil {
		log.Debug(err)
		return err
	}
	// For every policyfile, ensure its tarball (downloadable through their download_url) has been downloaded to the server ...
	err = downloadPolicyfiles(bootstrappingSvc, bsProcess)
	if err != nil {
		log.Debug(err)
		return err
	}
	//... and clean off any tarball that is no longer needed.
	err = cleanObsoletePolicyfiles(bsProcess)
	if err != nil {
		log.Debug(err)
		return err
	}
	// Store the attributes as JSON in a file with name `attrs-<attribute_revision_id>.json`
	err = saveAttributes(bsProcess)
	if err != nil {
		log.Debug(err)
		return err
	}
	// Process tarballs policies
	err = processPolicyfiles(bootstrappingSvc, bsProcess)
	// Finishing time
	bsProcess.finishedAt = time.Now().UTC()

	// Inform the platform of applied changes via a `PUT /blueprint/applied_configuration` request with a JSON payload similar to
	log.Debug("reporting applied policy files")
	reportErr := reportAppliedConfiguration(bootstrappingSvc, bsProcess)
	if reportErr != nil {
		log.Debug(err)
		return err
	}
	if err != nil {
		return err
	}
	return completeBootstrappingSequence(bsProcess)
}

func initializePrototype(bsConfiguration *types.BootstrappingConfiguration, bsProcess *bootstrappingProcess) error {
	log.Debug("initializePrototype")

	// Attributes
	bsProcess.attributes.revisionID = bsConfiguration.AttributeRevisionID
	bsProcess.attributes.rawData = bsConfiguration.Attributes

	// Policies
	for _, bsConfPolicyfile := range bsConfiguration.Policyfiles {
		bsProcess.policyfiles = append(bsProcess.policyfiles, policyfile(bsConfPolicyfile))
	}
	log.Debug(bsProcess)
	return nil
}

// downloadPolicyfiles For every policy file, ensure its tarball (downloadable through their download_url) has been downloaded to the server ...
func downloadPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error {
	log.Debug("downloadPolicyfiles")

	for _, bsPolicyfile := range bsProcess.policyfiles {
		tarballPath := bsPolicyfile.TarballPath(bsProcess.directoryPath)
		log.Debug("downloading: ", tarballPath)
		queryURL, err := bsPolicyfile.QueryURL()
		if err != nil {
			return err
		}
		_, status, err := bootstrappingSvc.DownloadPolicyfile(queryURL, tarballPath)
		if err == nil && status != 200 {
			err = fmt.Errorf("obtained non-ok response when downloading policyfile %s", queryURL)
		}
		if err != nil {
			return err
		}
		if err = utils.Untar(tarballPath, bsPolicyfile.Path(bsProcess.directoryPath)); err != nil {
			return err
		}
	}
	return nil
}

// cleanObsoletePolicyfiles cleans off any tarball that is no longer needed.
func cleanObsoletePolicyfiles(bsProcess *bootstrappingProcess) error {
	log.Debug("cleanObsoletePolicyfiles")

	// evaluates working folder
	deletableFiles, err := ioutil.ReadDir(bsProcess.directoryPath)
	if err != nil {
		return err
	}

	// builds an array of currently processable files at this looping time
	currentlyProcessableFiles := []string{bsProcess.attributes.FileName()} // saved attributes file name
	for _, bsPolicyFile := range bsProcess.policyfiles {
		currentlyProcessableFiles = append(currentlyProcessableFiles, bsPolicyFile.FileName()) // Downloaded tgz file names
		currentlyProcessableFiles = append(currentlyProcessableFiles, bsPolicyFile.Name())     // Uncompressed folder names
	}

	// removes from deletableFiles array the policy files currently applied
	for _, f := range deletableFiles {
		if !utils.Contains(currentlyProcessableFiles, f.Name()) {
			log.Debug("removing: ", f.Name())
			if err := os.RemoveAll(filepath.Join(bsProcess.directoryPath, f.Name())); err != nil {
				return err
			}
		}
	}
	return nil
}

// saveAttributes stores the attributes as JSON in a file with name `attrs-<attribute_revision_id>.json`
func saveAttributes(bsProcess *bootstrappingProcess) error {
	log.Debug("saveAttributes")

	attrs, err := json.Marshal(bsProcess.attributes.rawData)
	if err != nil {
		return err
	}
	if err := ioutil.WriteFile(bsProcess.attributes.FilePath(bsProcess.directoryPath), attrs, 0600); err != nil {
		return err
	}
	return nil
}

// processPolicyfiles applies for each policy the required chef commands, reporting in bunches of N lines
func processPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error {
	log.Debug("processPolicyfiles")

	for _, bsPolicyfile := range bsProcess.policyfiles {
		command := fmt.Sprintf("cd %s", bsPolicyfile.Path(bsProcess.directoryPath))
		if runtime.GOOS == "windows" {
			command = fmt.Sprintf("%s\nSET \"PATH=%PATH%;C:\\ruby\\bin;C:\\opscode\\chef\\bin;C:\\opscode\\chef\\embedded\\bin\"", command)
		}
		command = fmt.Sprintf("%s\nchef-client -z -j %s", command, bsProcess.attributes.FilePath(bsProcess.directoryPath))
		log.Debug(command)

		// Custom method for chunks processing
		fn := func(chunk string) error {
			log.Debug("sendChunks")
			err := utils.Retry(RetriesNumber, time.Second, func() error {
				log.Debug("Sending: ", chunk)

				commandIn := map[string]interface{}{
					"stdout": chunk,
				}

				_, statusCode, err := bootstrappingSvc.ReportBootstrappingLog(&commandIn)
				switch {
				// 0<100 error cases??
				case statusCode == 0:
					return fmt.Errorf("communication error %v %v", statusCode, err)
				case statusCode >= 500:
					return fmt.Errorf("server error %v %v", statusCode, err)
				case statusCode >= 400:
					return fmt.Errorf("client error %v %v", statusCode, err)
				default:
					return nil
				}
			})

			if err != nil {
				return fmt.Errorf("cannot send the chunk data, %v", err)
			}
			return nil
		}

		exitCode, err := utils.RunContinuousCmd(fn, command, -1, bsProcess.thresholdLines)
		if err == nil && exitCode != 0 {
			err = fmt.Errorf("policyfile application exited with %d code", exitCode)
		}
		if err != nil {
			return err
		}

		log.Info("completed: ", exitCode)
		bsProcess.appliedPolicyfileRevisionIDs[bsPolicyfile.ID] = bsPolicyfile.RevisionID
	}
	return nil
}

// reportAppliedConfiguration Inform the platform of applied changes
func reportAppliedConfiguration(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error {
	log.Debug("reportAppliedConfiguration")

	payload := map[string]interface{}{
		"started_at":              bsProcess.startedAt,
		"finished_at":             bsProcess.finishedAt,
		"policyfile_revision_ids": bsProcess.appliedPolicyfileRevisionIDs,
		"attribute_revision_id":   bsProcess.attributes.revisionID,
	}
	return bootstrappingSvc.ReportBootstrappingAppliedConfiguration(&payload)
}

// completeBootstrappingSequence evaluates if the first iteration of policies was completed; If case, execute the "scripts boot" command.
func completeBootstrappingSequence(bsProcess *bootstrappingProcess) error {
	log.Debug("completeBootstrappingSequence")

	if !allPolicyfilesSuccessfullyApplied {
		log.Debug("run the boot scripts")
		//run the boot scripts for the server by executing the scripts boot sub-command (as an external process).
		if output, exit, _, _ := utils.RunCmd(strings.Join([]string{os.Args[0], "scripts", "boot"}, " ")); exit != 0 {
			return fmt.Errorf("boot scripts run failed with exit code %d and following output: %s", exit, output)
		}
		allPolicyfilesSuccessfullyApplied = true
	}
	return nil
}<commit_msg>Make more readable the processing error messages (issue #90)<commit_after>package bootstrapping

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/url"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
	"time"

	log "github.com/Sirupsen/logrus"
	singleinstance "github.com/allan-simon/go-singleinstance"
	"github.com/codegangsta/cli"
	"github.com/ingrammicro/concerto/api/blueprint"
	"github.com/ingrammicro/concerto/api/types"
	"github.com/ingrammicro/concerto/cmd"
	"github.com/ingrammicro/concerto/utils"
	"github.com/ingrammicro/concerto/utils/format"
)

const (
	//DefaultTimingInterval Default period for looping
	DefaultTimingInterval = 600 // 600 seconds = 10 minutes
	DefaultTimingSplay    = 360 // seconds
	DefaultThresholdLines = 10
	ProcessLockFile       = "imco-bootstrapping.lock"
	RetriesNumber         = 5
)

type bootstrappingProcess struct {
	startedAt                    time.Time
	finishedAt                   time.Time
	policyfiles                  []policyfile
	attributes                   attributes
	thresholdLines               int
	directoryPath                string
	appliedPolicyfileRevisionIDs map[string]string
}
type attributes struct {
	revisionID string
	rawData    *json.RawMessage
}

var allPolicyfilesSuccessfullyApplied bool

type policyfile types.BootstrappingPolicyfile

func (pf policyfile) Name() string {
	return strings.Join([]string{pf.ID, "-", pf.RevisionID}, "")
}

func (pf *policyfile) FileName() string {
	return strings.Join([]string{pf.Name(), "tgz"}, ".")
}

func (pf *policyfile) QueryURL() (string, error) {
	if pf.DownloadURL == "" {
		return "", fmt.Errorf("obtaining URL query: empty download URL")
	}
	url, err := url.Parse(pf.DownloadURL)
	if err != nil {
		return "", fmt.Errorf("parsing URL to extract query: %v", err)
	}
	return fmt.Sprintf("%s?%s", url.Path, url.RawQuery), nil
}

func (pf *policyfile) TarballPath(dir string) string {
	return filepath.Join(dir, pf.FileName())
}

func (pf *policyfile) Path(dir string) string {
	return filepath.Join(dir, pf.Name())
}

func (a *attributes) FileName() string {
	return fmt.Sprintf("attrs-%s.json", a.revisionID)
}

func (a *attributes) FilePath(dir string) string {
	return filepath.Join(dir, a.FileName())
}

// Handle signals
func handleSysSignals(cancelFunc context.CancelFunc) {
	log.Debug("handleSysSignals")

	gracefulStop := make(chan os.Signal, 1)
	signal.Notify(gracefulStop, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL)
	log.Debug("Ending, signal detected:", <-gracefulStop)
	cancelFunc()
}

// Returns the full path to the tmp directory joined with lock management file name
func getProcessLockFilePath() string {
	return filepath.Join(os.TempDir(), string(os.PathSeparator), ProcessLockFile)
}

// Returns the full path to the tmp directory
func generateWorkspaceDir() (string, error) {
	dir := filepath.Join(os.TempDir(), "imco")
	dirInfo, err := os.Stat(dir)
	if err != nil {
		err := os.Mkdir(dir, 0777)
		if err != nil {
			return "", err
		}
	} else {
		if !dirInfo.Mode().IsDir() {
			return "", fmt.Errorf("%s exists but is not a directory", dir)
		}
	}
	return dir, nil
}

// Start the bootstrapping process
func start(c *cli.Context) error {
	log.Debug("start")

	lockFile, err := singleinstance.CreateLockFile(getProcessLockFilePath())
	if err != nil {
		return err
	}
	defer lockFile.Close()

	formatter := format.GetFormatter()

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	go handleSysSignals(cancel)

	timingInterval := c.Int64("interval")
	if !(timingInterval > 0) {
		timingInterval = DefaultTimingInterval
	}

	timingSplay := c.Int64("splay")
	if !(timingSplay > 0) {
		timingSplay = DefaultTimingSplay
	}

	thresholdLines := c.Int("lines")
	if !(thresholdLines > 0) {
		thresholdLines = DefaultThresholdLines
	}
	log.Debug("routine lines threshold: ", thresholdLines)

	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	bootstrappingSvc, formatter := cmd.WireUpBootstrapping(c)
	for {
		applyPolicyfiles(bootstrappingSvc, formatter, thresholdLines)

		// Sleep for a configured amount of time plus a random amount of time (10 minutes plus 0 to 5 minutes, for instance)
		ticker := time.NewTicker(time.Duration(timingInterval+int64(r.Intn(int(timingSplay)))) * time.Second)

		select {
		case <-ticker.C:
			log.Debug("ticker")
		case <-ctx.Done():
			log.Debug(ctx.Err())
			log.Debug("closing bootstrapping")
		}
		ticker.Stop()
		if ctx.Err() != nil {
			break
		}
	}

	return nil
}

// Stop the bootstrapping process
func stop(c *cli.Context) error {
	log.Debug("cmdStop")

	formatter := format.GetFormatter()
	if err := utils.StopProcess(getProcessLockFilePath()); err != nil {
		formatter.PrintFatal("cannot stop the bootstrapping process", err)
	}

	log.Info("Bootstrapping routine successfully stopped")
	return nil
}

// Subsidiary routine for commands processing
func applyPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, formatter format.Formatter, thresholdLines int) error {
	log.Debug("applyPolicyfiles")

	// Inquire about desired configuration changes to be applied by querying the `GET /blueprint/configuration` endpoint. This will provide a JSON response with the desired configuration changes
	bsConfiguration, status, err := bootstrappingSvc.GetBootstrappingConfiguration()
	if err == nil && status != 200 {
		err = fmt.Errorf("received non-ok %d response", status)
	}
	if err != nil {
		formatter.PrintError("couldn't receive bootstrapping data", err)
		return err
	}
	dir, err := generateWorkspaceDir()
	if err != nil {
		formatter.PrintError("couldn't generated workspace directory", err)
		return err
	}
	bsProcess := &bootstrappingProcess{
		startedAt:                    time.Now().UTC(),
		thresholdLines:               thresholdLines,
		directoryPath:                dir,
		appliedPolicyfileRevisionIDs: make(map[string]string),
	}

	// proto structures
	err = initializePrototype(bsConfiguration, bsProcess)
	if err != nil {
		formatter.PrintError("couldn't initialize prototype", err)
		return err
	}
	// For every policyfile, ensure its tarball (downloadable through their download_url) has been downloaded to the server ...
	err = downloadPolicyfiles(bootstrappingSvc, bsProcess)
	if err != nil {
		formatter.PrintError("couldn't download policy files", err)
		return err
	}
	//... and clean off any tarball that is no longer needed.
	err = cleanObsoletePolicyfiles(bsProcess)
	if err != nil {
		formatter.PrintError("couldn't clean obsolete policy files", err)
		return err
	}
	// Store the attributes as JSON in a file with name `attrs-<attribute_revision_id>.json`
	err = saveAttributes(bsProcess)
	if err != nil {
		formatter.PrintError("couldn't save attributes for policy files", err)
		return err
	}
	// Process tarballs policies
	err = processPolicyfiles(bootstrappingSvc, bsProcess)
	// Finishing time
	bsProcess.finishedAt = time.Now().UTC()

	// Inform the platform of applied changes via a `PUT /blueprint/applied_configuration` request with a JSON payload similar to
	log.Debug("reporting applied policy files")
	reportErr := reportAppliedConfiguration(bootstrappingSvc, bsProcess)
	if reportErr != nil {
		formatter.PrintError("couldn't report applied status for policy files", err)
		return err
	}
	if err != nil {
		formatter.PrintError("couldn't process policy files", err)
		return err
	}
	return completeBootstrappingSequence(bsProcess)
}

func initializePrototype(bsConfiguration *types.BootstrappingConfiguration, bsProcess *bootstrappingProcess) error {
	log.Debug("initializePrototype")

	// Attributes
	bsProcess.attributes.revisionID = bsConfiguration.AttributeRevisionID
	bsProcess.attributes.rawData = bsConfiguration.Attributes

	// Policies
	for _, bsConfPolicyfile := range bsConfiguration.Policyfiles {
		bsProcess.policyfiles = append(bsProcess.policyfiles, policyfile(bsConfPolicyfile))
	}
	log.Debug(bsProcess)
	return nil
}

// downloadPolicyfiles For every policy file, ensure its tarball (downloadable through their download_url) has been downloaded to the server ...
func downloadPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error {
	log.Debug("downloadPolicyfiles")

	for _, bsPolicyfile := range bsProcess.policyfiles {
		tarballPath := bsPolicyfile.TarballPath(bsProcess.directoryPath)
		log.Debug("downloading: ", tarballPath)
		queryURL, err := bsPolicyfile.QueryURL()
		if err != nil {
			return err
		}
		_, status, err := bootstrappingSvc.DownloadPolicyfile(queryURL, tarballPath)
		if err == nil && status != 200 {
			err = fmt.Errorf("obtained non-ok response when downloading policyfile %s", queryURL)
		}
		if err != nil {
			return err
		}
		if err = utils.Untar(tarballPath, bsPolicyfile.Path(bsProcess.directoryPath)); err != nil {
			return err
		}
	}
	return nil
}

// cleanObsoletePolicyfiles cleans off any tarball that is no longer needed.
func cleanObsoletePolicyfiles(bsProcess *bootstrappingProcess) error {
	log.Debug("cleanObsoletePolicyfiles")

	// evaluates working folder
	deletableFiles, err := ioutil.ReadDir(bsProcess.directoryPath)
	if err != nil {
		return err
	}

	// builds an array of currently processable files at this looping time
	currentlyProcessableFiles := []string{bsProcess.attributes.FileName()} // saved attributes file name
	for _, bsPolicyFile := range bsProcess.policyfiles {
		currentlyProcessableFiles = append(currentlyProcessableFiles, bsPolicyFile.FileName()) // Downloaded tgz file names
		currentlyProcessableFiles = append(currentlyProcessableFiles, bsPolicyFile.Name())     // Uncompressed folder names
	}

	// removes from deletableFiles array the policy files currently applied
	for _, f := range deletableFiles {
		if !utils.Contains(currentlyProcessableFiles, f.Name()) {
			log.Debug("removing: ", f.Name())
			if err := os.RemoveAll(filepath.Join(bsProcess.directoryPath, f.Name())); err != nil {
				return err
			}
		}
	}
	return nil
}

// saveAttributes stores the attributes as JSON in a file with name `attrs-<attribute_revision_id>.json`
func saveAttributes(bsProcess *bootstrappingProcess) error {
	log.Debug("saveAttributes")

	attrs, err := json.Marshal(bsProcess.attributes.rawData)
	if err != nil {
		return err
	}
	if err := ioutil.WriteFile(bsProcess.attributes.FilePath(bsProcess.directoryPath), attrs, 0600); err != nil {
		return err
	}
	return nil
}

// processPolicyfiles applies for each policy the required chef commands, reporting in bunches of N lines
func processPolicyfiles(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error {
	log.Debug("processPolicyfiles")

	for _, bsPolicyfile := range bsProcess.policyfiles {
		command := fmt.Sprintf("cd %s", bsPolicyfile.Path(bsProcess.directoryPath))
		if runtime.GOOS == "windows" {
			command = fmt.Sprintf("%s\nSET \"PATH=%PATH%;C:\\ruby\\bin;C:\\opscode\\chef\\bin;C:\\opscode\\chef\\embedded\\bin\"", command)
		}
		command = fmt.Sprintf("%s\nchef-client -z -j %s", command, bsProcess.attributes.FilePath(bsProcess.directoryPath))
		log.Debug(command)

		// Custom method for chunks processing
		fn := func(chunk string) error {
			log.Debug("sendChunks")
			err := utils.Retry(RetriesNumber, time.Second, func() error {
				log.Debug("Sending: ", chunk)

				commandIn := map[string]interface{}{
					"stdout": chunk,
				}

				_, statusCode, err := bootstrappingSvc.ReportBootstrappingLog(&commandIn)
				switch {
				// 0<100 error cases??
				case statusCode == 0:
					return fmt.Errorf("communication error %v %v", statusCode, err)
				case statusCode >= 500:
					return fmt.Errorf("server error %v %v", statusCode, err)
				case statusCode >= 400:
					return fmt.Errorf("client error %v %v", statusCode, err)
				default:
					return nil
				}
			})

			if err != nil {
				return fmt.Errorf("cannot send the chunk data, %v", err)
			}
			return nil
		}

		exitCode, err := utils.RunContinuousCmd(fn, command, -1, bsProcess.thresholdLines)
		if err == nil && exitCode != 0 {
			err = fmt.Errorf("policyfile application exited with %d code", exitCode)
		}
		if err != nil {
			return err
		}

		log.Info("completed: ", exitCode)
		bsProcess.appliedPolicyfileRevisionIDs[bsPolicyfile.ID] = bsPolicyfile.RevisionID
	}
	return nil
}

// reportAppliedConfiguration Inform the platform of applied changes
func reportAppliedConfiguration(bootstrappingSvc *blueprint.BootstrappingService, bsProcess *bootstrappingProcess) error {
	log.Debug("reportAppliedConfiguration")

	payload := map[string]interface{}{
		"started_at":              bsProcess.startedAt,
		"finished_at":             bsProcess.finishedAt,
		"policyfile_revision_ids": bsProcess.appliedPolicyfileRevisionIDs,
		"attribute_revision_id":   bsProcess.attributes.revisionID,
	}
	return bootstrappingSvc.ReportBootstrappingAppliedConfiguration(&payload)
}

// completeBootstrappingSequence evaluates if the first iteration of policies was completed; If case, execute the "scripts boot" command.
func completeBootstrappingSequence(bsProcess *bootstrappingProcess) error {
	log.Debug("completeBootstrappingSequence")

	if !allPolicyfilesSuccessfullyApplied {
		log.Debug("run the boot scripts")
		//run the boot scripts for the server by executing the scripts boot sub-command (as an external process).
		if output, exit, _, _ := utils.RunCmd(strings.Join([]string{os.Args[0], "scripts", "boot"}, " ")); exit != 0 {
			return fmt.Errorf("boot scripts run failed with exit code %d and following output: %s", exit, output)
		}
		allPolicyfilesSuccessfullyApplied = true
	}
	return nil
}<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs

import (
	"io"
	"log"
	"os"
	"path/filepath"
	"sync"
	"syscall"
	"time"

	"github.com/google/gitfs/cache"
	"github.com/google/gitfs/gitiles"
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/fuse/nodefs"
	git "github.com/libgit2/git2go"
)

// gitilesRoot is the root for a FUSE filesystem backed by a Gitiles
// service.
type gitilesRoot struct {
	nodefs.Node

	nodeCache *nodeCache

	cache   *cache.Cache
	service *gitiles.RepoService
	tree    *gitiles.Tree
	opts    GitilesOptions

	// TODO(hanwen): enable this again. After mount, set this to
	// server.KernelSettings().Flags&fuse.CAP_NO_OPEN_SUPPORT != 0.
	// This requires a suitably new kernel, though.
	handleLessIO bool

	// OID => path
	shaMap map[git.Oid]string

	lazyRepo *cache.LazyRepo
}

type linkNode struct {
	nodefs.Node
	linkTarget []byte
}

func (n *linkNode) Deletable() bool { return false }

func newLinkNode(target string) *linkNode {
	return &linkNode{
		Node:       nodefs.NewDefaultNode(),
		linkTarget: []byte(target),
	}
}

func (n *linkNode) GetAttr(out *fuse.Attr, file nodefs.File, context *fuse.Context) (code fuse.Status) {
	out.Size = uint64(len(n.linkTarget))
	out.Mode = fuse.S_IFLNK
	return fuse.OK
}

func (n *linkNode) Readlink(c *fuse.Context) ([]byte, fuse.Status) {
	return n.linkTarget, fuse.OK
}

// gitilesNode represents a read-only blob in the FUSE filesystem.
type gitilesNode struct {
	nodefs.Node

	root *gitilesRoot

	// Data from Git metadata.
	mode       uint32
	size       int64
	id         git.Oid
	linkTarget []byte

	// if set, clone the repo on reading this file.
	clone bool

	// The timestamp is writable; protect it with a mutex.
	mtimeMu sync.Mutex
	mtime   time.Time
}

func (n *gitilesNode) Utimens(file nodefs.File, atime *time.Time, mtime *time.Time, context *fuse.Context) (code fuse.Status) {
	if mtime != nil {
		n.mtimeMu.Lock()
		n.mtime = *mtime
		n.mtimeMu.Unlock()
	}
	return fuse.OK
}

func (n *gitilesNode) Readlink(c *fuse.Context) ([]byte, fuse.Status) {
	return n.linkTarget, fuse.OK
}

func (n *gitilesNode) GetAttr(out *fuse.Attr, file nodefs.File, context *fuse.Context) (code fuse.Status) {
	out.Size = uint64(n.size)
	out.Mode = n.mode

	n.mtimeMu.Lock()
	t := n.mtime
	n.mtimeMu.Unlock()

	out.SetTimes(nil, &t, nil)
	return fuse.OK
}

const xattrName = "user.gitsha1"

func (n *gitilesNode) GetXAttr(attribute string, context *fuse.Context) (data []byte, code fuse.Status) {
	if attribute != xattrName {
		return nil, fuse.ENODATA
	}
	return []byte(n.id.String()), fuse.OK
}

func (n *gitilesNode) ListXAttr(context *fuse.Context) (attrs []string, code fuse.Status) {
	return []string{xattrName}, fuse.OK
}

func (n *gitilesNode) Open(flags uint32, context *fuse.Context) (file nodefs.File, code fuse.Status) {
	if n.root.handleLessIO {
		// We say ENOSYS so FUSE on Linux uses handle-less I/O.
		return nil, fuse.ENOSYS
	}

	f, err := n.root.openFile(n.id, n.clone)
	if err != nil {
		return nil, fuse.ToStatus(err)
	}
	return nodefs.NewLoopbackFile(f), fuse.OK
}

func (n *gitilesNode) Read(file nodefs.File, dest []byte, off int64, context *fuse.Context) (fuse.ReadResult, fuse.Status) {
	if n.root.handleLessIO {
		return n.handleLessRead(file, dest, off, context)
	}

	return file.Read(dest, off)
}

func (n *gitilesNode) handleLessRead(file nodefs.File, dest []byte, off int64, context *fuse.Context) (fuse.ReadResult, fuse.Status) {
	// TODO(hanwen): for large files this is not efficient. Should
	// have a cache of open file handles.
	f, err := n.root.openFile(n.id, n.clone)
	if err != nil {
		return nil, fuse.ToStatus(err)
	}

	m, err := f.ReadAt(dest, off)
	if err == io.EOF {
		err = nil
	}
	f.Close()
	return fuse.ReadResultData(dest[:m]), fuse.ToStatus(err)
}

// openFile returns a file handle for the given blob. If `clone` is
// given, we may try a clone of the git repository
func (r *gitilesRoot) openFile(id git.Oid, clone bool) (*os.File, error) {
	f, ok := r.cache.Blob.Open(id)
	if !ok {
		repo := r.lazyRepo.Repository()
		if clone && repo == nil {
			r.lazyRepo.Clone()
		}
		if repo != nil {
			defer repo.Free()
		}

		var content []byte
		if repo != nil {
			blob, err := repo.LookupBlob(&id)
			if err != nil {
				log.Println("LookupBlob: %v", err)
				return nil, syscall.ESPIPE
			}
			defer blob.Free()
			content = blob.Contents()
		} else {
			path := r.shaMap[id]

			var err error
			content, err = r.service.GetBlob(r.opts.Revision, path)
			if err != nil {
				log.Printf("GetBlob(%s, %s): %v", r.opts.Revision, path, err)
				return nil, syscall.EDOM
			}
		}

		if err := r.cache.Blob.Write(id, content); err != nil {
			return nil, err
		}

		f, ok = r.cache.Blob.Open(id)
		if !ok {
			return nil, syscall.EROFS
		}
	}
	return f, nil
}

// dataNode makes arbitrary data available as a file.
type dataNode struct {
	nodefs.Node
	data []byte
}

func (n *dataNode) GetAttr(out *fuse.Attr, file nodefs.File, context *fuse.Context) (code fuse.Status) {
	out.Size = uint64(len(n.data))
	out.Mode = fuse.S_IFREG | 0644
	return fuse.OK
}

func (d *dataNode) Open(flags uint32, content *fuse.Context) (nodefs.File, fuse.Status) {
	return nodefs.NewDataFile(d.data), fuse.OK
}

func (d *dataNode) GetXAttr(attribute string, context *fuse.Context) (data []byte, code fuse.Status) {
	return nil, fuse.ENODATA
}

func (n *dataNode) Deletable() bool { return false }

func newDataNode(c []byte) nodefs.Node {
	return &dataNode{nodefs.NewDefaultNode(), c}
}

// NewGitilesRoot returns the root node for a file system.
func NewGitilesRoot(c *cache.Cache, tree *gitiles.Tree, service *gitiles.RepoService, options GitilesOptions) nodefs.Node {
	r := &gitilesRoot{
		Node:      nodefs.NewDefaultNode(),
		service:   service,
		nodeCache: newNodeCache(),
		cache:     c,
		shaMap:    map[git.Oid]string{},
		tree:      tree,
		opts:      options,
		lazyRepo:  cache.NewLazyRepo(options.CloneURL, c),
	}

	return r
}

func (r *gitilesRoot) Deletable() bool { return false }

func (n *gitilesRoot) GetXAttr(attribute string, context *fuse.Context) (data []byte, code fuse.Status) {
	return nil, fuse.ENODATA
}

func (r *gitilesRoot) OnMount(fsConn *nodefs.FileSystemConnector) {
	if err := r.onMount(fsConn); err != nil {
		log.Printf("onMount: %v", err)
		for k := range r.Inode().Children() {
			r.Inode().RmChild(k)
		}
		r.Inode().NewChild("ERROR", false, newDataNode([]byte(err.Error())))
	}
}

func (r *gitilesRoot) onMount(fsConn *nodefs.FileSystemConnector) error {
	for _, e := range r.tree.Entries {
		if e.Type == "commit" {
			// TODO(hanwen): support submodules.
			continue
		}
		if e.Type != "blob" {
			log.Panicf("unexpected object type %s", e.Type)
		}

		p := e.Name
		dir, base := filepath.Split(p)

		parent, left := fsConn.Node(r.Inode(), dir)
		for _, l := range left {
			ch := parent.NewChild(l, true, nodefs.NewDefaultNode())
			parent = ch
		}
		id, err := git.NewOid(e.ID)
		if err != nil {
			return nil
		}

		// Determine if file should trigger a clone.
		clone := r.opts.CloneURL != ""
		if clone {
			for _, e := range r.opts.CloneOption {
				if e.RE.FindString(p) != "" {
					clone = e.Clone
					break
				}
			}
		}

		xbit := e.Mode&0111 != 0
		n := r.nodeCache.get(id, xbit)
		if n == nil {
			n = &gitilesNode{
				Node:  nodefs.NewDefaultNode(),
				id:    *id,
				mode:  uint32(e.Mode),
				clone: clone,
				root:  r,
				// Ninja uses mtime == 0 as "doesn't exist"
				// flag, (see ninja/files/src/graph.h:66), so
				// use a nonzero timestamp here.
				mtime: time.Unix(1, 0),
			}
			if e.Size != nil {
				n.size = int64(*e.Size)
			}

			if e.Target != nil {
				n.linkTarget = []byte(*e.Target)
				n.size = int64(len(n.linkTarget))
			}

			r.shaMap[*id] = p
			parent.NewChild(base, false, n)
			r.nodeCache.add(n)
		} else {
			parent.AddChild(base, n.Inode())
		}

	}

	r.Inode().NewChild(".gitid",
		false, newDataNode([]byte(r.tree.ID)))

	// We don't need the tree data anymore.
	r.tree = nil
	return nil
}
<commit_msg>Fix error message.<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs

import (
	"io"
	"log"
	"os"
	"path/filepath"
	"sync"
	"syscall"
	"time"

	"github.com/google/gitfs/cache"
	"github.com/google/gitfs/gitiles"
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/fuse/nodefs"
	git "github.com/libgit2/git2go"
)

// gitilesRoot is the root for a FUSE filesystem backed by a Gitiles
// service.
type gitilesRoot struct {
	nodefs.Node

	nodeCache *nodeCache

	cache   *cache.Cache
	service *gitiles.RepoService
	tree    *gitiles.Tree
	opts    GitilesOptions

	// TODO(hanwen): enable this again. After mount, set this to
	// server.KernelSettings().Flags&fuse.CAP_NO_OPEN_SUPPORT != 0.
	// This requires a suitably new kernel, though.
	handleLessIO bool

	// OID => path
	shaMap map[git.Oid]string

	lazyRepo *cache.LazyRepo
}

type linkNode struct {
	nodefs.Node
	linkTarget []byte
}

func (n *linkNode) Deletable() bool { return false }

func newLinkNode(target string) *linkNode {
	return &linkNode{
		Node:       nodefs.NewDefaultNode(),
		linkTarget: []byte(target),
	}
}

func (n *linkNode) GetAttr(out *fuse.Attr, file nodefs.File, context *fuse.Context) (code fuse.Status) {
	out.Size = uint64(len(n.linkTarget))
	out.Mode = fuse.S_IFLNK
	return fuse.OK
}

func (n *linkNode) Readlink(c *fuse.Context) ([]byte, fuse.Status) {
	return n.linkTarget, fuse.OK
}

// gitilesNode represents a read-only blob in the FUSE filesystem.
type gitilesNode struct {
	nodefs.Node

	root *gitilesRoot

	// Data from Git metadata.
	mode       uint32
	size       int64
	id         git.Oid
	linkTarget []byte

	// if set, clone the repo on reading this file.
	clone bool

	// The timestamp is writable; protect it with a mutex.
	mtimeMu sync.Mutex
	mtime   time.Time
}

func (n *gitilesNode) Utimens(file nodefs.File, atime *time.Time, mtime *time.Time, context *fuse.Context) (code fuse.Status) {
	if mtime != nil {
		n.mtimeMu.Lock()
		n.mtime = *mtime
		n.mtimeMu.Unlock()
	}
	return fuse.OK
}

func (n *gitilesNode) Readlink(c *fuse.Context) ([]byte, fuse.Status) {
	return n.linkTarget, fuse.OK
}

func (n *gitilesNode) GetAttr(out *fuse.Attr, file nodefs.File, context *fuse.Context) (code fuse.Status) {
	out.Size = uint64(n.size)
	out.Mode = n.mode

	n.mtimeMu.Lock()
	t := n.mtime
	n.mtimeMu.Unlock()

	out.SetTimes(nil, &t, nil)
	return fuse.OK
}

const xattrName = "user.gitsha1"

func (n *gitilesNode) GetXAttr(attribute string, context *fuse.Context) (data []byte, code fuse.Status) {
	if attribute != xattrName {
		return nil, fuse.ENODATA
	}
	return []byte(n.id.String()), fuse.OK
}

func (n *gitilesNode) ListXAttr(context *fuse.Context) (attrs []string, code fuse.Status) {
	return []string{xattrName}, fuse.OK
}

func (n *gitilesNode) Open(flags uint32, context *fuse.Context) (file nodefs.File, code fuse.Status) {
	if n.root.handleLessIO {
		// We say ENOSYS so FUSE on Linux uses handle-less I/O.
		return nil, fuse.ENOSYS
	}

	f, err := n.root.openFile(n.id, n.clone)
	if err != nil {
		return nil, fuse.ToStatus(err)
	}
	return nodefs.NewLoopbackFile(f), fuse.OK
}

func (n *gitilesNode) Read(file nodefs.File, dest []byte, off int64, context *fuse.Context) (fuse.ReadResult, fuse.Status) {
	if n.root.handleLessIO {
		return n.handleLessRead(file, dest, off, context)
	}

	return file.Read(dest, off)
}

func (n *gitilesNode) handleLessRead(file nodefs.File, dest []byte, off int64, context *fuse.Context) (fuse.ReadResult, fuse.Status) {
	// TODO(hanwen): for large files this is not efficient. Should
	// have a cache of open file handles.
	f, err := n.root.openFile(n.id, n.clone)
	if err != nil {
		return nil, fuse.ToStatus(err)
	}

	m, err := f.ReadAt(dest, off)
	if err == io.EOF {
		err = nil
	}
	f.Close()
	return fuse.ReadResultData(dest[:m]), fuse.ToStatus(err)
}

// openFile returns a file handle for the given blob. If `clone` is
// given, we may try a clone of the git repository
func (r *gitilesRoot) openFile(id git.Oid, clone bool) (*os.File, error) {
	f, ok := r.cache.Blob.Open(id)
	if !ok {
		repo := r.lazyRepo.Repository()
		if clone && repo == nil {
			r.lazyRepo.Clone()
		}
		if repo != nil {
			defer repo.Free()
		}

		var content []byte
		if repo != nil {
			blob, err := repo.LookupBlob(&id)
			if err != nil {
				log.Printf("LookupBlob: %v", err)
				return nil, syscall.ESPIPE
			}
			defer blob.Free()
			content = blob.Contents()
		} else {
			path := r.shaMap[id]

			var err error
			content, err = r.service.GetBlob(r.opts.Revision, path)
			if err != nil {
				log.Printf("GetBlob(%s, %s): %v", r.opts.Revision, path, err)
				return nil, syscall.EDOM
			}
		}

		if err := r.cache.Blob.Write(id, content); err != nil {
			return nil, err
		}

		f, ok = r.cache.Blob.Open(id)
		if !ok {
			return nil, syscall.EROFS
		}
	}
	return f, nil
}

// dataNode makes arbitrary data available as a file.
type dataNode struct {
	nodefs.Node
	data []byte
}

func (n *dataNode) GetAttr(out *fuse.Attr, file nodefs.File, context *fuse.Context) (code fuse.Status) {
	out.Size = uint64(len(n.data))
	out.Mode = fuse.S_IFREG | 0644
	return fuse.OK
}

func (d *dataNode) Open(flags uint32, content *fuse.Context) (nodefs.File, fuse.Status) {
	return nodefs.NewDataFile(d.data), fuse.OK
}

func (d *dataNode) GetXAttr(attribute string, context *fuse.Context) (data []byte, code fuse.Status) {
	return nil, fuse.ENODATA
}

func (n *dataNode) Deletable() bool { return false }

func newDataNode(c []byte) nodefs.Node {
	return &dataNode{nodefs.NewDefaultNode(), c}
}

// NewGitilesRoot returns the root node for a file system.
func NewGitilesRoot(c *cache.Cache, tree *gitiles.Tree, service *gitiles.RepoService, options GitilesOptions) nodefs.Node {
	r := &gitilesRoot{
		Node:      nodefs.NewDefaultNode(),
		service:   service,
		nodeCache: newNodeCache(),
		cache:     c,
		shaMap:    map[git.Oid]string{},
		tree:      tree,
		opts:      options,
		lazyRepo:  cache.NewLazyRepo(options.CloneURL, c),
	}

	return r
}

func (r *gitilesRoot) Deletable() bool { return false }

func (n *gitilesRoot) GetXAttr(attribute string, context *fuse.Context) (data []byte, code fuse.Status) {
	return nil, fuse.ENODATA
}

func (r *gitilesRoot) OnMount(fsConn *nodefs.FileSystemConnector) {
	if err := r.onMount(fsConn); err != nil {
		log.Printf("onMount: %v", err)
		for k := range r.Inode().Children() {
			r.Inode().RmChild(k)
		}
		r.Inode().NewChild("ERROR", false, newDataNode([]byte(err.Error())))
	}
}

func (r *gitilesRoot) onMount(fsConn *nodefs.FileSystemConnector) error {
	for _, e := range r.tree.Entries {
		if e.Type == "commit" {
			// TODO(hanwen): support submodules.
			continue
		}
		if e.Type != "blob" {
			log.Panicf("unexpected object type %s", e.Type)
		}

		p := e.Name
		dir, base := filepath.Split(p)

		parent, left := fsConn.Node(r.Inode(), dir)
		for _, l := range left {
			ch := parent.NewChild(l, true, nodefs.NewDefaultNode())
			parent = ch
		}
		id, err := git.NewOid(e.ID)
		if err != nil {
			return nil
		}

		// Determine if file should trigger a clone.
		clone := r.opts.CloneURL != ""
		if clone {
			for _, e := range r.opts.CloneOption {
				if e.RE.FindString(p) != "" {
					clone = e.Clone
					break
				}
			}
		}

		xbit := e.Mode&0111 != 0
		n := r.nodeCache.get(id, xbit)
		if n == nil {
			n = &gitilesNode{
				Node:  nodefs.NewDefaultNode(),
				id:    *id,
				mode:  uint32(e.Mode),
				clone: clone,
				root:  r,
				// Ninja uses mtime == 0 as "doesn't exist"
				// flag, (see ninja/files/src/graph.h:66), so
				// use a nonzero timestamp here.
				mtime: time.Unix(1, 0),
			}
			if e.Size != nil {
				n.size = int64(*e.Size)
			}

			if e.Target != nil {
				n.linkTarget = []byte(*e.Target)
				n.size = int64(len(n.linkTarget))
			}

			r.shaMap[*id] = p
			parent.NewChild(base, false, n)
			r.nodeCache.add(n)
		} else {
			parent.AddChild(base, n.Inode())
		}

	}

	r.Inode().NewChild(".gitid",
		false, newDataNode([]byte(r.tree.ID)))

	// We don't need the tree data anymore.
	r.tree = nil
	return nil
}
<|endoftext|>


<commit_before>//
// Ceph Nautilus is the first release that includes rbd_list2() and
// rbd_get_create_timestamp().

package rbd

// #cgo LDFLAGS: -lrbd
// #include <rados/librados.h>
// #include <rbd/librbd.h>
// #include <errno.h>
import "C"

import (
	"unsafe"

	"github.com/ceph/go-ceph/internal/retry"
	ts "github.com/ceph/go-ceph/internal/timespec"
	"github.com/ceph/go-ceph/rados"
)

// GetImageNames returns the list of current RBD images.
func GetImageNames(ioctx *rados.IOContext) ([]string, error) {
	var (
		err    error
		images []C.rbd_image_spec_t
		size   C.size_t
	)
	retry.WithSizes(32, 4096, func(s int) retry.Hint {
		size = C.size_t(s)
		images = make([]C.rbd_image_spec_t, size)
		ret := C.rbd_list2(
			cephIoctx(ioctx),
			(*C.rbd_image_spec_t)(unsafe.Pointer(&images[0])),
			&size)
		err = getErrorIfNegative(ret)
		return retry.Size(int(size)).If(err == errRange)
	})
	if err != nil {
		return nil, err
	}
	defer C.rbd_image_spec_list_cleanup((*C.rbd_image_spec_t)(unsafe.Pointer(&images[0])), size)

	names := make([]string, size)
	for i, image := range images[:size] {
		names[i] = C.GoString(image.name)
	}
	return names, nil
}

// GetCreateTimestamp returns the time the rbd image was created.
//
// Implements:
//  int rbd_get_create_timestamp(rbd_image_t image, struct timespec *timestamp);
func (image *Image) GetCreateTimestamp() (Timespec, error) {
	if err := image.validate(imageIsOpen); err != nil {
		return Timespec{}, err
	}

	var cts C.struct_timespec

	if ret := C.rbd_get_create_timestamp(image.image, &cts); ret < 0 {
		return Timespec{}, getError(ret)
	}

	return Timespec(ts.CStructToTimespec(ts.CTimespecPtr(&cts))), nil
}

// GetAccessTimestamp returns the time the rbd image was last accessed.
//
// Implements:
//  int rbd_get_access_timestamp(rbd_image_t image, struct timespec *timestamp);
func (image *Image) GetAccessTimestamp() (Timespec, error) {
	if err := image.validate(imageIsOpen); err != nil {
		return Timespec{}, err
	}

	var cts C.struct_timespec

	if ret := C.rbd_get_access_timestamp(image.image, &cts); ret < 0 {
		return Timespec{}, getError(ret)
	}

	return Timespec(ts.CStructToTimespec(ts.CTimespecPtr(&cts))), nil
}

// GetModifyTimestamp returns the time the rbd image was last modified.
//
// Implements:
//  int rbd_get_modify_timestamp(rbd_image_t image, struct timespec *timestamp);
func (image *Image) GetModifyTimestamp() (Timespec, error) {
	if err := image.validate(imageIsOpen); err != nil {
		return Timespec{}, err
	}

	var cts C.struct_timespec

	if ret := C.rbd_get_modify_timestamp(image.image, &cts); ret < 0 {
		return Timespec{}, getError(ret)
	}

	return Timespec(ts.CStructToTimespec(ts.CTimespecPtr(&cts))), nil
}

// Sparsify makes an image sparse by deallocating runs of zeros.
// The sparseSize value will be used to find runs of zeros and must be
// a power of two no less than 4096 and no larger than the image size.
//
// Implements:
//  int rbd_sparsify(rbd_image_t image, size_t sparse_size);
func (image *Image) Sparsify(sparseSize uint) error {
	if err := image.validate(imageIsOpen); err != nil {
		return err
	}

	return getError(C.rbd_sparsify(image.image, C.size_t(sparseSize)))
}
<commit_msg>rbd: don't cap the buffer size used in GetImageNames<commit_after>//
// Ceph Nautilus is the first release that includes rbd_list2() and
// rbd_get_create_timestamp().

package rbd

// #cgo LDFLAGS: -lrbd
// #include <rados/librados.h>
// #include <rbd/librbd.h>
// #include <errno.h>
import "C"

import (
	"unsafe"

	ts "github.com/ceph/go-ceph/internal/timespec"
	"github.com/ceph/go-ceph/rados"
)

// GetImageNames returns the list of current RBD images.
func GetImageNames(ioctx *rados.IOContext) ([]string, error) {
	size := C.size_t(4096)
	images, err := getImageNames(ioctx, &size)
	if err != nil {
		return nil, err
	}
	defer C.rbd_image_spec_list_cleanup((*C.rbd_image_spec_t)(unsafe.Pointer(&images[0])), size)

	names := make([]string, size)
	for i, image := range images[:size] {
		names[i] = C.GoString(image.name)
	}
	return names, nil
}

func getImageNames(ioctx *rados.IOContext, size *C.size_t) ([]C.rbd_image_spec_t, error) {
	images := make([]C.rbd_image_spec_t, *size)
	ret := C.rbd_list2(
		cephIoctx(ioctx),
		(*C.rbd_image_spec_t)(unsafe.Pointer(&images[0])),
		size)
	err := getErrorIfNegative(ret)
	if err != nil {
		if err == errRange {
			return getImageNames(ioctx, size)
		}
		return nil, err
	}
	return images, nil
}

// GetCreateTimestamp returns the time the rbd image was created.
//
// Implements:
//  int rbd_get_create_timestamp(rbd_image_t image, struct timespec *timestamp);
func (image *Image) GetCreateTimestamp() (Timespec, error) {
	if err := image.validate(imageIsOpen); err != nil {
		return Timespec{}, err
	}

	var cts C.struct_timespec

	if ret := C.rbd_get_create_timestamp(image.image, &cts); ret < 0 {
		return Timespec{}, getError(ret)
	}

	return Timespec(ts.CStructToTimespec(ts.CTimespecPtr(&cts))), nil
}

// GetAccessTimestamp returns the time the rbd image was last accessed.
//
// Implements:
//  int rbd_get_access_timestamp(rbd_image_t image, struct timespec *timestamp);
func (image *Image) GetAccessTimestamp() (Timespec, error) {
	if err := image.validate(imageIsOpen); err != nil {
		return Timespec{}, err
	}

	var cts C.struct_timespec

	if ret := C.rbd_get_access_timestamp(image.image, &cts); ret < 0 {
		return Timespec{}, getError(ret)
	}

	return Timespec(ts.CStructToTimespec(ts.CTimespecPtr(&cts))), nil
}

// GetModifyTimestamp returns the time the rbd image was last modified.
//
// Implements:
//  int rbd_get_modify_timestamp(rbd_image_t image, struct timespec *timestamp);
func (image *Image) GetModifyTimestamp() (Timespec, error) {
	if err := image.validate(imageIsOpen); err != nil {
		return Timespec{}, err
	}

	var cts C.struct_timespec

	if ret := C.rbd_get_modify_timestamp(image.image, &cts); ret < 0 {
		return Timespec{}, getError(ret)
	}

	return Timespec(ts.CStructToTimespec(ts.CTimespecPtr(&cts))), nil
}

// Sparsify makes an image sparse by deallocating runs of zeros.
// The sparseSize value will be used to find runs of zeros and must be
// a power of two no less than 4096 and no larger than the image size.
//
// Implements:
//  int rbd_sparsify(rbd_image_t image, size_t sparse_size);
func (image *Image) Sparsify(sparseSize uint) error {
	if err := image.validate(imageIsOpen); err != nil {
		return err
	}

	return getError(C.rbd_sparsify(image.image, C.size_t(sparseSize)))
}
<|endoftext|>


<commit_before>package reddit

import (
	"net/http"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/clientcredentials"
)

var oauthScopes = []string{
	"identity",
	"read",
	"privatemessages",
	"submit",
	"history",
}

type appClient struct {
	baseClient
	cfg    clientConfig
	cli    *http.Client
	expiry time.Time
}

func (a *appClient) Do(req *http.Request) ([]byte, error) {
	if time.Until(a.expiry) < time.Minute*5 {
		if err := a.authorize(); err != nil {
			return nil, err
		}
	}

	return a.baseClient.Do(req)
}

func (a *appClient) authorize() error {
	ctx := context.WithValue(oauth2.NoContext, oauth2.HTTPClient, a.cli)

	if a.cfg.app.Username == "" || a.cfg.app.Password == "" {
		a.baseClient.cli = a.clientCredentialsClient(ctx)
		return nil
	}

	cfg := &oauth2.Config{
		ClientID:     a.cfg.app.ID,
		ClientSecret: a.cfg.app.Secret,
		Endpoint:     oauth2.Endpoint{TokenURL: a.cfg.app.tokenURL},
		Scopes:       oauthScopes,
	}

	token, err := cfg.PasswordCredentialsToken(
		ctx,
		a.cfg.app.Username,
		a.cfg.app.Password,
	)

	a.baseClient.cli = cfg.Client(ctx, token)
	a.expiry = token.Expiry
	return err
}

func (a *appClient) clientCredentialsClient(ctx context.Context) *http.Client {
	cfg := &clientcredentials.Config{
		ClientID:     a.cfg.app.ID,
		ClientSecret: a.cfg.app.Secret,
		TokenURL:     a.cfg.app.tokenURL,
		Scopes:       oauthScopes,
	}

	return cfg.Client(ctx)
}

func newAppClient(c clientConfig) (*appClient, error) {
	a := &appClient{
		cli: clientWithAgent(c.agent),
		cfg: c,
	}
	return a, a.authorize()
}
<commit_msg>Adding error handling on PasswordCredentialsToken<commit_after>package reddit

import (
	"net/http"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/clientcredentials"
)

var oauthScopes = []string{
	"identity",
	"read",
	"privatemessages",
	"submit",
	"history",
}

type appClient struct {
	baseClient
	cfg    clientConfig
	cli    *http.Client
	expiry time.Time
}

func (a *appClient) Do(req *http.Request) ([]byte, error) {
	if time.Until(a.expiry) < time.Minute*5 {
		if err := a.authorize(); err != nil {
			return nil, err
		}
	}

	return a.baseClient.Do(req)
}

func (a *appClient) authorize() error {
	ctx := context.WithValue(oauth2.NoContext, oauth2.HTTPClient, a.cli)

	if a.cfg.app.Username == "" || a.cfg.app.Password == "" {
		a.baseClient.cli = a.clientCredentialsClient(ctx)
		return nil
	}

	cfg := &oauth2.Config{
		ClientID:     a.cfg.app.ID,
		ClientSecret: a.cfg.app.Secret,
		Endpoint:     oauth2.Endpoint{TokenURL: a.cfg.app.tokenURL},
		Scopes:       oauthScopes,
	}

	token, err := cfg.PasswordCredentialsToken(
		ctx,
		a.cfg.app.Username,
		a.cfg.app.Password,
	)
	if err != nil {
		return err
	}

	a.baseClient.cli = cfg.Client(ctx, token)
	a.expiry = token.Expiry
	return err
}

func (a *appClient) clientCredentialsClient(ctx context.Context) *http.Client {
	cfg := &clientcredentials.Config{
		ClientID:     a.cfg.app.ID,
		ClientSecret: a.cfg.app.Secret,
		TokenURL:     a.cfg.app.tokenURL,
		Scopes:       oauthScopes,
	}

	return cfg.Client(ctx)
}

func newAppClient(c clientConfig) (*appClient, error) {
	a := &appClient{
		cli: clientWithAgent(c.agent),
		cfg: c,
	}
	return a, a.authorize()
}
<|endoftext|>


<commit_before>package profile

import (
	"encoding/json"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/sodafoundation/nbp/client/opensds"
	"net/http"
	"os"
)

func GetProfile(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	key := vars["id"]
	opensdsEndpoint, errr := os.LookupEnv("OPENSDS_ENDPOINT")
	if !errr {
		fmt.Println("No env variables found for endpoint, switching to default")
		opensdsEndpoint = "http://127.0.0.1:50040"
	}
	client, err := opensds.GetClient(opensdsEndpoint, "keystone")
	if client == nil || err != nil {
		fmt.Printf("get opensds client failed: %v", err)
	}
	profile, errosds := client.GetProfile(key)
	if errosds != nil {
		fmt.Printf("Got error in GetProfile  : %s ===== %s", errosds.Error())
	}
	json.NewEncoder(w).Encode(profile.CustomProperties)
}
<commit_msg>Fix review comments<commit_after>package profile

import (
	"encoding/json"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/sodafoundation/nbp/client/opensds"
	"net/http"
	"os"
)

func GetProfile(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	key := vars["id"]
	opensdsEndpoint, errLookUp := os.LookupEnv("OPENSDS_ENDPOINT")
	if !errLookUp {
		fmt.Println("No env variables found for endpoint, switching to default")
		opensdsEndpoint = "http://127.0.0.1:50040"
	}
	client, err := opensds.GetClient(opensdsEndpoint, "keystone")
	if client == nil || err != nil {
		fmt.Printf("get opensds client failed: %v", err)
	}
	profile, errosds := client.GetProfile(key)
	if errosds != nil {
		fmt.Printf("got error in GetProfile  : %s", errosds.Error())
	}
	json.NewEncoder(w).Encode(profile.CustomProperties)
}
<|endoftext|>


<commit_before>package utils

import (
	"sync"
	"time"

	"fmt"

	"github.com/sirupsen/logrus"
)

type logItem struct {
	logTime time.Time
	logMsg  string
}

type AuditLog struct {
	log     *logrus.Logger
	dbUtils *DbUtils
	queue   chan logItem
	wg      *sync.WaitGroup
}

func (a *AuditLog) SetWaitGroup(wg *sync.WaitGroup) {
	a.wg = wg
}

func (a *AuditLog) SetLoggerAndDatabase(log *logrus.Logger, dbUtils *DbUtils) {
	a.log = log
	a.dbUtils = dbUtils
	a.queue = make(chan logItem, 128)
}

func (a *AuditLog) processQueue() {
	item := <-a.queue
	defer a.wg.Done()

	query := a.dbUtils.PQuery(`
		INSERT INTO audit_log (
			log_time, audit_msg
		)
		VALUES (?, ?)
	`)

	_, err := a.dbUtils.db.Exec(query, item.logTime, item.logMsg)

	if err != nil {
		fmt.Println("log error:", err)
	}
}

func (a AuditLog) Write(p []byte) (n int, err error) {
	item := logItem{
		logTime: time.Now().UTC(),
		logMsg:  string(p),
	}

	go a.processQueue()

	a.wg.Add(1)
	a.queue <- item

	return len(p), nil
}

func (a AuditLog) Log(err error, msgType string, msg string, details ...interface{}) {
	fields := make(map[string]interface{})

	if len(msgType) > 0 {
		fields["msg_type"] = msgType
	}

	isErr := false
	if err != nil {
		isErr = true
	}

	if isErr {
		fields["status"] = "failed"
	} else {
		fields["status"] = "successful"
	}

	if details != nil {
		var key string

		for i, detail := range details {
			if i%2 == 0 {
				key = detail.(string)
			} else {
				fields[key] = detail
			}
		}
	}

	hasKeys := false

	if len(fields) > 0 {
		hasKeys = true
	}

	if isErr {
		if hasKeys {
			if err != nil {
				a.log.WithError(err).WithFields(fields).Error(msg)
			} else {
				a.log.WithFields(fields).Error(msg)
			}
		} else {
			if err != nil {
				a.log.WithError(err).Error(msg)
			} else {
				a.log.Error(msg)
			}
		}
	} else {
		if hasKeys {
			if err != nil {
				a.log.WithError(err).WithFields(fields).Info(msg)
			} else {
				a.log.WithFields(fields).Info(msg)
			}
		} else {
			if err != nil {
				a.log.WithError(err).Info(msg)
			} else {
				a.log.Info(msg)
			}
		}
	}
}
<commit_msg>bug fix - if wait group was not inituialised the logging would not work<commit_after>package utils

import (
	"sync"
	"time"

	"fmt"

	"github.com/sirupsen/logrus"
)

type logItem struct {
	logTime time.Time
	logMsg  string
}

type AuditLog struct {
	log     *logrus.Logger
	dbUtils *DbUtils
	queue   chan logItem
	wg      *sync.WaitGroup
}

func (a *AuditLog) SetWaitGroup(wg *sync.WaitGroup) {
	a.wg = wg
}

func (a *AuditLog) SetLoggerAndDatabase(log *logrus.Logger, dbUtils *DbUtils) {
	a.log = log
	a.dbUtils = dbUtils
	a.queue = make(chan logItem, 128)
}

func (a *AuditLog) processQueue() {
	item := <-a.queue
	if a.wg != nil {
		defer a.wg.Done()
	}

	query := a.dbUtils.PQuery(`
		INSERT INTO audit_log (
			log_time, audit_msg
		)
		VALUES (?, ?)
	`)

	_, err := a.dbUtils.db.Exec(query, item.logTime, item.logMsg)

	if err != nil {
		fmt.Println("log error:", err)
	}
}

func (a AuditLog) Write(p []byte) (n int, err error) {
	item := logItem{
		logTime: time.Now().UTC(),
		logMsg:  string(p),
	}

	go a.processQueue()

	if a.wg != nil {
		a.wg.Add(1)
	}

	a.queue <- item

	return len(p), nil
}

func (a AuditLog) Log(err error, msgType string, msg string, details ...interface{}) {
	fields := make(map[string]interface{})

	if len(msgType) > 0 {
		fields["msg_type"] = msgType
	}

	isErr := false
	if err != nil {
		isErr = true
	}

	if isErr {
		fields["status"] = "failed"
	} else {
		fields["status"] = "successful"
	}

	if details != nil {
		var key string

		for i, detail := range details {
			if i%2 == 0 {
				key = detail.(string)
			} else {
				fields[key] = detail
			}
		}
	}

	hasKeys := false

	if len(fields) > 0 {
		hasKeys = true
	}

	if isErr {
		if hasKeys {
			if err != nil {
				a.log.WithError(err).WithFields(fields).Error(msg)
			} else {
				a.log.WithFields(fields).Error(msg)
			}
		} else {
			if err != nil {
				a.log.WithError(err).Error(msg)
			} else {
				a.log.Error(msg)
			}
		}
	} else {
		if hasKeys {
			if err != nil {
				a.log.WithError(err).WithFields(fields).Info(msg)
			} else {
				a.log.WithFields(fields).Info(msg)
			}
		} else {
			if err != nil {
				a.log.WithError(err).Info(msg)
			} else {
				a.log.Info(msg)
			}
		}
	}
}
<|endoftext|>