<commit_before>package cloudflare

import (
	"encoding/json"
	"fmt"
	"net/url"
	"strconv"
	"time"

	"github.com/pkg/errors"
)

// AccessGroup defines a group for allowing or disallowing access to
// one or more Access applications.
type AccessGroup struct {
	ID        string     `json:"id,omitempty"`
	CreatedAt *time.Time `json:"created_at"`
	UpdatedAt *time.Time `json:"updated_at"`
	Name      string     `json:"name"`

	// The include group works like an OR logical operator. The user must
	// satisfy one of the rules.
	Include []interface{} `json:"include"`

	// The exclude group works like a NOT logical operator. The user must
	// not satisfy all of the rules in exclude.
	Exclude []interface{} `json:"exclude"`

	// The require group works like a AND logical operator. The user must
	// satisfy all of the rules in require.
	Require []interface{} `json:"require"`
}

// AccessGroupEmail is used for managing access based on the email.
// For example, restrict access to users with the email addresses
// `test@example.com` or `someone@example.com`.
type AccessGroupEmail struct {
	Email struct {
		Email string `json:"email"`
	} `json:"email"`
}

// AccessGroupEmailDomain is used for managing access based on an email
// domain domain such as `example.com` instead of individual addresses.
type AccessGroupEmailDomain struct {
	EmailDomain struct {
		Domain string `json:"domain"`
	} `json:"email_domain"`
}

// AccessGroupIP is used for managing access based in the IP. It
// accepts individual IPs or CIDRs.
type AccessGroupIP struct {
	IP struct {
		IP string `json:"ip"`
	} `json:"ip"`
}

// AccessGroupEveryone is used for managing access to everyone.
type AccessGroupEveryone struct {
	Everyone struct{} `json:"everyone"`
}

// AccessGroupServiceToken is used for managing access based on a specific
// service token.
type AccessGroupServiceToken struct {
	ServiceToken struct {
		ID string `json:"token_id"`
	} `json:"service_token"`
}

// AccessGroupAnyValidServiceToken is used for managing access for all valid
// service tokens (not restricted).
type AccessGroupAnyValidServiceToken struct {
	AnyValidServiceToken struct{} `json:"any_valid_service_token"`
}

// AccessGroupAccessGroup is used for managing access based on an
// access group.
type AccessGroupAccessGroup struct {
	Group struct {
		ID string `json:"id"`
	} `json:"group"`
}

// AccessGroupListResponse represents the response from the list
// access group endpoint.
type AccessGroupListResponse struct {
	Result []AccessGroup `json:"result"`
	Response
	ResultInfo `json:"result_info"`
}

// AccessGroupDetailResponse is the API response, containing a single
// access group.
type AccessGroupDetailResponse struct {
	Success  bool        `json:"success"`
	Errors   []string    `json:"errors"`
	Messages []string    `json:"messages"`
	Result   AccessGroup `json:"result"`
}

// AccessGroups returns all access groups for an access application.
//
// API reference: https://api.cloudflare.com/#access-groups-list-access-groups
func (api *API) AccessGroups(accountID string, pageOpts PaginationOptions) ([]AccessGroup, ResultInfo, error) {
	v := url.Values{}
	if pageOpts.PerPage > 0 {
		v.Set("per_page", strconv.Itoa(pageOpts.PerPage))
	}
	if pageOpts.Page > 0 {
		v.Set("page", strconv.Itoa(pageOpts.Page))
	}

	uri := fmt.Sprintf(
		"/accounts/%s/access/groups",
		accountID,
	)

	if len(v) > 0 {
		uri = uri + "?" + v.Encode()
	}

	res, err := api.makeRequest("GET", uri, nil)
	if err != nil {
		return []AccessGroup{}, ResultInfo{}, errors.Wrap(err, errMakeRequestError)
	}

	var accessGroupListResponse AccessGroupListResponse
	err = json.Unmarshal(res, &accessGroupListResponse)
	if err != nil {
		return []AccessGroup{}, ResultInfo{}, errors.Wrap(err, errUnmarshalError)
	}

	return accessGroupListResponse.Result, accessGroupListResponse.ResultInfo, nil
}

// AccessGroup returns a single group based on the group ID.
//
// API reference: https://api.cloudflare.com/#access-groups-access-group-details
func (api *API) AccessGroup(accountID, groupID string) (AccessGroup, error) {
	uri := fmt.Sprintf(
		"/accounts/%s/access/groups/%s",
		accountID,
		groupID,
	)

	res, err := api.makeRequest("GET", uri, nil)
	if err != nil {
		return AccessGroup{}, errors.Wrap(err, errMakeRequestError)
	}

	var accessGroupDetailResponse AccessGroupDetailResponse
	err = json.Unmarshal(res, &accessGroupDetailResponse)
	if err != nil {
		return AccessGroup{}, errors.Wrap(err, errUnmarshalError)
	}

	return accessGroupDetailResponse.Result, nil
}

// CreateAccessGroup creates a new access group.
//
// API reference: https://api.cloudflare.com/#access-groups-create-access-group
func (api *API) CreateAccessGroup(accountID string, accessGroup AccessGroup) (AccessGroup, error) {
	uri := fmt.Sprintf(
		"/accounts/%s/access/groups",
		accountID,
	)

	res, err := api.makeRequest("POST", uri, accessGroup)
	if err != nil {
		return AccessGroup{}, errors.Wrap(err, errMakeRequestError)
	}

	var accessGroupDetailResponse AccessGroupDetailResponse
	err = json.Unmarshal(res, &accessGroupDetailResponse)
	if err != nil {
		return AccessGroup{}, errors.Wrap(err, errUnmarshalError)
	}

	return accessGroupDetailResponse.Result, nil
}

// UpdateAccessGroup updates an existing access group.
//
// API reference: https://api.cloudflare.com/#access-groups-update-access-group
func (api *API) UpdateAccessGroup(accountID string, accessGroup AccessGroup) (AccessGroup, error) {
	if accessGroup.ID == "" {
		return AccessGroup{}, errors.Errorf("access group ID cannot be empty")
	}
	uri := fmt.Sprintf(
		"/accounts/%s/access/groups/%s",
		accountID,
		accessGroup.ID,
	)

	res, err := api.makeRequest("PUT", uri, accessGroup)
	if err != nil {
		return AccessGroup{}, errors.Wrap(err, errMakeRequestError)
	}

	var accessGroupDetailResponse AccessGroupDetailResponse
	err = json.Unmarshal(res, &accessGroupDetailResponse)
	if err != nil {
		return AccessGroup{}, errors.Wrap(err, errUnmarshalError)
	}

	return accessGroupDetailResponse.Result, nil
}

// DeleteAccessGroup deletes an access group.
//
// API reference: https://api.cloudflare.com/#access-groups-delete-access-group
func (api *API) DeleteAccessGroup(accountID, groupID string) error {
	uri := fmt.Sprintf(
		"/accounts/%s/access/groups/%s",
		accountID,
		groupID,
	)

	_, err := api.makeRequest("DELETE", uri, nil)
	if err != nil {
		return errors.Wrap(err, errMakeRequestError)
	}

	return nil
}
<commit_msg>access_group: Add missing remaining configurations (#429)<commit_after>package cloudflare

import (
	"encoding/json"
	"fmt"
	"net/url"
	"strconv"
	"time"

	"github.com/pkg/errors"
)

// AccessGroup defines a group for allowing or disallowing access to
// one or more Access applications.
type AccessGroup struct {
	ID        string     `json:"id,omitempty"`
	CreatedAt *time.Time `json:"created_at"`
	UpdatedAt *time.Time `json:"updated_at"`
	Name      string     `json:"name"`

	// The include group works like an OR logical operator. The user must
	// satisfy one of the rules.
	Include []interface{} `json:"include"`

	// The exclude group works like a NOT logical operator. The user must
	// not satisfy all of the rules in exclude.
	Exclude []interface{} `json:"exclude"`

	// The require group works like a AND logical operator. The user must
	// satisfy all of the rules in require.
	Require []interface{} `json:"require"`
}

// AccessGroupEmail is used for managing access based on the email.
// For example, restrict access to users with the email addresses
// `test@example.com` or `someone@example.com`.
type AccessGroupEmail struct {
	Email struct {
		Email string `json:"email"`
	} `json:"email"`
}

// AccessGroupEmailDomain is used for managing access based on an email
// domain domain such as `example.com` instead of individual addresses.
type AccessGroupEmailDomain struct {
	EmailDomain struct {
		Domain string `json:"domain"`
	} `json:"email_domain"`
}

// AccessGroupIP is used for managing access based in the IP. It
// accepts individual IPs or CIDRs.
type AccessGroupIP struct {
	IP struct {
		IP string `json:"ip"`
	} `json:"ip"`
}

// AccessGroupEveryone is used for managing access to everyone.
type AccessGroupEveryone struct {
	Everyone struct{} `json:"everyone"`
}

// AccessGroupServiceToken is used for managing access based on a specific
// service token.
type AccessGroupServiceToken struct {
	ServiceToken struct {
		ID string `json:"token_id"`
	} `json:"service_token"`
}

// AccessGroupAnyValidServiceToken is used for managing access for all valid
// service tokens (not restricted).
type AccessGroupAnyValidServiceToken struct {
	AnyValidServiceToken struct{} `json:"any_valid_service_token"`
}

// AccessGroupAccessGroup is used for managing access based on an
// access group.
type AccessGroupAccessGroup struct {
	Group struct {
		ID string `json:"id"`
	} `json:"group"`
}

// AccessGroupCertificate is used for managing access to based on a valid
// mTLS cerificate being presented.
type AccessGroupCertificate struct {
	Certificate struct{} `json:"certificate"`
}

// AccessGroupCertificateCommonName is used for managing access based on a
// common name within a certificate.
type AccessGroupCertificateCommonName struct {
	CommonName struct {
		CommonName string `json:"common_name"`
	} `json:"common_name"`
}

// AccessGroupGSuite is used to configure access based on GSuite group.
type AccessGroupGSuite struct {
	Gsuite struct {
		Email              string `json:"email"`
		IdentityProviderID string `json:"identity_provider_id"`
	} `json:"gsuite"`
}

// AccessGroupGitHub is used to configure access based on a GitHub organisation.
type AccessGroupGitHub struct {
	GitHubOrganization struct {
		Name               string `json:"name"`
		IdentityProviderID string `json:"identity_provider_id"`
	} `json:"github-organization"`
}

// AccessGroupAzure is used to configure access based on a Azure group.
type AccessGroupAzure struct {
	AzureAD struct {
		ID                 string `json:"id"`
		IdentityProviderID string `json:"identity_provider_id"`
	} `json:"azureAD"`
}

// AccessGroupOkta is used to configure access based on a Okta group.
type AccessGroupOkta struct {
	Otka struct {
		Name               string `json:"name"`
		IdentityProviderID string `json:"identity_provider_id"`
	} `json:"otka"`
}

// AccessGroupSAML is used to allow SAML users with a specific attribute
// configuration.
type AccessGroupSAML struct {
	Saml struct {
		AttributeName      string `json:"attribute_name"`
		AttributeValue     string `json:"attribute_value"`
		IdentityProviderID string `json:"identity_provider_id"`
	} `json:"saml"`
}

// AccessGroupListResponse represents the response from the list
// access group endpoint.
type AccessGroupListResponse struct {
	Result []AccessGroup `json:"result"`
	Response
	ResultInfo `json:"result_info"`
}

// AccessGroupDetailResponse is the API response, containing a single
// access group.
type AccessGroupDetailResponse struct {
	Success  bool        `json:"success"`
	Errors   []string    `json:"errors"`
	Messages []string    `json:"messages"`
	Result   AccessGroup `json:"result"`
}

// AccessGroups returns all access groups for an access application.
//
// API reference: https://api.cloudflare.com/#access-groups-list-access-groups
func (api *API) AccessGroups(accountID string, pageOpts PaginationOptions) ([]AccessGroup, ResultInfo, error) {
	v := url.Values{}
	if pageOpts.PerPage > 0 {
		v.Set("per_page", strconv.Itoa(pageOpts.PerPage))
	}
	if pageOpts.Page > 0 {
		v.Set("page", strconv.Itoa(pageOpts.Page))
	}

	uri := fmt.Sprintf(
		"/accounts/%s/access/groups",
		accountID,
	)

	if len(v) > 0 {
		uri = uri + "?" + v.Encode()
	}

	res, err := api.makeRequest("GET", uri, nil)
	if err != nil {
		return []AccessGroup{}, ResultInfo{}, errors.Wrap(err, errMakeRequestError)
	}

	var accessGroupListResponse AccessGroupListResponse
	err = json.Unmarshal(res, &accessGroupListResponse)
	if err != nil {
		return []AccessGroup{}, ResultInfo{}, errors.Wrap(err, errUnmarshalError)
	}

	return accessGroupListResponse.Result, accessGroupListResponse.ResultInfo, nil
}

// AccessGroup returns a single group based on the group ID.
//
// API reference: https://api.cloudflare.com/#access-groups-access-group-details
func (api *API) AccessGroup(accountID, groupID string) (AccessGroup, error) {
	uri := fmt.Sprintf(
		"/accounts/%s/access/groups/%s",
		accountID,
		groupID,
	)

	res, err := api.makeRequest("GET", uri, nil)
	if err != nil {
		return AccessGroup{}, errors.Wrap(err, errMakeRequestError)
	}

	var accessGroupDetailResponse AccessGroupDetailResponse
	err = json.Unmarshal(res, &accessGroupDetailResponse)
	if err != nil {
		return AccessGroup{}, errors.Wrap(err, errUnmarshalError)
	}

	return accessGroupDetailResponse.Result, nil
}

// CreateAccessGroup creates a new access group.
//
// API reference: https://api.cloudflare.com/#access-groups-create-access-group
func (api *API) CreateAccessGroup(accountID string, accessGroup AccessGroup) (AccessGroup, error) {
	uri := fmt.Sprintf(
		"/accounts/%s/access/groups",
		accountID,
	)

	res, err := api.makeRequest("POST", uri, accessGroup)
	if err != nil {
		return AccessGroup{}, errors.Wrap(err, errMakeRequestError)
	}

	var accessGroupDetailResponse AccessGroupDetailResponse
	err = json.Unmarshal(res, &accessGroupDetailResponse)
	if err != nil {
		return AccessGroup{}, errors.Wrap(err, errUnmarshalError)
	}

	return accessGroupDetailResponse.Result, nil
}

// UpdateAccessGroup updates an existing access group.
//
// API reference: https://api.cloudflare.com/#access-groups-update-access-group
func (api *API) UpdateAccessGroup(accountID string, accessGroup AccessGroup) (AccessGroup, error) {
	if accessGroup.ID == "" {
		return AccessGroup{}, errors.Errorf("access group ID cannot be empty")
	}
	uri := fmt.Sprintf(
		"/accounts/%s/access/groups/%s",
		accountID,
		accessGroup.ID,
	)

	res, err := api.makeRequest("PUT", uri, accessGroup)
	if err != nil {
		return AccessGroup{}, errors.Wrap(err, errMakeRequestError)
	}

	var accessGroupDetailResponse AccessGroupDetailResponse
	err = json.Unmarshal(res, &accessGroupDetailResponse)
	if err != nil {
		return AccessGroup{}, errors.Wrap(err, errUnmarshalError)
	}

	return accessGroupDetailResponse.Result, nil
}

// DeleteAccessGroup deletes an access group.
//
// API reference: https://api.cloudflare.com/#access-groups-delete-access-group
func (api *API) DeleteAccessGroup(accountID, groupID string) error {
	uri := fmt.Sprintf(
		"/accounts/%s/access/groups/%s",
		accountID,
		groupID,
	)

	_, err := api.makeRequest("DELETE", uri, nil)
	if err != nil {
		return errors.Wrap(err, errMakeRequestError)
	}

	return nil
}
<|endoftext|>


<commit_before>package gmime

/*
#cgo pkg-config: gmime-2.6
#include <stdlib.h>
#include <gmime/gmime.h>
*/
import "C"
import (
	"unsafe"
)

type Message interface {
	Object
	SetSender(string)
	Sender() (string, bool)
	SetReplyTo(string)
	ReplyTo() (string, bool)
	SetSubject(string)
	Subject() (string, bool)
	SetMessageId(string)
	MessageId() (string, bool)
	DateAsString() (string, bool)
	SetDateAsString(string)
	AddTo(string, string)
	To() *InternetAddressList
	AddCc(string, string)
	Cc() *InternetAddressList
	AddBcc(string, string)
	Bcc() *InternetAddressList
	AllRecipients() *InternetAddressList
	SetMimePart(Object)
	MimePart() Object
	Date() string
}

type aMessage struct {
	*anObject
}

type rawMessage interface {
	Message
	rawMessage() *C.GMimeMessage
}

func CastMessage(m *C.GMimeMessage) *aMessage {
	return &aMessage{CastObject((*C.GMimeObject)(unsafe.Pointer(m)))}
}

func NewMessage() Message {
	m := C.g_mime_message_new(0)
	defer unref(C.gpointer(m))
	return CastMessage(m)
}

func (m *aMessage) SetSender(sender string) {
	var cSender *C.char = C.CString(sender)
	C.g_mime_message_set_sender(m.rawMessage(), cSender)
	C.free(unsafe.Pointer(cSender))
}

func (m *aMessage) Sender() (string, bool) {
	sender := C.g_mime_message_get_sender(m.rawMessage())
	return maybeGoString(sender)
}

func (m *aMessage) SetReplyTo(replyTo string) {
	var cReply *C.char = C.CString(replyTo)
	C.g_mime_message_set_reply_to(m.rawMessage(), cReply)
	C.free(unsafe.Pointer(cReply))
}

func (m *aMessage) ReplyTo() (string, bool) {
	replyTo := C.g_mime_message_get_reply_to(m.rawMessage())
	return maybeGoString(replyTo)
}

func (m *aMessage) SetSubject(subject string) {
	var cSubject *C.char = C.CString(subject)
	C.g_mime_message_set_subject(m.rawMessage(), cSubject)
	C.free(unsafe.Pointer(cSubject))
}

func (m *aMessage) Subject() (string, bool) {
	subject := C.g_mime_message_get_subject(m.rawMessage())
	return maybeGoString(subject)
}

func (m *aMessage) SetMessageId(messageId string) {
	var cMessageId *C.char = C.CString(messageId)
	C.g_mime_message_set_message_id(m.rawMessage(), cMessageId)
	C.free(unsafe.Pointer(cMessageId))
}

func (m *aMessage) MessageId() (string, bool) {
	messageId := C.g_mime_message_get_message_id(m.rawMessage())
	return maybeGoString(messageId)
}

func (m *aMessage) DateAsString() (string, bool) {
	date := C.g_mime_message_get_date_as_string(m.rawMessage())
	defer C.free(unsafe.Pointer(date))
	return maybeGoString(date)
}

func (m *aMessage) SetDateAsString(date string) {
	var cDate *C.char = C.CString(date)
	C.g_mime_message_set_date_as_string(m.rawMessage(), cDate)
	C.free(unsafe.Pointer(cDate))
}

func (m *aMessage) addRecipient(recipientType C.GMimeRecipientType, name string, address string) {
	var cName *C.char = C.CString(name)
	var cAddress *C.char = C.CString(address)
	C.g_mime_message_add_recipient(m.rawMessage(), recipientType, cName, cAddress)
	C.free(unsafe.Pointer(cName))
	C.free(unsafe.Pointer(cAddress))
}

func (m *aMessage) AddTo(name string, address string) {
	m.addRecipient(C.GMIME_RECIPIENT_TYPE_TO, name, address)
}

func (m *aMessage) To() *InternetAddressList {
	cList := C.g_mime_message_get_recipients(m.rawMessage(), C.GMIME_RECIPIENT_TYPE_TO)
	return CastInternetAddressList(cList)
}

func (m *aMessage) AddCc(name string, address string) {
	m.addRecipient(C.GMIME_RECIPIENT_TYPE_CC, name, address)
}

func (m *aMessage) Cc() *InternetAddressList {
	cList := C.g_mime_message_get_recipients(m.rawMessage(), C.GMIME_RECIPIENT_TYPE_CC)
	return CastInternetAddressList(cList)
}

func (m *aMessage) AddBcc(name string, address string) {
	m.addRecipient(C.GMIME_RECIPIENT_TYPE_BCC, name, address)
}

func (m *aMessage) Bcc() *InternetAddressList {
	cList := C.g_mime_message_get_recipients(m.rawMessage(), C.GMIME_RECIPIENT_TYPE_BCC)
	return CastInternetAddressList(cList)
}

func (m *aMessage) AllRecipients() *InternetAddressList {
	// This is major exception: we have newly allocated list here
	cList := C.g_mime_message_get_all_recipients(m.rawMessage())
	defer unref(C.gpointer(cList))
	return CastInternetAddressList(cList)
}

func (m *aMessage) SetMimePart(mimePart Object) {
	part := mimePart.(rawObject)
	switch mimePart.(type) {
	case Part:
		C.g_mime_message_set_mime_part(m.rawMessage(), part.rawObject())
	case Multipart:
		C.g_mime_message_set_mime_part(m.rawMessage(), part.rawObject())
	}
}

func (m *aMessage) MimePart() Object {
	object := C.g_mime_message_get_mime_part(m.rawMessage())
	return objectAsSubclass(object)
}

func (m *aMessage) Date() string {
	cDate := C.g_mime_message_get_date_as_string(m.rawMessage())

	return C.GoString(cDate)
}

func (m *aMessage) rawMessage() *C.GMimeMessage {
	return (*C.GMimeMessage)(m.pointer())
}
<commit_msg>Export gmime_message_get_body<commit_after>package gmime

/*
#cgo pkg-config: gmime-2.6
#include <stdlib.h>
#include <gmime/gmime.h>
*/
import "C"
import (
	"unsafe"
)

type Message interface {
	Object
	SetSender(string)
	Sender() (string, bool)
	SetReplyTo(string)
	ReplyTo() (string, bool)
	SetSubject(string)
	Subject() (string, bool)
	SetMessageId(string)
	MessageId() (string, bool)
	DateAsString() (string, bool)
	SetDateAsString(string)
	AddTo(string, string)
	To() *InternetAddressList
	AddCc(string, string)
	Cc() *InternetAddressList
	AddBcc(string, string)
	Bcc() *InternetAddressList
	AllRecipients() *InternetAddressList
	SetMimePart(Object)
	MimePart() Object
	Body() Object
	Date() string
}

type aMessage struct {
	*anObject
}

type rawMessage interface {
	Message
	rawMessage() *C.GMimeMessage
}

func CastMessage(m *C.GMimeMessage) *aMessage {
	return &aMessage{CastObject((*C.GMimeObject)(unsafe.Pointer(m)))}
}

func NewMessage() Message {
	m := C.g_mime_message_new(0)
	defer unref(C.gpointer(m))
	return CastMessage(m)
}

func (m *aMessage) SetSender(sender string) {
	var cSender *C.char = C.CString(sender)
	C.g_mime_message_set_sender(m.rawMessage(), cSender)
	C.free(unsafe.Pointer(cSender))
}

func (m *aMessage) Sender() (string, bool) {
	sender := C.g_mime_message_get_sender(m.rawMessage())
	return maybeGoString(sender)
}

func (m *aMessage) SetReplyTo(replyTo string) {
	var cReply *C.char = C.CString(replyTo)
	C.g_mime_message_set_reply_to(m.rawMessage(), cReply)
	C.free(unsafe.Pointer(cReply))
}

func (m *aMessage) ReplyTo() (string, bool) {
	replyTo := C.g_mime_message_get_reply_to(m.rawMessage())
	return maybeGoString(replyTo)
}

func (m *aMessage) SetSubject(subject string) {
	var cSubject *C.char = C.CString(subject)
	C.g_mime_message_set_subject(m.rawMessage(), cSubject)
	C.free(unsafe.Pointer(cSubject))
}

func (m *aMessage) Subject() (string, bool) {
	subject := C.g_mime_message_get_subject(m.rawMessage())
	return maybeGoString(subject)
}

func (m *aMessage) SetMessageId(messageId string) {
	var cMessageId *C.char = C.CString(messageId)
	C.g_mime_message_set_message_id(m.rawMessage(), cMessageId)
	C.free(unsafe.Pointer(cMessageId))
}

func (m *aMessage) MessageId() (string, bool) {
	messageId := C.g_mime_message_get_message_id(m.rawMessage())
	return maybeGoString(messageId)
}

func (m *aMessage) DateAsString() (string, bool) {
	date := C.g_mime_message_get_date_as_string(m.rawMessage())
	defer C.free(unsafe.Pointer(date))
	return maybeGoString(date)
}

func (m *aMessage) SetDateAsString(date string) {
	var cDate *C.char = C.CString(date)
	C.g_mime_message_set_date_as_string(m.rawMessage(), cDate)
	C.free(unsafe.Pointer(cDate))
}

func (m *aMessage) addRecipient(recipientType C.GMimeRecipientType, name string, address string) {
	var cName *C.char = C.CString(name)
	var cAddress *C.char = C.CString(address)
	C.g_mime_message_add_recipient(m.rawMessage(), recipientType, cName, cAddress)
	C.free(unsafe.Pointer(cName))
	C.free(unsafe.Pointer(cAddress))
}

func (m *aMessage) AddTo(name string, address string) {
	m.addRecipient(C.GMIME_RECIPIENT_TYPE_TO, name, address)
}

func (m *aMessage) To() *InternetAddressList {
	cList := C.g_mime_message_get_recipients(m.rawMessage(), C.GMIME_RECIPIENT_TYPE_TO)
	return CastInternetAddressList(cList)
}

func (m *aMessage) AddCc(name string, address string) {
	m.addRecipient(C.GMIME_RECIPIENT_TYPE_CC, name, address)
}

func (m *aMessage) Cc() *InternetAddressList {
	cList := C.g_mime_message_get_recipients(m.rawMessage(), C.GMIME_RECIPIENT_TYPE_CC)
	return CastInternetAddressList(cList)
}

func (m *aMessage) AddBcc(name string, address string) {
	m.addRecipient(C.GMIME_RECIPIENT_TYPE_BCC, name, address)
}

func (m *aMessage) Bcc() *InternetAddressList {
	cList := C.g_mime_message_get_recipients(m.rawMessage(), C.GMIME_RECIPIENT_TYPE_BCC)
	return CastInternetAddressList(cList)
}

func (m *aMessage) AllRecipients() *InternetAddressList {
	// This is major exception: we have newly allocated list here
	cList := C.g_mime_message_get_all_recipients(m.rawMessage())
	defer unref(C.gpointer(cList))
	return CastInternetAddressList(cList)
}

func (m *aMessage) SetMimePart(mimePart Object) {
	part := mimePart.(rawObject)
	switch mimePart.(type) {
	case Part:
		C.g_mime_message_set_mime_part(m.rawMessage(), part.rawObject())
	case Multipart:
		C.g_mime_message_set_mime_part(m.rawMessage(), part.rawObject())
	}
}

func (m *aMessage) MimePart() Object {
	object := C.g_mime_message_get_mime_part(m.rawMessage())
	return objectAsSubclass(object)
}

func (m *aMessage) Body() Object {
	object := C.g_mime_message_get_body(m.rawMessage())
	return objectAsSubclass(object)
}

func (m *aMessage) Date() string {
	cDate := C.g_mime_message_get_date_as_string(m.rawMessage())

	return C.GoString(cDate)
}

func (m *aMessage) rawMessage() *C.GMimeMessage {
	return (*C.GMimeMessage)(m.pointer())
}
<|endoftext|>


<commit_before>//
// Code related to temporal pooler stats
//

package htm

import (
	//"fmt"
	//"github.com/cznic/mathutil"
	"github.com/zacg/floats"
	//"github.com/zacg/go.matrix"
	//"math"
	//"math/rand"
	//"sort"
)

type TpStats struct {
	NInfersSinceReset       int
	NPredictions            int
	PredictionScoreTotal    float64
	PredictionScoreTotal2   float64
	FalseNegativeScoreTotal float64
	FalsePositiveScoreTotal float64
	PctExtraTotal           float64
	PctMissingTotal         float64
	TotalMissing            float64
	TotalExtra              float64

	CurPredictionScore    float64
	CurPredictionScore2   float64
	CurFalseNegativeScore float64
	CurFalsePositiveScore float64
	CurMissing            float64
	CurExtra              float64
}

/*
 This function produces goodness-of-match scores for a set of input patterns,
by checking for their presence in the current and predicted output of the
TP. Returns a global count of the number of extra and missing bits, the
confidence scores for each input pattern, and (if requested) the
bits in each input pattern that were not present in the TP's prediction.

@param patternNZs a list of input patterns that we want to check for. Each
element is a list of the non-zeros in that pattern.
@param output The output of the TP. If not specified, then use the
TP's current output. This can be specified if you are
trying to check the prediction metric for an output from
the past.
@param colConfidence The column confidences. If not specified, then use the
TP's current self.colConfidence. This can be specified if you
are trying to check the prediction metrics for an output
from the past.
@param details if True, also include details of missing bits per pattern.

@returns list containing:

[
totalExtras,
totalMissing,
[conf_1, conf_2, ...],
[missing1, missing2, ...]
]

@retval totalExtras a global count of the number of 'extras', i.e. bits that
are on in the current output but not in the or of all the
passed in patterns
@retval totalMissing a global count of all the missing bits, i.e. the bits
that are on in the or of the patterns, but not in the
current output
@retval conf_i the confidence score for the i'th pattern inpatternsToCheck
This consists of 3 items as a tuple:
(predictionScore, posPredictionScore, negPredictionScore)
@retval missing_i the bits in the i'th pattern that were missing
in the output. This list is only returned if details is
True.
*/

type confidence struct {
	PredictionScore         float64
	PositivePredictionScore float64
	NegativePredictionScore float64
}

func (tp *TemporalPooler) checkPrediction2(patternNZs [][]int, output *SparseBinaryMatrix,
	colConfidence []float64, details bool) (int, int, []confidence, []int) {

	// Get the non-zeros in each pattern
	numPatterns := len(patternNZs)

	// Compute the union of all the expected patterns
	var orAll []int
	for _, row := range patternNZs {
		for _, col := range row {
			if !ContainsInt(col, orAll) {
				orAll = append(orAll, col)
			}
		}
	}

	var outputIdxs []int

	// Get the list of active columns in the output
	if output == nil {
		if tp.CurrentOutput == nil {
			panic("Expected tp output")
		}
		outputIdxs = tp.CurrentOutput.NonZeroRows()
	} else {
		outputIdxs = output.NonZeroRows()
	}

	// Compute the total extra and missing in the output
	totalExtras := 0
	totalMissing := 0

	for _, val := range outputIdxs {
		if !ContainsInt(val, orAll) {
			totalExtras++
		}
	}

	for _, val := range orAll {
		if !ContainsInt(val, outputIdxs) {
			totalMissing++
		}
	}

	// Get the percent confidence level per column by summing the confidence
	// levels of the cells in the column. During training, each segment's
	// confidence number is computed as a running average of how often it
	// correctly predicted bottom-up activity on that column. A cell's
	// confidence number is taken from the first active segment found in the
	// cell. Note that confidence will only be non-zero for predicted columns.

	if colConfidence == nil {
		copy(colConfidence, tp.DynamicState.colConfidence)
	}

	// Assign confidences to each pattern
	var confidences []confidence

	for i := 0; i < numPatterns; i++ {
		// Sum of the column confidences for this pattern
		//positivePredictionSum = colConfidence[patternNZs[i]].sum()
		positivePredictionSum := floats.Sum(floats.SubSet(colConfidence, patternNZs[i]))

		// How many columns in this pattern
		positiveColumnCount := len(patternNZs[i])

		// Sum of all the column confidences
		totalPredictionSum := floats.Sum(colConfidence)
		// Total number of columns
		totalColumnCount := len(colConfidence)

		negativePredictionSum := totalPredictionSum - positivePredictionSum
		negativeColumnCount := totalColumnCount - positiveColumnCount

		positivePredictionScore := 0.0
		// Compute the average confidence score per column for this pattern
		if positiveColumnCount != 0 {
			positivePredictionScore = positivePredictionSum
		}

		// Compute the average confidence score per column for the other patterns
		negativePredictionScore := 0.0
		if negativeColumnCount != 0 {
			negativePredictionScore = negativePredictionSum
		}

		// Scale the positive and negative prediction scores so that they sum to
		// 1.0
		currentSum := negativePredictionScore + positivePredictionScore
		if currentSum > 0 {
			positivePredictionScore *= 1.0 / currentSum
			negativePredictionScore *= 1.0 / currentSum
		}

		predictionScore := positivePredictionScore - negativePredictionScore
		newConf := confidence{predictionScore, positivePredictionScore, negativePredictionScore}
		confidences = append(confidences, newConf)

	}

	// Include detail? (bits in each pattern that were missing from the output)
	if details {
		var missingPatternBits []int
		for _, pattern := range patternNZs {
			for _, val := range pattern {
				if !ContainsInt(val, outputIdxs) &&
					!ContainsInt(val, missingPatternBits) {
					missingPatternBits = append(missingPatternBits, val)
				}
			}

		}
		return totalExtras, totalMissing, confidences, missingPatternBits
	} else {
		return totalExtras, totalMissing, confidences, nil
	}

}
<commit_msg>implemented updateStatsInferEnd()<commit_after>//
// Code related to temporal pooler stats
//

package htm

import (
	//"fmt"
	"github.com/cznic/mathutil"
	"github.com/zacg/floats"
	"github.com/zacg/go.matrix"
	//"math"
	//"math/rand"
	//"sort"
)

type TpStats struct {
	NInfersSinceReset       int
	NPredictions            int
	PredictionScoreTotal    float64
	PredictionScoreTotal2   float64
	FalseNegativeScoreTotal float64
	FalsePositiveScoreTotal float64
	PctExtraTotal           float64
	PctMissingTotal         float64
	TotalMissing            float64
	TotalExtra              float64

	CurPredictionScore    float64
	CurPredictionScore2   float64
	CurFalseNegativeScore float64
	CurFalsePositiveScore float64
	CurMissing            float64
	CurExtra              float64
	ConfHistogram         matrix.DenseMatrix
}

/*
 This function produces goodness-of-match scores for a set of input patterns,
by checking for their presence in the current and predicted output of the
TP. Returns a global count of the number of extra and missing bits, the
confidence scores for each input pattern, and (if requested) the
bits in each input pattern that were not present in the TP's prediction.

param patternNZs a list of input patterns that we want to check for. Each
element is a list of the non-zeros in that pattern.
param output The output of the TP. If not specified, then use the
TP's current output. This can be specified if you are
trying to check the prediction metric for an output from
the past.
param colConfidence The column confidences. If not specified, then use the
TP's current colConfidence. This can be specified if you
are trying to check the prediction metrics for an output
from the past.
param details if True, also include details of missing bits per pattern.

returns list containing:

[
totalExtras,
totalMissing,
[conf_1, conf_2, ...],
[missing1, missing2, ...]
]

retval totalExtras a global count of the number of 'extras', i.e. bits that
are on in the current output but not in the or of all the
passed in patterns
retval totalMissing a global count of all the missing bits, i.e. the bits
that are on in the or of the patterns, but not in the
current output
retval conf_i the confidence score for the i'th pattern inpatternsToCheck
This consists of 3 items as a tuple:
(predictionScore, posPredictionScore, negPredictionScore)
retval missing_i the bits in the i'th pattern that were missing
in the output. This list is only returned if details is
True.
*/

type confidence struct {
	PredictionScore         float64
	PositivePredictionScore float64
	NegativePredictionScore float64
}

func (tp *TemporalPooler) checkPrediction2(patternNZs [][]int, output *SparseBinaryMatrix,
	colConfidence []float64, details bool) (int, int, []confidence, []int) {

	// Get the non-zeros in each pattern
	numPatterns := len(patternNZs)

	// Compute the union of all the expected patterns
	var orAll []int
	for _, row := range patternNZs {
		for _, col := range row {
			if !ContainsInt(col, orAll) {
				orAll = append(orAll, col)
			}
		}
	}

	var outputIdxs []int

	// Get the list of active columns in the output
	if output == nil {
		if tp.CurrentOutput == nil {
			panic("Expected tp output")
		}
		outputIdxs = tp.CurrentOutput.NonZeroRows()
	} else {
		outputIdxs = output.NonZeroRows()
	}

	// Compute the total extra and missing in the output
	totalExtras := 0
	totalMissing := 0

	for _, val := range outputIdxs {
		if !ContainsInt(val, orAll) {
			totalExtras++
		}
	}

	for _, val := range orAll {
		if !ContainsInt(val, outputIdxs) {
			totalMissing++
		}
	}

	// Get the percent confidence level per column by summing the confidence
	// levels of the cells in the column. During training, each segment's
	// confidence number is computed as a running average of how often it
	// correctly predicted bottom-up activity on that column. A cell's
	// confidence number is taken from the first active segment found in the
	// cell. Note that confidence will only be non-zero for predicted columns.

	if colConfidence == nil {
		copy(colConfidence, tp.DynamicState.colConfidence)
	}

	// Assign confidences to each pattern
	var confidences []confidence

	for i := 0; i < numPatterns; i++ {
		// Sum of the column confidences for this pattern
		//positivePredictionSum = colConfidence[patternNZs[i]].sum()
		positivePredictionSum := floats.Sum(floats.SubSet(colConfidence, patternNZs[i]))

		// How many columns in this pattern
		positiveColumnCount := len(patternNZs[i])

		// Sum of all the column confidences
		totalPredictionSum := floats.Sum(colConfidence)
		// Total number of columns
		totalColumnCount := len(colConfidence)

		negativePredictionSum := totalPredictionSum - positivePredictionSum
		negativeColumnCount := totalColumnCount - positiveColumnCount

		positivePredictionScore := 0.0
		// Compute the average confidence score per column for this pattern
		if positiveColumnCount != 0 {
			positivePredictionScore = positivePredictionSum
		}

		// Compute the average confidence score per column for the other patterns
		negativePredictionScore := 0.0
		if negativeColumnCount != 0 {
			negativePredictionScore = negativePredictionSum
		}

		// Scale the positive and negative prediction scores so that they sum to
		// 1.0
		currentSum := negativePredictionScore + positivePredictionScore
		if currentSum > 0 {
			positivePredictionScore *= 1.0 / currentSum
			negativePredictionScore *= 1.0 / currentSum
		}

		predictionScore := positivePredictionScore - negativePredictionScore
		newConf := confidence{predictionScore, positivePredictionScore, negativePredictionScore}
		confidences = append(confidences, newConf)

	}

	// Include detail? (bits in each pattern that were missing from the output)
	if details {
		var missingPatternBits []int
		for _, pattern := range patternNZs {
			for _, val := range pattern {
				if !ContainsInt(val, outputIdxs) &&
					!ContainsInt(val, missingPatternBits) {
					missingPatternBits = append(missingPatternBits, val)
				}
			}

		}
		return totalExtras, totalMissing, confidences, missingPatternBits
	} else {
		return totalExtras, totalMissing, confidences, nil
	}

}

/*
	 Called at the end of learning and inference, this routine will update
a number of stats in our _internalStats dictionary, including our computed
prediction score.

param stats internal stats dictionary
param bottomUpNZ list of the active bottom-up inputs
param predictedState The columns we predicted on the last time step (should
match the current bottomUpNZ in the best case)
param colConfidence Column confidences we determined on the last time step
*/

func (tp *TemporalPooler) updateStatsInferEnd(stats *TpStats, bottomUpNZ []int,
	predictedState *SparseBinaryMatrix, colConfidence []float64) {
	// Return if not collecting stats
	if !tp.params.CollectStats {
		return
	}

	stats.NInfersSinceReset++

	// Compute the prediction score, how well the prediction from the last
	// time step predicted the current bottom-up input
	numExtra2, numMissing2, confidences2, _ := tp.checkPrediction2([][]int{bottomUpNZ}, predictedState, colConfidence, false)
	predictionScore := confidences2[0].PredictionScore
	positivePredictionScore := confidences2[0].PositivePredictionScore
	negativePredictionScore := confidences2[0].NegativePredictionScore

	// Store the stats that don't depend on burn-in
	stats.CurPredictionScore2 = predictionScore
	stats.CurFalseNegativeScore = negativePredictionScore
	stats.CurFalsePositiveScore = positivePredictionScore

	stats.CurMissing = float64(numMissing2)
	stats.CurExtra = float64(numExtra2)

	// If we are passed the burn-in period, update the accumulated stats
	// Here's what various burn-in values mean:
	// 0: try to predict the first element of each sequence and all subsequent
	// 1: try to predict the second element of each sequence and all subsequent
	// etc.
	if stats.NInfersSinceReset <= tp.params.BurnIn {
		return
	}

	// Burn-in related stats
	stats.NPredictions++
	numExpected := mathutil.Max(1, len(bottomUpNZ))

	stats.TotalMissing += float64(numMissing2)
	stats.TotalExtra += float64(numExtra2)
	stats.PctExtraTotal += 100.0 * float64(numExtra2) / float64(numExpected)
	stats.PctMissingTotal += 100.0 * float64(numMissing2) / float64(numExpected)
	stats.PredictionScoreTotal2 += predictionScore
	stats.FalseNegativeScoreTotal += 1.0 - positivePredictionScore
	stats.FalsePositiveScoreTotal += negativePredictionScore

	if tp.collectSequenceStats {
		// Collect cell confidences for every cell that correctly predicted current
		// bottom up input. Normalize confidence across each column
		cc := tp.DynamicState.cellConfidence.Copy()

		for r := 0; r < cc.Rows(); r++ {
			for c := 0; c < cc.Cols(); c++ {
				if !tp.DynamicState.infActiveState.Get(r, c) {
					cc.Set(r, c, 0)
				}
			}
		}
		sconf := make([]int, cc.Rows())
		for r := 0; r < cc.Rows(); r++ {
			count := 0
			for c := 0; c < cc.Cols(); c++ {
				if cc.Get(r, c) > 0 {
					count++
				}
			}
			sconf[r] = count
		}

		for r := 0; r < cc.Rows(); r++ {
			for c := 0; c < cc.Cols(); c++ {
				temp := cc.Get(r, c)
				cc.Set(r, c, temp/float64(sconf[r]))
			}
		}

		// Update cell confidence histogram: add column-normalized confidence
		// scores to the histogram
		stats.ConfHistogram.Add(cc)
	}

}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path"
	"regexp"
	"strings"
	"time"

	"github.com/hashicorp/go-version"
	"github.com/urfave/cli"
	"gopkg.in/yaml.v2"
)

type ProjectSync struct {
	BaseCommand
}

// Minimal compose file struct to discover volumes
type ComposeFile struct {
	Volumes map[string]Volume
}

type Volume struct {
	External bool
}

const UNISON_PORT = 5000
const MAX_WATCHES = "100000"

func (cmd *ProjectSync) Commands() []cli.Command {
	start := cli.Command{
		Name:        "sync:start",
		Aliases:     []string{"sync"},
		Usage:       "Start a unison sync on local project directory. Optionally provide a volume name.",
		ArgsUsage:   "[optional volume name]",
		Description: "Volume name will be discovered in the following order: argument to this command > outrigger project config > docker-compose file > current directory name",
		Flags: []cli.Flag{
			cli.IntFlag{
				Name:   "initial-sync-timeout",
				Value:  60,
				Usage:  "Maximum amount of time in seconds to allow for detecting each of start of the unison container and start of initial sync",
				EnvVar: "RIG_PROJECT_SYNC_TIMEOUT",
			},
			cli.IntFlag{
				Name:   "initial-sync-wait",
				Value:  5,
				Usage:  "Time in seconds to wait between checks to see if initial sync has finished.",
				EnvVar: "RIG_PROJECT_INITIAL_SYNC_WAIT",
			},
		},
		Before: cmd.Before,
		Action: cmd.RunStart,
	}
	stop := cli.Command{
		Name:        "sync:stop",
		Usage:       "Stops a unison sync on local project directory. Optionally provide a volume name.",
		ArgsUsage:   "[optional volume name]",
		Description: "Volume name will be discovered in the following order: argument to this command > outrigger project config > docker-compose file > current directory name",
		Before:      cmd.Before,
		Action:      cmd.RunStop,
	}

	return []cli.Command{start, stop}
}

// Start the unison sync process
func (cmd *ProjectSync) RunStart(ctx *cli.Context) error {
	config := NewProjectConfig()
	volumeName := cmd.GetVolumeName(ctx, config)
	cmd.out.Verbose.Printf("Starting sync with volume: %s", volumeName)

	// Ensure the processes can handle a large number of watches
	if err := cmd.machine.SetSysctl("fs.inotify.max_user_watches", MAX_WATCHES); err != nil {
		cmd.out.Error.Fatalf("Error configuring file watches on Docker Machine: %v", err)
	}

	cmd.out.Info.Printf("Starting sync volume: %s", volumeName)
	exec.Command("docker", "volume", "create", volumeName).Run()

	cmd.out.Info.Println("Starting unison container")
	unisonMinorVersion := cmd.GetUnisonMinorVersion()
	cmd.out.Verbose.Printf("Local unison version for compatibilty: %s", unisonMinorVersion)
	exec.Command("docker", "container", "stop", volumeName).Run()
	err := exec.Command("docker", "container", "run", "--detach", "--rm",
		"-v", fmt.Sprintf("%s:/unison", volumeName),
		"-e", "UNISON_DIR=/unison",
		"-l", fmt.Sprintf("com.dnsdock.name=%s", volumeName),
		"-l", "com.dnsdock.image=volume.outrigger",
		"--name", volumeName,
		fmt.Sprintf("outrigger/unison:%s", unisonMinorVersion),
	).Run()
	if err != nil {
		cmd.out.Error.Fatalf("Error starting sync container %s: %v", volumeName, err)
	}

	var ip = cmd.WaitForUnisonContainer(volumeName, ctx.Int("initial-sync-timeout"))

	cmd.out.Info.Println("Initializing sync")

	// Remove the log file, the existence of the log file will mean that sync is up and running
	var logFile = fmt.Sprintf("%s.log", volumeName)
	exec.Command("rm", "-f", logFile).Run()

	unisonArgs := []string{
		".",
		fmt.Sprintf("socket://%s:%d/", ip, UNISON_PORT),
		"-auto", "-batch", "-silent", "-contactquietly",
		"-repeat", "watch",
		"-prefer", ".",
		"-logfile", logFile,
		"-ignore", fmt.Sprintf("Name %s", logFile),
	}
	// Append ProjectConfig ignores here
	if config.Sync != nil {
		for _, ignore := range config.Sync.Ignore {
			unisonArgs = append(unisonArgs, "-ignore", ignore)
		}
	}

	cmd.out.Verbose.Printf("Unison Args: %s", strings.Join(unisonArgs[:], " "))
	if err = exec.Command("unison", unisonArgs...).Start(); err != nil {
		cmd.out.Error.Fatalf("Error starting local unison process: %v", err)
	}

	cmd.WaitForSyncInit(logFile, ctx.Int("initial-sync-timeout"), ctx.Int("initial-sync-wait"))

	return nil
}

// Start the unison sync process
func (cmd *ProjectSync) RunStop(ctx *cli.Context) error {
	config := NewProjectConfig()
	volumeName := cmd.GetVolumeName(ctx, config)
	cmd.out.Verbose.Printf("Stopping sync with volume: %s", volumeName)

	cmd.out.Info.Println("Stopping unison container")
	exec.Command("docker", "container", "stop", volumeName).Run()

	return nil
}

// Find the volume name through a variety of fall backs
func (cmd *ProjectSync) GetVolumeName(ctx *cli.Context, config *ProjectConfig) string {
	// 1. Check for argument
	if ctx.Args().Present() {
		return ctx.Args().First()
	}

	// 2. Check for config
	if config.Sync != nil && config.Sync.Volume != "" {
		return config.Sync.Volume
	}

	// 3. Parse compose file looking for an external volume named *-sync
	if composeConfig, err := cmd.LoadComposeFile(); err == nil {
		for name, volume := range composeConfig.Volumes {
			if strings.HasSuffix(name, "-sync") && volume.External {
				return name
			}
		}
	}

	// 4. Use local dir for the volume name
	if dir, err := os.Getwd(); err == nil {
		var _, folder = path.Split(dir)
		return fmt.Sprintf("%s-sync", folder)
	} else {
		cmd.out.Error.Println(err)
	}

	cmd.out.Error.Fatal("Unable to determine a name for the sync volume")
	return ""
}

// Load the proper compose file
func (cmd *ProjectSync) LoadComposeFile() (*ComposeFile, error) {
	yamlFile, err := ioutil.ReadFile("./docker-compose.yml")

	if err == nil {
		var config ComposeFile
		if err := yaml.Unmarshal(yamlFile, &config); err != nil {
			cmd.out.Error.Fatalf("YAML Parsing Error: %s", err)
		}
		return &config, nil
	}

	return nil, err
}

// Wait for the unison container port to allow connections
// Due to the fact that we don't compile with -cgo (so we can build using Docker),
// we need to discover the IP address of the container instead of using the DNS name
// when compiled without -cgo this executable will not use the native mac dns resolution
// which is how we have configured dnsdock to provide names for containers.
func (cmd *ProjectSync) WaitForUnisonContainer(containerName string, timeoutSeconds int) string {
	cmd.out.Info.Println("Waiting for container to start")

	var timeoutLoopSleep = time.Duration(100) * time.Millisecond
	// * 10 here because we loop once every 100 ms and we want to get to seconds
	var timeoutLoops = timeoutSeconds * 10

	output, err := exec.Command("docker", "inspect", "--format", "{{.NetworkSettings.IPAddress}}", containerName).Output()
	if err != nil {
		cmd.out.Error.Fatalf("Error inspecting sync container %s: %v", containerName, err)
	}
	ip := strings.Trim(string(output), "\n")

	cmd.out.Verbose.Printf("Checking for unison network connection on %s %d", ip, UNISON_PORT)
	for i := 1; i <= timeoutLoops; i++ {
		if conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", ip, UNISON_PORT)); err == nil {
			conn.Close()
			return ip
		} else {
			cmd.out.Info.Printf("Error: %v", err)
			time.Sleep(timeoutLoopSleep)
		}
	}
	cmd.out.Error.Fatal("Sync container failed to start!")
	return ""
}

// The local unison process is finished initializing when the log file exists
// and has stopped growing in size
func (cmd *ProjectSync) WaitForSyncInit(logFile string, timeoutSeconds int, syncWaitSeconds int) {
	cmd.out.Info.Print("Waiting for initial sync detection")

	var tempFile = fmt.Sprintf(".%s.tmp", logFile)
	var timeoutLoopSleep = time.Duration(100) * time.Millisecond
	// * 10 here because we loop once every 100 ms and we want to get to seconds
	var timeoutLoops = timeoutSeconds * 10

	// Create a temp file to cause a sync action
	exec.Command("touch", tempFile).Run()

	for i := 1; i <= timeoutLoops; i++ {
		if i%10 == 0 {
			os.Stdout.WriteString(".")
		}
		if statInfo, err := os.Stat(logFile); err == nil {
			os.Stdout.WriteString(" initial sync detected\n")
			// Remove the temp file now that we are running
			exec.Command("rm", "-f", tempFile).Run()

			cmd.out.Info.Print("Waiting for initial sync to finish")
			// Arbitrary sleep length but anything less than 3 wasn't catching
			// ongoing very quick file updates during a test
			var statSleep = time.Duration(syncWaitSeconds) * time.Second
			// Initialize at -2 to force at least one loop
			var lastSize = int64(-2)
			for lastSize != statInfo.Size() {
				os.Stdout.WriteString(".")
				time.Sleep(statSleep)
				lastSize = statInfo.Size()
				if statInfo, err = os.Stat(logFile); err != nil {
					cmd.out.Info.Print(err.Error())
					lastSize = -1
				}
			}
			os.Stdout.WriteString(" done\n")
			return
		} else {
			time.Sleep(timeoutLoopSleep)
		}
	}

	// The log file was not created, the sync has not started yet
	exec.Command("rm", "-f", tempFile).Run()
	cmd.out.Error.Fatal("Failed to detect start of initial sync! Check sync log file.")
}

// Get the local Unison version to try to load a compatible unison image
// This function discovers a semver like 2.48.4 and return 2.48
func (cmd *ProjectSync) GetUnisonMinorVersion() string {
	output, _ := exec.Command("unison", "-version").Output()
	re := regexp.MustCompile("unison version (\\d+\\.\\d+\\.\\d+)")
	rawVersion := re.FindAllStringSubmatch(string(output), -1)[0][1]
	v := version.Must(version.NewVersion(rawVersion))
	segments := v.Segments()
	return fmt.Sprintf("%d.%d", segments[0], segments[1])
}
<commit_msg>Tweak error message as it should only occur when there is no log file.<commit_after>package commands

import (
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path"
	"regexp"
	"strings"
	"time"

	"github.com/hashicorp/go-version"
	"github.com/urfave/cli"
	"gopkg.in/yaml.v2"
)

type ProjectSync struct {
	BaseCommand
}

// Minimal compose file struct to discover volumes
type ComposeFile struct {
	Volumes map[string]Volume
}

type Volume struct {
	External bool
}

const UNISON_PORT = 5000
const MAX_WATCHES = "100000"

func (cmd *ProjectSync) Commands() []cli.Command {
	start := cli.Command{
		Name:        "sync:start",
		Aliases:     []string{"sync"},
		Usage:       "Start a unison sync on local project directory. Optionally provide a volume name.",
		ArgsUsage:   "[optional volume name]",
		Description: "Volume name will be discovered in the following order: argument to this command > outrigger project config > docker-compose file > current directory name",
		Flags: []cli.Flag{
			cli.IntFlag{
				Name:   "initial-sync-timeout",
				Value:  60,
				Usage:  "Maximum amount of time in seconds to allow for detecting each of start of the unison container and start of initial sync",
				EnvVar: "RIG_PROJECT_SYNC_TIMEOUT",
			},
			cli.IntFlag{
				Name:   "initial-sync-wait",
				Value:  5,
				Usage:  "Time in seconds to wait between checks to see if initial sync has finished.",
				EnvVar: "RIG_PROJECT_INITIAL_SYNC_WAIT",
			},
		},
		Before: cmd.Before,
		Action: cmd.RunStart,
	}
	stop := cli.Command{
		Name:        "sync:stop",
		Usage:       "Stops a unison sync on local project directory. Optionally provide a volume name.",
		ArgsUsage:   "[optional volume name]",
		Description: "Volume name will be discovered in the following order: argument to this command > outrigger project config > docker-compose file > current directory name",
		Before:      cmd.Before,
		Action:      cmd.RunStop,
	}

	return []cli.Command{start, stop}
}

// Start the unison sync process
func (cmd *ProjectSync) RunStart(ctx *cli.Context) error {
	config := NewProjectConfig()
	volumeName := cmd.GetVolumeName(ctx, config)
	cmd.out.Verbose.Printf("Starting sync with volume: %s", volumeName)

	// Ensure the processes can handle a large number of watches
	if err := cmd.machine.SetSysctl("fs.inotify.max_user_watches", MAX_WATCHES); err != nil {
		cmd.out.Error.Fatalf("Error configuring file watches on Docker Machine: %v", err)
	}

	cmd.out.Info.Printf("Starting sync volume: %s", volumeName)
	exec.Command("docker", "volume", "create", volumeName).Run()

	cmd.out.Info.Println("Starting unison container")
	unisonMinorVersion := cmd.GetUnisonMinorVersion()
	cmd.out.Verbose.Printf("Local unison version for compatibilty: %s", unisonMinorVersion)
	exec.Command("docker", "container", "stop", volumeName).Run()
	err := exec.Command("docker", "container", "run", "--detach", "--rm",
		"-v", fmt.Sprintf("%s:/unison", volumeName),
		"-e", "UNISON_DIR=/unison",
		"-l", fmt.Sprintf("com.dnsdock.name=%s", volumeName),
		"-l", "com.dnsdock.image=volume.outrigger",
		"--name", volumeName,
		fmt.Sprintf("outrigger/unison:%s", unisonMinorVersion),
	).Run()
	if err != nil {
		cmd.out.Error.Fatalf("Error starting sync container %s: %v", volumeName, err)
	}

	var ip = cmd.WaitForUnisonContainer(volumeName, ctx.Int("initial-sync-timeout"))

	cmd.out.Info.Println("Initializing sync")

	// Remove the log file, the existence of the log file will mean that sync is up and running
	var logFile = fmt.Sprintf("%s.log", volumeName)
	exec.Command("rm", "-f", logFile).Run()

	unisonArgs := []string{
		".",
		fmt.Sprintf("socket://%s:%d/", ip, UNISON_PORT),
		"-auto", "-batch", "-silent", "-contactquietly",
		"-repeat", "watch",
		"-prefer", ".",
		"-logfile", logFile,
		"-ignore", fmt.Sprintf("Name %s", logFile),
	}
	// Append ProjectConfig ignores here
	if config.Sync != nil {
		for _, ignore := range config.Sync.Ignore {
			unisonArgs = append(unisonArgs, "-ignore", ignore)
		}
	}

	cmd.out.Verbose.Printf("Unison Args: %s", strings.Join(unisonArgs[:], " "))
	if err = exec.Command("unison", unisonArgs...).Start(); err != nil {
		cmd.out.Error.Fatalf("Error starting local unison process: %v", err)
	}

	cmd.WaitForSyncInit(logFile, ctx.Int("initial-sync-timeout"), ctx.Int("initial-sync-wait"))

	return nil
}

// Start the unison sync process
func (cmd *ProjectSync) RunStop(ctx *cli.Context) error {
	config := NewProjectConfig()
	volumeName := cmd.GetVolumeName(ctx, config)
	cmd.out.Verbose.Printf("Stopping sync with volume: %s", volumeName)

	cmd.out.Info.Println("Stopping unison container")
	exec.Command("docker", "container", "stop", volumeName).Run()

	return nil
}

// Find the volume name through a variety of fall backs
func (cmd *ProjectSync) GetVolumeName(ctx *cli.Context, config *ProjectConfig) string {
	// 1. Check for argument
	if ctx.Args().Present() {
		return ctx.Args().First()
	}

	// 2. Check for config
	if config.Sync != nil && config.Sync.Volume != "" {
		return config.Sync.Volume
	}

	// 3. Parse compose file looking for an external volume named *-sync
	if composeConfig, err := cmd.LoadComposeFile(); err == nil {
		for name, volume := range composeConfig.Volumes {
			if strings.HasSuffix(name, "-sync") && volume.External {
				return name
			}
		}
	}

	// 4. Use local dir for the volume name
	if dir, err := os.Getwd(); err == nil {
		var _, folder = path.Split(dir)
		return fmt.Sprintf("%s-sync", folder)
	} else {
		cmd.out.Error.Println(err)
	}

	cmd.out.Error.Fatal("Unable to determine a name for the sync volume")
	return ""
}

// Load the proper compose file
func (cmd *ProjectSync) LoadComposeFile() (*ComposeFile, error) {
	yamlFile, err := ioutil.ReadFile("./docker-compose.yml")

	if err == nil {
		var config ComposeFile
		if err := yaml.Unmarshal(yamlFile, &config); err != nil {
			cmd.out.Error.Fatalf("YAML Parsing Error: %s", err)
		}
		return &config, nil
	}

	return nil, err
}

// Wait for the unison container port to allow connections
// Due to the fact that we don't compile with -cgo (so we can build using Docker),
// we need to discover the IP address of the container instead of using the DNS name
// when compiled without -cgo this executable will not use the native mac dns resolution
// which is how we have configured dnsdock to provide names for containers.
func (cmd *ProjectSync) WaitForUnisonContainer(containerName string, timeoutSeconds int) string {
	cmd.out.Info.Println("Waiting for container to start")

	var timeoutLoopSleep = time.Duration(100) * time.Millisecond
	// * 10 here because we loop once every 100 ms and we want to get to seconds
	var timeoutLoops = timeoutSeconds * 10

	output, err := exec.Command("docker", "inspect", "--format", "{{.NetworkSettings.IPAddress}}", containerName).Output()
	if err != nil {
		cmd.out.Error.Fatalf("Error inspecting sync container %s: %v", containerName, err)
	}
	ip := strings.Trim(string(output), "\n")

	cmd.out.Verbose.Printf("Checking for unison network connection on %s %d", ip, UNISON_PORT)
	for i := 1; i <= timeoutLoops; i++ {
		if conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", ip, UNISON_PORT)); err == nil {
			conn.Close()
			return ip
		} else {
			cmd.out.Info.Printf("Error: %v", err)
			time.Sleep(timeoutLoopSleep)
		}
	}
	cmd.out.Error.Fatal("Sync container failed to start!")
	return ""
}

// The local unison process is finished initializing when the log file exists
// and has stopped growing in size
func (cmd *ProjectSync) WaitForSyncInit(logFile string, timeoutSeconds int, syncWaitSeconds int) {
	cmd.out.Info.Print("Waiting for initial sync detection")

	var tempFile = fmt.Sprintf(".%s.tmp", logFile)
	var timeoutLoopSleep = time.Duration(100) * time.Millisecond
	// * 10 here because we loop once every 100 ms and we want to get to seconds
	var timeoutLoops = timeoutSeconds * 10

	// Create a temp file to cause a sync action
	exec.Command("touch", tempFile).Run()

	for i := 1; i <= timeoutLoops; i++ {
		if i%10 == 0 {
			os.Stdout.WriteString(".")
		}
		if statInfo, err := os.Stat(logFile); err == nil {
			os.Stdout.WriteString(" initial sync detected\n")
			// Remove the temp file now that we are running
			exec.Command("rm", "-f", tempFile).Run()

			cmd.out.Info.Print("Waiting for initial sync to finish")
			// Arbitrary sleep length but anything less than 3 wasn't catching
			// ongoing very quick file updates during a test
			var statSleep = time.Duration(syncWaitSeconds) * time.Second
			// Initialize at -2 to force at least one loop
			var lastSize = int64(-2)
			for lastSize != statInfo.Size() {
				os.Stdout.WriteString(".")
				time.Sleep(statSleep)
				lastSize = statInfo.Size()
				if statInfo, err = os.Stat(logFile); err != nil {
					cmd.out.Info.Print(err.Error())
					lastSize = -1
				}
			}
			os.Stdout.WriteString(" done\n")
			return
		} else {
			time.Sleep(timeoutLoopSleep)
		}
	}

	// The log file was not created, the sync has not started yet
	exec.Command("rm", "-f", tempFile).Run()
	cmd.out.Error.Fatal("Failed to detect start of initial sync!")
}

// Get the local Unison version to try to load a compatible unison image
// This function discovers a semver like 2.48.4 and return 2.48
func (cmd *ProjectSync) GetUnisonMinorVersion() string {
	output, _ := exec.Command("unison", "-version").Output()
	re := regexp.MustCompile("unison version (\\d+\\.\\d+\\.\\d+)")
	rawVersion := re.FindAllStringSubmatch(string(output), -1)[0][1]
	v := version.Must(version.NewVersion(rawVersion))
	segments := v.Segments()
	return fmt.Sprintf("%d.%d", segments[0], segments[1])
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"runtime"
	"sync"

	jsonw "github.com/keybase/go-jsonw"
)

type jsonFileTransaction struct {
	f       *JSONFile
	tmpname string
}

var _ ConfigWriterTransacter = (*jsonFileTransaction)(nil)

type JSONFile struct {
	Contextified
	filename string
	which    string
	jw       *jsonw.Wrapper
	exists   bool

	txMutex sync.Mutex
	tx      *jsonFileTransaction
}

func NewJSONFile(g *GlobalContext, filename, which string) *JSONFile {
	return &JSONFile{
		filename:     filename,
		which:        which,
		jw:           jsonw.NewDictionary(),
		Contextified: NewContextified(g),
	}
}

func (f *JSONFile) GetWrapper() *jsonw.Wrapper {
	return f.jw
}
func (f *JSONFile) Exists() bool { return f.exists }

func (f *JSONFile) Load(warnOnNotFound bool) error {
	f.G().Log.Debug("+ loading %s file: %s", f.which, f.filename)
	file, err := os.Open(f.filename)
	if err != nil {
		if os.IsNotExist(err) {
			msg := fmt.Sprintf("No %s file found; tried %s", f.which, f.filename)
			if warnOnNotFound {
				f.G().Log.Warning(msg)
			} else {
				f.G().Log.Debug(msg)
			}
			return nil
		}

		if os.IsPermission(err) {
			f.G().Log.Warning("Permission denied opening %s file %s", f.which, f.filename)
			return nil
		}

		return err
	}
	f.exists = true
	defer file.Close()
	decoder := json.NewDecoder(file)
	obj := make(map[string]interface{})
	// Treat empty files like an empty dictionary
	if err = decoder.Decode(&obj); err != nil && err != io.EOF {
		f.G().Log.Errorf("Error decoding %s file %s", f.which, f.filename)
		return err
	}
	f.jw = jsonw.NewWrapper(obj)
	f.G().Log.Debug("- successfully loaded %s file", f.which)
	return nil
}

func (f *JSONFile) Nuke() error {
	f.G().Log.Debug("+ nuke file %s", f.filename)
	err := os.Remove(f.filename)
	f.G().Log.Debug("- nuke file %s -> %s", f.filename, ErrToOk(err))
	return err
}

func (f *JSONFile) BeginTransaction() (ConfigWriterTransacter, error) {
	tx, err := newJSONFileTransaction(f)
	if err != nil {
		return nil, err
	}
	err = f.setTx(tx)
	if err != nil {
		return nil, err
	}
	return tx, nil
}

func (f *JSONFile) setTx(tx *jsonFileTransaction) error {
	f.txMutex.Lock()
	defer f.txMutex.Unlock()
	if f.tx != nil && tx != nil {
		return fmt.Errorf("Provision transaction already in progress")
	}
	f.tx = tx
	return nil
}

func (f *JSONFile) getOrMakeTx() (*jsonFileTransaction, bool, error) {
	f.txMutex.Lock()
	defer f.txMutex.Unlock()

	// if a transaction exists, use it
	if f.tx != nil {
		return f.tx, false, nil
	}

	// make a new transaction
	tx, err := newJSONFileTransaction(f)
	if err != nil {
		return nil, false, err
	}

	f.tx = tx

	// return true so caller knows that a transaction was created
	return f.tx, true, nil
}

func newJSONFileTransaction(f *JSONFile) (*jsonFileTransaction, error) {
	ret := &jsonFileTransaction{f: f}
	sffx, err := RandString("", 15)
	if err != nil {
		return nil, err
	}
	ret.tmpname = f.filename + "." + sffx
	return ret, nil
}

func (f *JSONFile) Save() error {
	tx, txCreated, err := f.getOrMakeTx()
	if err != nil {
		return err
	}
	if txCreated {
		// if Save() created a transaction, then abort it if it
		// still exists on exit
		defer func() {
			if tx != nil {
				tx.Abort()
			}
		}()
	}

	if err := f.save(); err != nil {
		return err
	}

	if txCreated {
		// this Save() call created a transaction, so commit it
		if err := tx.Commit(); err != nil {
			return err
		}

		// Commit worked, clear the transaction so defer() doesn't
		// abort it.
		tx = nil
	}

	return nil
}

func (f *JSONFile) save() (err error) {
	if f.tx == nil {
		return errors.New("save() called with nil transaction")
	}
	filename := f.tx.tmpname
	f.G().Log.Debug("+ saving %s file %s", f.which, filename)

	err = MakeParentDirs(filename)
	if err != nil {
		f.G().Log.Errorf("Failed to make parent dirs for %s", filename)
		return err
	}

	var dat interface{}

	if f.jw == nil {
		// Make a default Dictionary if none already exists
		dat = make(map[string]interface{})
		f.G().Log.Warning("No value for %s file; assuming empty value (i.e., {})",
			f.which)
	} else {
		dat, err = f.jw.GetData()
		if err != nil {
			f.G().Log.Errorf("Failed to encode data for %s file", f.which)
			return err
		}
	}
	var writer *os.File
	flags := (os.O_WRONLY | os.O_CREATE | os.O_TRUNC)
	writer, err = os.OpenFile(filename, flags, PermFile)
	if err != nil {
		f.G().Log.Errorf("Failed to open %s file %s for writing: %s",
			f.which, filename, err)
		return err
	}
	defer writer.Close()

	encoded, err := json.MarshalIndent(dat, "", "    ")
	if err != nil {
		f.G().Log.Errorf("Error marshaling data to %s file %s: %s", f.which, filename, err)
		return err
	}

	n, err := writer.Write(encoded)
	if err != nil {
		f.G().Log.Errorf("Error writing encoded data to %s file %s: %s", f.which, filename, err)
		return err
	}
	if n != len(encoded) {
		f.G().Log.Errorf("Error writing encoded data to %s file %s: wrote %d bytes, expected %d", f.which, filename, n, len(encoded))
		return io.ErrShortWrite
	}

	err = writer.Sync()
	if err != nil {
		f.G().Log.Errorf("Error syncing %s file %s: %s", f.which, filename, err)
		return err
	}

	err = writer.Close()
	if err != nil {
		f.G().Log.Errorf("Error closing %s file %s: %s", f.which, filename, err)
		return err
	}

	f.G().Log.Debug("- saved %s file %s", f.which, filename)

	if runtime.GOOS == "android" {
		f.G().Log.Debug("| Android extra-checking in JSONFile.save")
		info, err := os.Stat(filename)
		if err != nil {
			f.G().Log.Errorf("| Error os.Stat(%s): %s", filename, err)
			return err
		}
		f.G().Log.Debug("| File info: name = %s", info.Name())
		f.G().Log.Debug("| File info: size = %d", info.Size())
		f.G().Log.Debug("| File info: mode = %s", info.Mode())
		f.G().Log.Debug("| File info: mod time = %s", info.ModTime())

		f.G().Log.Debug("| Android extra-checking done")
	}

	return nil
}

func (f *jsonFileTransaction) Abort() error {
	f.f.G().Log.Debug("+ Aborting %s rewrite %s", f.f.which, f.tmpname)
	err := os.Remove(f.tmpname)
	f.f.setTx(nil)
	f.f.G().Log.Debug("- Abort -> %s\n", ErrToOk(err))
	return err
}

func (f *jsonFileTransaction) Commit() (err error) {
	f.f.G().Log.Debug("+ Commit %s rewrite %s", f.f.which, f.tmpname)
	defer func() { f.f.G().Log.Debug("- Commit %s rewrite %s", f.f.which, ErrToOk(err)) }()

	f.f.G().Log.Debug("| Commit: making parent directories for %q", f.f.filename)
	if err = MakeParentDirs(f.f.filename); err != nil {
		return err
	}
	f.f.G().Log.Debug("| Commit : renaming %q => %q", f.tmpname, f.f.filename)
	err = renameFile(f.f.G(), f.tmpname, f.f.filename)
	if err != nil {
		f.f.G().Log.Debug("| Commit: rename %q => %q error: %s", f.tmpname, f.f.filename, err)
	}
	f.f.setTx(nil)

	return err
}
<commit_msg>extra logging<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"runtime"
	"sync"

	jsonw "github.com/keybase/go-jsonw"
)

type jsonFileTransaction struct {
	f       *JSONFile
	tmpname string
}

var _ ConfigWriterTransacter = (*jsonFileTransaction)(nil)

type JSONFile struct {
	Contextified
	filename string
	which    string
	jw       *jsonw.Wrapper
	exists   bool

	txMutex sync.Mutex
	tx      *jsonFileTransaction
}

func NewJSONFile(g *GlobalContext, filename, which string) *JSONFile {
	return &JSONFile{
		filename:     filename,
		which:        which,
		jw:           jsonw.NewDictionary(),
		Contextified: NewContextified(g),
	}
}

func (f *JSONFile) GetWrapper() *jsonw.Wrapper {
	return f.jw
}
func (f *JSONFile) Exists() bool { return f.exists }

func (f *JSONFile) Load(warnOnNotFound bool) error {
	f.G().Log.Debug("+ loading %s file: %s", f.which, f.filename)
	file, err := os.Open(f.filename)
	if err != nil {
		if os.IsNotExist(err) {
			msg := fmt.Sprintf("No %s file found; tried %s", f.which, f.filename)
			if warnOnNotFound {
				f.G().Log.Warning(msg)
			} else {
				f.G().Log.Debug(msg)
			}
			return nil
		}

		if os.IsPermission(err) {
			f.G().Log.Warning("Permission denied opening %s file %s", f.which, f.filename)
			return nil
		}

		return err
	}
	f.exists = true
	defer file.Close()
	decoder := json.NewDecoder(file)
	obj := make(map[string]interface{})
	// Treat empty files like an empty dictionary
	if err = decoder.Decode(&obj); err != nil && err != io.EOF {
		f.G().Log.Errorf("Error decoding %s file %s", f.which, f.filename)
		return err
	}
	f.jw = jsonw.NewWrapper(obj)
	f.G().Log.Debug("- successfully loaded %s file", f.which)
	return nil
}

func (f *JSONFile) Nuke() error {
	f.G().Log.Debug("+ nuke file %s", f.filename)
	err := os.Remove(f.filename)
	f.G().Log.Debug("- nuke file %s -> %s", f.filename, ErrToOk(err))
	return err
}

func (f *JSONFile) BeginTransaction() (ConfigWriterTransacter, error) {
	tx, err := newJSONFileTransaction(f)
	if err != nil {
		return nil, err
	}
	err = f.setTx(tx)
	if err != nil {
		return nil, err
	}
	return tx, nil
}

func (f *JSONFile) setTx(tx *jsonFileTransaction) error {
	f.txMutex.Lock()
	defer f.txMutex.Unlock()
	if f.tx != nil && tx != nil {
		return fmt.Errorf("Provision transaction already in progress")
	}
	f.tx = tx
	return nil
}

func (f *JSONFile) getOrMakeTx() (*jsonFileTransaction, bool, error) {
	f.txMutex.Lock()
	defer f.txMutex.Unlock()

	// if a transaction exists, use it
	if f.tx != nil {
		return f.tx, false, nil
	}

	// make a new transaction
	tx, err := newJSONFileTransaction(f)
	if err != nil {
		return nil, false, err
	}

	f.tx = tx

	// return true so caller knows that a transaction was created
	return f.tx, true, nil
}

func newJSONFileTransaction(f *JSONFile) (*jsonFileTransaction, error) {
	ret := &jsonFileTransaction{f: f}
	sffx, err := RandString("", 15)
	if err != nil {
		return nil, err
	}
	ret.tmpname = f.filename + "." + sffx
	return ret, nil
}

func (f *JSONFile) Save() error {
	tx, txCreated, err := f.getOrMakeTx()
	if err != nil {
		return err
	}
	if txCreated {
		// if Save() created a transaction, then abort it if it
		// still exists on exit
		defer func() {
			if tx != nil {
				tx.Abort()
			}
		}()
	}

	if err := f.save(); err != nil {
		return err
	}

	if txCreated {
		// this Save() call created a transaction, so commit it
		if err := tx.Commit(); err != nil {
			return err
		}

		// Commit worked, clear the transaction so defer() doesn't
		// abort it.
		tx = nil
	}

	return nil
}

func (f *JSONFile) save() (err error) {
	if f.tx == nil {
		return errors.New("save() called with nil transaction")
	}
	filename := f.tx.tmpname
	f.G().Log.Debug("+ saving %s file %s", f.which, filename)

	err = MakeParentDirs(filename)
	if err != nil {
		f.G().Log.Errorf("Failed to make parent dirs for %s", filename)
		return err
	}

	var dat interface{}

	if f.jw == nil {
		// Make a default Dictionary if none already exists
		dat = make(map[string]interface{})
		f.G().Log.Warning("No value for %s file; assuming empty value (i.e., {})",
			f.which)
	} else {
		dat, err = f.jw.GetData()
		if err != nil {
			f.G().Log.Errorf("Failed to encode data for %s file", f.which)
			return err
		}
	}
	var writer *os.File
	flags := (os.O_WRONLY | os.O_CREATE | os.O_TRUNC)
	writer, err = os.OpenFile(filename, flags, PermFile)
	if err != nil {
		f.G().Log.Errorf("Failed to open %s file %s for writing: %s",
			f.which, filename, err)
		return err
	}
	defer writer.Close()

	encoded, err := json.MarshalIndent(dat, "", "    ")
	if err != nil {
		f.G().Log.Errorf("Error marshaling data to %s file %s: %s", f.which, filename, err)
		return err
	}

	n, err := writer.Write(encoded)
	if err != nil {
		f.G().Log.Errorf("Error writing encoded data to %s file %s: %s", f.which, filename, err)
		return err
	}
	if n != len(encoded) {
		f.G().Log.Errorf("Error writing encoded data to %s file %s: wrote %d bytes, expected %d", f.which, filename, n, len(encoded))
		return io.ErrShortWrite
	}

	err = writer.Sync()
	if err != nil {
		f.G().Log.Errorf("Error syncing %s file %s: %s", f.which, filename, err)
		return err
	}

	err = writer.Close()
	if err != nil {
		f.G().Log.Errorf("Error closing %s file %s: %s", f.which, filename, err)
		return err
	}

	f.G().Log.Debug("- saved %s file %s", f.which, filename)

	if runtime.GOOS == "android" {
		f.G().Log.Debug("| Android extra-checking in JSONFile.save")
		info, err := os.Stat(filename)
		if err != nil {
			f.G().Log.Errorf("| Error os.Stat(%s): %s", filename, err)
			return err
		}
		f.G().Log.Debug("| File info: name = %s", info.Name())
		f.G().Log.Debug("| File info: size = %d", info.Size())
		f.G().Log.Debug("| File info: mode = %s", info.Mode())
		f.G().Log.Debug("| File info: mod time = %s", info.ModTime())

		if info.Size() != int64(len(encoded)) {
			f.G().Log.Errorf("| File info size (%d) does not match encoded len (%d)", info.Size(), len(encoded))
			return fmt.Errorf("file info size (%d) does not match encoded len (%d)", info.Size(), len(encoded))
		}

		f.G().Log.Debug("| Android extra-checking done")
	}

	return nil
}

func (f *jsonFileTransaction) Abort() error {
	f.f.G().Log.Debug("+ Aborting %s rewrite %s", f.f.which, f.tmpname)
	err := os.Remove(f.tmpname)
	f.f.setTx(nil)
	f.f.G().Log.Debug("- Abort -> %s\n", ErrToOk(err))
	return err
}

func (f *jsonFileTransaction) Commit() (err error) {
	f.f.G().Log.Debug("+ Commit %s rewrite %s", f.f.which, f.tmpname)
	defer func() { f.f.G().Log.Debug("- Commit %s rewrite %s", f.f.which, ErrToOk(err)) }()

	f.f.G().Log.Debug("| Commit: making parent directories for %q", f.f.filename)
	if err = MakeParentDirs(f.f.filename); err != nil {
		return err
	}
	f.f.G().Log.Debug("| Commit : renaming %q => %q", f.tmpname, f.f.filename)
	err = renameFile(f.f.G(), f.tmpname, f.f.filename)
	if err != nil {
		f.f.G().Log.Debug("| Commit: rename %q => %q error: %s", f.tmpname, f.f.filename, err)
	}
	f.f.setTx(nil)

	return err
}
<|endoftext|>


<commit_before>package loader

import (
	"bytes"
	"io"

	"github.com/lunixbochs/usercorn/go/models"
)

var cgcMagic = []byte{0x7f, 0x43, 0x47, 0x43}
var elfMagicReader = bytes.NewReader(elfMagic)

func MatchCgc(r io.ReaderAt) bool {
	return bytes.Equal(getMagic(r), cgcMagic)
}

type FakeCgcReader struct {
	io.ReaderAt
}

func (f *FakeCgcReader) ReadAt(p []byte, off int64) (int, error) {
	n := 0
	if off < 4 {
		n, _ = elfMagicReader.ReadAt(p, off)
		if n == len(p) {
			return n, nil
		}
		p = p[n:]
		off = 4
	}
	n1, err := f.ReaderAt.ReadAt(p, off)
	return n1 + n, err
}

type CgcLoader struct {
	models.Loader
}

func (c *CgcLoader) OS() string {
	return "cgc"
}

func NewCgcLoader(r io.ReaderAt, arch string) (models.Loader, error) {
	l, err := NewElfLoader(&FakeCgcReader{r}, arch)
	return &CgcLoader{l}, err
}
<commit_msg>only pretend to be an ELF for the first CGC read<commit_after>package loader

import (
	"bytes"
	"io"

	"github.com/lunixbochs/usercorn/go/models"
)

var cgcMagic = []byte{0x7f, 0x43, 0x47, 0x43}
var elfMagicReader = bytes.NewReader(elfMagic)

func MatchCgc(r io.ReaderAt) bool {
	return bytes.Equal(getMagic(r), cgcMagic)
}

type FakeCgcReader struct {
	io.ReaderAt
	first bool
}

func (f *FakeCgcReader) ReadAt(p []byte, off int64) (int, error) {
	n := 0
	if off < 4 && f.first {
		f.first = false
		n, _ = elfMagicReader.ReadAt(p, off)
		if n == len(p) {
			return n, nil
		}
		p = p[n:]
		off = 4
	}
	n1, err := f.ReaderAt.ReadAt(p, off)
	return n1 + n, err
}

type CgcLoader struct {
	models.Loader
}

func (c *CgcLoader) OS() string {
	return "cgc"
}

func NewCgcLoader(r io.ReaderAt, arch string) (models.Loader, error) {
	l, err := NewElfLoader(&FakeCgcReader{r, true}, arch)
	return &CgcLoader{l}, err
}
<|endoftext|>


<commit_before>package gode

import (
	"bytes"
	"encoding/json"
	"errors"
	"os"
	"os/exec"
	"path/filepath"
)

// Package represents an npm package.
type Package struct {
	Name    string `json:"name"`
	Version string `json:"version"`
}

// Packages returns a list of npm packages installed.
func Packages() ([]Package, error) {
	stdout, stderr, err := execNpm("list", "--json", "--depth=0")
	if err != nil {
		return nil, errors.New(stderr)
	}
	var response map[string]map[string]Package
	if err := json.Unmarshal([]byte(stdout), &response); err != nil {
		return nil, errors.New(stderr)
	}
	packages := make([]Package, 0, len(response["dependencies"]))
	for name, p := range response["dependencies"] {
		p.Name = name
		packages = append(packages, p)
	}
	return packages, nil
}

// InstallPackages installs a npm packages.
func InstallPackages(packages ...string) error {
	args := append([]string{"install"}, packages...)
	_, stderr, err := execNpm(args...)
	if err != nil {
		return errors.New("Error installing package. \n" + stderr + "\nTry running again with GODE_DEBUG=info to see more output.")
	}
	return nil
}

// RemovePackages removes a npm packages.
func RemovePackages(packages ...string) error {
	args := append([]string{"remove"}, packages...)
	_, stderr, err := execNpm(args...)
	if err != nil {
		return errors.New(stderr)
	}
	return nil
}

// OutdatedPackages returns a map of packages and their latest version
func OutdatedPackages(names ...string) (map[string]string, error) {
	args := append([]string{"outdated", "--json"}, names...)
	stdout, stderr, err := execNpm(args...)
	if err != nil {
		return nil, errors.New(stderr)
	}
	var outdated map[string]struct{ Latest string }
	json.Unmarshal([]byte(stdout), &outdated)
	packages := make(map[string]string, len(outdated))
	for name, versions := range outdated {
		packages[name] = versions.Latest
	}
	return packages, nil
}

// ClearCache clears the npm cache
func ClearCache() error {
	cmd, err := npmCmd("cache", "clean")
	if err != nil {
		return err
	}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func npmCmd(args ...string) (*exec.Cmd, error) {
	if err := os.MkdirAll(filepath.Join(rootPath, "node_modules"), 0755); err != nil {
		return nil, err
	}
	nodePath, err := filepath.Rel(rootPath, nodePath)
	if err != nil {
		return nil, err
	}
	npmPath, err := filepath.Rel(rootPath, npmPath)
	if err != nil {
		return nil, err
	}
	args = append([]string{npmPath}, args...)
	if debugging() {
		args = append(args, "--loglevel="+os.Getenv("GODE_DEBUG"))
	}
	cmd := exec.Command(nodePath, args...)
	cmd.Dir = rootPath
	cmd.Env = environ()
	return cmd, nil
}

func execNpm(args ...string) (string, string, error) {
	cmd, err := npmCmd(args...)
	if err != nil {
		return "", "", err
	}
	var stdout, stderr bytes.Buffer
	if debugging() {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	} else {
		cmd.Stdout = &stdout
		cmd.Stderr = &stderr
	}
	err = cmd.Run()
	return stdout.String(), stderr.String(), err
}

func environ() []string {
	env := append(os.Environ(), "NPM_CONFIG_ALWAYS_AUTH=false")
	env = append(env, "NPM_CONFIG_CACHE="+filepath.Join(rootPath, ".npm-cache"))
	env = append(env, "NPM_CONFIG_REGISTRY="+registry)
	return env
}

func debugging() bool {
	e := os.Getenv("GODE_DEBUG")
	return e != "" && e != "0" && e != "false"
}
<commit_msg>Set global=false so that cwd node_modules is used<commit_after>package gode

import (
	"bytes"
	"encoding/json"
	"errors"
	"os"
	"os/exec"
	"path/filepath"
)

// Package represents an npm package.
type Package struct {
	Name    string `json:"name"`
	Version string `json:"version"`
}

// Packages returns a list of npm packages installed.
func Packages() ([]Package, error) {
	stdout, stderr, err := execNpm("list", "--json", "--depth=0")
	if err != nil {
		return nil, errors.New(stderr)
	}
	var response map[string]map[string]Package
	if err := json.Unmarshal([]byte(stdout), &response); err != nil {
		return nil, errors.New(stderr)
	}
	packages := make([]Package, 0, len(response["dependencies"]))
	for name, p := range response["dependencies"] {
		p.Name = name
		packages = append(packages, p)
	}
	return packages, nil
}

// InstallPackages installs a npm packages.
func InstallPackages(packages ...string) error {
	args := append([]string{"install"}, packages...)
	_, stderr, err := execNpm(args...)
	if err != nil {
		return errors.New("Error installing package. \n" + stderr + "\nTry running again with GODE_DEBUG=info to see more output.")
	}
	return nil
}

// RemovePackages removes a npm packages.
func RemovePackages(packages ...string) error {
	args := append([]string{"remove"}, packages...)
	_, stderr, err := execNpm(args...)
	if err != nil {
		return errors.New(stderr)
	}
	return nil
}

// OutdatedPackages returns a map of packages and their latest version
func OutdatedPackages(names ...string) (map[string]string, error) {
	args := append([]string{"outdated", "--json"}, names...)
	stdout, stderr, err := execNpm(args...)
	if err != nil {
		return nil, errors.New(stderr)
	}
	var outdated map[string]struct{ Latest string }
	json.Unmarshal([]byte(stdout), &outdated)
	packages := make(map[string]string, len(outdated))
	for name, versions := range outdated {
		packages[name] = versions.Latest
	}
	return packages, nil
}

// ClearCache clears the npm cache
func ClearCache() error {
	cmd, err := npmCmd("cache", "clean")
	if err != nil {
		return err
	}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func npmCmd(args ...string) (*exec.Cmd, error) {
	if err := os.MkdirAll(filepath.Join(rootPath, "node_modules"), 0755); err != nil {
		return nil, err
	}
	nodePath, err := filepath.Rel(rootPath, nodePath)
	if err != nil {
		return nil, err
	}
	npmPath, err := filepath.Rel(rootPath, npmPath)
	if err != nil {
		return nil, err
	}
	args = append([]string{npmPath}, args...)
	if debugging() {
		args = append(args, "--loglevel="+os.Getenv("GODE_DEBUG"))
	}
	cmd := exec.Command(nodePath, args...)
	cmd.Dir = rootPath
	cmd.Env = environ()
	return cmd, nil
}

func execNpm(args ...string) (string, string, error) {
	cmd, err := npmCmd(args...)
	if err != nil {
		return "", "", err
	}
	var stdout, stderr bytes.Buffer
	if debugging() {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	} else {
		cmd.Stdout = &stdout
		cmd.Stderr = &stderr
	}
	err = cmd.Run()
	return stdout.String(), stderr.String(), err
}

func environ() []string {
	env := append(os.Environ(), "NPM_CONFIG_ALWAYS_AUTH=false")
	env = append(env, "NPM_CONFIG_CACHE="+filepath.Join(rootPath, ".npm-cache"))
	env = append(env, "NPM_CONFIG_REGISTRY="+registry)
	env = append(env, "NPM_CONFIG_GLOBAL=false")
	return env
}

func debugging() bool {
	e := os.Getenv("GODE_DEBUG")
	return e != "" && e != "0" && e != "false"
}
<|endoftext|>


<commit_before>package google

import (
	"errors"
	"github.com/graymeta/stow"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	storage "google.golang.org/api/storage/v1"
	"net/http"
	"net/url"
	"strings"
)

// Kind represents the name of the location/storage type.
const Kind = "google"

const (
	// The service account json blob
	ConfigJSON      = "json"
	ConfigProjectId = "project_id"
	ConfigScopes    = "scopes"
)

func init() {

	makefn := func(config stow.Config) (stow.Location, error) {
		_, ok := config.Config(ConfigJSON)
		if !ok {
			return nil, errors.New("missing JSON configuration")
		}

		_, ok = config.Config(ConfigProjectId)
		if !ok {
			return nil, errors.New("missing Project ID")
		}

		// Create a new client
		client, err := newGoogleStorageClient(config)
		if err != nil {
			return nil, err
		}

		// Create a location with given config and client
		loc := &Location{
			config: config,
			client: client,
		}

		return loc, nil
	}

	kindfn := func(u *url.URL) bool {
		return u.Scheme == Kind
	}

	stow.Register(Kind, makefn, kindfn)
}

// Attempts to create a session based on the information given.
func newGoogleStorageClient(config stow.Config) (*storage.Service, error) {
	json, _ := config.Config(ConfigJSON)
	var httpClint *http.Client
	scopes := []string{storage.DevstorageReadWriteScope}
	if s, ok := config.Config(ConfigScopes); ok && s != "" {
		scopes = strings.Split(s, ",")
	}
	if json != "" {
		jwtConf, err := google.JWTConfigFromJSON([]byte(json), scopes...)
		if err != nil {
			return nil, err
		}
		httpClint = jwtConf.Client(context.Background())

	} else {
		creds, err := google.FindDefaultCredentials(context.Background(), strings.Join(scopes, ","))
		if err != nil {
			return nil, err
		}
		httpClint = oauth2.NewClient(context.Background(), creds.TokenSource)
	}
	service, err := storage.New(httpClint)
	if err != nil {
		return nil, err
	}

	return service, nil
}
<commit_msg>Review changes<commit_after>package google

import (
	"errors"
	"net/http"
	"net/url"
	"strings"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	storage "google.golang.org/api/storage/v1"

	"github.com/graymeta/stow"
)

// Kind represents the name of the location/storage type.
const Kind = "google"

const (
	// The service account json blob
	ConfigJSON      = "json"
	ConfigProjectId = "project_id"
	ConfigScopes    = "scopes"
)

func init() {

	makefn := func(config stow.Config) (stow.Location, error) {
		_, ok := config.Config(ConfigJSON)
		if !ok {
			return nil, errors.New("missing JSON configuration")
		}

		_, ok = config.Config(ConfigProjectId)
		if !ok {
			return nil, errors.New("missing Project ID")
		}

		// Create a new client
		client, err := newGoogleStorageClient(config)
		if err != nil {
			return nil, err
		}

		// Create a location with given config and client
		loc := &Location{
			config: config,
			client: client,
		}

		return loc, nil
	}

	kindfn := func(u *url.URL) bool {
		return u.Scheme == Kind
	}

	stow.Register(Kind, makefn, kindfn)
}

// Attempts to create a session based on the information given.
func newGoogleStorageClient(config stow.Config) (*storage.Service, error) {
	json, _ := config.Config(ConfigJSON)
	var httpClient *http.Client
	scopes := []string{storage.DevstorageReadWriteScope}
	if s, ok := config.Config(ConfigScopes); ok && s != "" {
		scopes = strings.Split(s, ",")
	}
	if json != "" {
		jwtConf, err := google.JWTConfigFromJSON([]byte(json), scopes...)
		if err != nil {
			return nil, err
		}
		httpClient = jwtConf.Client(context.Background())

	} else {
		creds, err := google.FindDefaultCredentials(context.Background(), strings.Join(scopes, ","))
		if err != nil {
			return nil, err
		}
		httpClient = oauth2.NewClient(context.Background(), creds.TokenSource)
	}
	service, err := storage.New(httpClient)
	if err != nil {
		return nil, err
	}

	return service, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bigtable

import (
	"reflect"
	"sort"
	"testing"
	"time"

	"golang.org/x/net/context"
)

func TestAdminIntegration(t *testing.T) {
	testEnv, err := NewIntegrationEnv()
	if err != nil {
		t.Fatalf("IntegrationEnv: %v", err)
	}
	defer testEnv.Close()

	timeout := 2 * time.Second
	if testEnv.Config().UseProd {
		timeout = 5 * time.Minute
	}
	ctx, _ := context.WithTimeout(context.Background(), timeout)

	adminClient, err := testEnv.NewAdminClient()
	if err != nil {
		t.Fatalf("NewAdminClient: %v", err)
	}
	defer adminClient.Close()

	list := func() []string {
		tbls, err := adminClient.Tables(ctx)
		if err != nil {
			t.Fatalf("Fetching list of tables: %v", err)
		}
		sort.Strings(tbls)
		return tbls
	}
	if err := adminClient.CreateTable(ctx, "mytable"); err != nil {
		t.Fatalf("Creating table: %v", err)
	}
	if err := adminClient.CreateTable(ctx, "myothertable"); err != nil {
		t.Fatalf("Creating table: %v", err)
	}
	if got, want := list(), []string{"myothertable", "mytable"}; !reflect.DeepEqual(got, want) {
		t.Errorf("adminClient.Tables returned %#v, want %#v", got, want)
	}
	if err := adminClient.DeleteTable(ctx, "myothertable"); err != nil {
		t.Fatalf("Deleting table: %v", err)
	}
	if got, want := list(), []string{"mytable"}; !reflect.DeepEqual(got, want) {
		t.Errorf("adminClient.Tables returned %#v, want %#v", got, want)
	}
}
<commit_msg>bigtable: fix admin integration test in production<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bigtable

import (
	"sort"
	"testing"
	"time"

	"golang.org/x/net/context"
)

func TestAdminIntegration(t *testing.T) {
	testEnv, err := NewIntegrationEnv()
	if err != nil {
		t.Fatalf("IntegrationEnv: %v", err)
	}
	defer testEnv.Close()

	timeout := 2 * time.Second
	if testEnv.Config().UseProd {
		timeout = 5 * time.Minute
	}
	ctx, _ := context.WithTimeout(context.Background(), timeout)

	adminClient, err := testEnv.NewAdminClient()
	if err != nil {
		t.Fatalf("NewAdminClient: %v", err)
	}
	defer adminClient.Close()

	list := func() []string {
		tbls, err := adminClient.Tables(ctx)
		if err != nil {
			t.Fatalf("Fetching list of tables: %v", err)
		}
		sort.Strings(tbls)
		return tbls
	}
	containsAll := func(got, want []string) bool {
		gotSet := make(map[string]bool)

		for _, s := range got {
			gotSet[s] = true
		}
		for _, s := range want {
			if !gotSet[s] {
				return false
			}
		}
		return true
	}

	defer adminClient.DeleteTable(ctx, "mytable")

	if err := adminClient.CreateTable(ctx, "mytable"); err != nil {
		t.Fatalf("Creating table: %v", err)
	}

	defer adminClient.DeleteTable(ctx, "myothertable")

	if err := adminClient.CreateTable(ctx, "myothertable"); err != nil {
		t.Fatalf("Creating table: %v", err)
	}

	if got, want := list(), []string{"myothertable", "mytable"}; !containsAll(got, want) {
		t.Errorf("adminClient.Tables returned %#v, want %#v", got, want)
	}
	if err := adminClient.DeleteTable(ctx, "myothertable"); err != nil {
		t.Fatalf("Deleting table: %v", err)
	}
	tables := list()
	if got, want := tables, []string{"mytable"}; !containsAll(got, want) {
		t.Errorf("adminClient.Tables returned %#v, want %#v", got, want)
	}
	if got, unwanted := tables, []string{"myothertable"}; containsAll(got, unwanted) {
		t.Errorf("adminClient.Tables return %#v. unwanted %#v", got, unwanted)
	}
}
<|endoftext|>


<commit_before>package discovery

import (
	"context"
	"math"
	"math/rand"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/livepeer/go-livepeer/core"
	"github.com/livepeer/go-livepeer/monitor"
	"github.com/livepeer/go-livepeer/net"
	"github.com/livepeer/go-livepeer/server"

	"github.com/golang/glog"
)

const getOrchestratorsTimeoutLoop = 1 * time.Hour

var serverGetOrchInfo = server.GetOrchestratorInfo

type orchestratorPool struct {
	uris  []*url.URL
	bcast server.Broadcaster
	pred  func(info *net.OrchestratorInfo) bool
}

var perm = func(len int) []int { return rand.Perm(len) }

func NewOrchestratorPool(node *core.LivepeerNode, addresses []string) *orchestratorPool {
	var uris []*url.URL

	for _, addr := range addresses {
		if !strings.HasPrefix(addr, "http") {
			addr = "https://" + addr
		}
		uri, err := url.ParseRequestURI(addr)
		if err != nil {
			glog.Error("Could not parse orchestrator URI: ", err)
			continue
		}
		uris = append(uris, uri)
	}

	if len(uris) <= 0 {
		glog.Error("Could not parse orchAddresses given - no URIs returned ")
	}

	var randomizedUris []*url.URL
	for _, i := range perm(len(uris)) {
		uri := uris[i]
		randomizedUris = append(randomizedUris, uri)
	}

	bcast := core.NewBroadcaster(node)
	return &orchestratorPool{bcast: bcast, uris: randomizedUris}
}

func NewOrchestratorPoolWithPred(node *core.LivepeerNode, addresses []string, pred func(*net.OrchestratorInfo) bool) *orchestratorPool {
	// if livepeer running in offchain mode, return nil
	if node.Eth == nil {
		glog.Error("Could not refresh DB list of orchestrators: LivepeerNode nil")
		return nil
	}
	pool := NewOrchestratorPool(node, addresses)
	pool.pred = pred
	return pool
}

func NewOnchainOrchestratorPool(node *core.LivepeerNode) *orchestratorPool {
	// if livepeer running in offchain mode, return nil
	if node.Eth == nil {
		glog.Error("Could not refresh DB list of orchestrators: LivepeerNode nil")
		return nil
	}

	orchestrators, err := node.Eth.RegisteredTranscoders()
	if err != nil {
		glog.Error("Could not refresh DB list of orchestrators: ", err)
		return nil
	}

	var addresses []string
	for _, orch := range orchestrators {
		addresses = append(addresses, orch.ServiceURI)
	}

	return NewOrchestratorPool(node, addresses)
}

func (o *orchestratorPool) GetURLs() []*url.URL {
	return o.uris
}

func (o *orchestratorPool) GetOrchestrators(numOrchestrators int) ([]*net.OrchestratorInfo, error) {
	numAvailableOrchs := len(o.uris)
	numOrchestrators = int(math.Min(float64(numAvailableOrchs), float64(numOrchestrators)))
	ctx, cancel := context.WithTimeout(context.Background(), getOrchestratorsTimeoutLoop)
	orchInfos := []*net.OrchestratorInfo{}
	orchChan := make(chan struct{}, len(o.uris))
	numResp := 0
	numSuccessResp := 0
	respLock := sync.Mutex{}

	getOrchInfo := func(uri *url.URL) {
		info, err := serverGetOrchInfo(ctx, o.bcast, uri)
		respLock.Lock()
		defer respLock.Unlock()
		numResp++
		if err == nil && (o.pred == nil || o.pred(info)) {
			orchInfos = append(orchInfos, info)
			numSuccessResp++
		} else if monitor.Enabled {
			monitor.LogDiscoveryError(err.Error())
		}
		if numSuccessResp >= numOrchestrators || numResp >= len(o.uris) {
			orchChan <- struct{}{}
		}
	}

	for _, uri := range o.uris {
		go getOrchInfo(uri)
	}

	select {
	case <-ctx.Done():
		respLock.Lock()
		if len(orchInfos) < numOrchestrators {
			numOrchestrators = len(orchInfos)
		}
		returnOrchs := orchInfos[:numOrchestrators]
		respLock.Unlock()
		glog.Info("Done fetching orch info for orchestrators, context timeout: ", returnOrchs)
		cancel()
		return returnOrchs, nil
	case <-orchChan:
		respLock.Lock()
		if len(orchInfos) < numOrchestrators {
			numOrchestrators = len(orchInfos)
		}
		returnOrchs := orchInfos[:numOrchestrators]
		respLock.Unlock()
		glog.Info("Done fetching orch info for orchestrators, numResponses fetched: ", returnOrchs)
		cancel()
		return returnOrchs, nil
	}
}

func (o *orchestratorPool) Size() int {
	return len(o.uris)
}
<commit_msg>discovery: Check for nil error before monitor.LogDiscoveryError<commit_after>package discovery

import (
	"context"
	"math"
	"math/rand"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/livepeer/go-livepeer/core"
	"github.com/livepeer/go-livepeer/monitor"
	"github.com/livepeer/go-livepeer/net"
	"github.com/livepeer/go-livepeer/server"

	"github.com/golang/glog"
)

const getOrchestratorsTimeoutLoop = 1 * time.Hour

var serverGetOrchInfo = server.GetOrchestratorInfo

type orchestratorPool struct {
	uris  []*url.URL
	bcast server.Broadcaster
	pred  func(info *net.OrchestratorInfo) bool
}

var perm = func(len int) []int { return rand.Perm(len) }

func NewOrchestratorPool(node *core.LivepeerNode, addresses []string) *orchestratorPool {
	var uris []*url.URL

	for _, addr := range addresses {
		if !strings.HasPrefix(addr, "http") {
			addr = "https://" + addr
		}
		uri, err := url.ParseRequestURI(addr)
		if err != nil {
			glog.Error("Could not parse orchestrator URI: ", err)
			continue
		}
		uris = append(uris, uri)
	}

	if len(uris) <= 0 {
		glog.Error("Could not parse orchAddresses given - no URIs returned ")
	}

	var randomizedUris []*url.URL
	for _, i := range perm(len(uris)) {
		uri := uris[i]
		randomizedUris = append(randomizedUris, uri)
	}

	bcast := core.NewBroadcaster(node)
	return &orchestratorPool{bcast: bcast, uris: randomizedUris}
}

func NewOrchestratorPoolWithPred(node *core.LivepeerNode, addresses []string, pred func(*net.OrchestratorInfo) bool) *orchestratorPool {
	// if livepeer running in offchain mode, return nil
	if node.Eth == nil {
		glog.Error("Could not refresh DB list of orchestrators: LivepeerNode nil")
		return nil
	}
	pool := NewOrchestratorPool(node, addresses)
	pool.pred = pred
	return pool
}

func NewOnchainOrchestratorPool(node *core.LivepeerNode) *orchestratorPool {
	// if livepeer running in offchain mode, return nil
	if node.Eth == nil {
		glog.Error("Could not refresh DB list of orchestrators: LivepeerNode nil")
		return nil
	}

	orchestrators, err := node.Eth.RegisteredTranscoders()
	if err != nil {
		glog.Error("Could not refresh DB list of orchestrators: ", err)
		return nil
	}

	var addresses []string
	for _, orch := range orchestrators {
		addresses = append(addresses, orch.ServiceURI)
	}

	return NewOrchestratorPool(node, addresses)
}

func (o *orchestratorPool) GetURLs() []*url.URL {
	return o.uris
}

func (o *orchestratorPool) GetOrchestrators(numOrchestrators int) ([]*net.OrchestratorInfo, error) {
	numAvailableOrchs := len(o.uris)
	numOrchestrators = int(math.Min(float64(numAvailableOrchs), float64(numOrchestrators)))
	ctx, cancel := context.WithTimeout(context.Background(), getOrchestratorsTimeoutLoop)
	orchInfos := []*net.OrchestratorInfo{}
	orchChan := make(chan struct{}, len(o.uris))
	numResp := 0
	numSuccessResp := 0
	respLock := sync.Mutex{}

	getOrchInfo := func(uri *url.URL) {
		info, err := serverGetOrchInfo(ctx, o.bcast, uri)
		respLock.Lock()
		defer respLock.Unlock()
		numResp++
		if err == nil && (o.pred == nil || o.pred(info)) {
			orchInfos = append(orchInfos, info)
			numSuccessResp++
		}
		if err != nil && monitor.Enabled {
			monitor.LogDiscoveryError(err.Error())
		}
		if numSuccessResp >= numOrchestrators || numResp >= len(o.uris) {
			orchChan <- struct{}{}
		}
	}

	for _, uri := range o.uris {
		go getOrchInfo(uri)
	}

	select {
	case <-ctx.Done():
		respLock.Lock()
		if len(orchInfos) < numOrchestrators {
			numOrchestrators = len(orchInfos)
		}
		returnOrchs := orchInfos[:numOrchestrators]
		respLock.Unlock()
		glog.Info("Done fetching orch info for orchestrators, context timeout: ", returnOrchs)
		cancel()
		return returnOrchs, nil
	case <-orchChan:
		respLock.Lock()
		if len(orchInfos) < numOrchestrators {
			numOrchestrators = len(orchInfos)
		}
		returnOrchs := orchInfos[:numOrchestrators]
		respLock.Unlock()
		glog.Info("Done fetching orch info for orchestrators, numResponses fetched: ", returnOrchs)
		cancel()
		return returnOrchs, nil
	}
}

func (o *orchestratorPool) Size() int {
	return len(o.uris)
}
<|endoftext|>