<commit_before>// Parse Sidestream tests.
package parser

import (
	"bytes"
	"cloud.google.com/go/bigquery"
	"errors"
	"log"
	"path/filepath"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/m-lab/etl/etl"
	"github.com/m-lab/etl/metrics"
	"github.com/m-lab/etl/schema"
	"github.com/m-lab/etl/web100"
)

type SSParser struct {
	inserter etl.Inserter
	etl.RowStats
}

func NewSSParser(ins etl.Inserter) *SSParser {
	return &SSParser{ins, ins}
}

// The legacy filename is like  "20170203T00:00:00Z_ALL0.web100"
// The current filename is like "20170315T01:00:00Z_173.205.3.39_0.web100"
// Return time stamp if the filename is in right format
func ExtractLogtimeFromFilename(fileName string) (int64, error) {
	testName := filepath.Base(fileName)
	if len(testName) < 19 || !strings.Contains(testName, ".web100") {
		log.Println(testName)
		return 0, errors.New("Invalid sidestream filename")
	}

	t, err := time.Parse("20060102T15:04:05.999999999Z_", testName[0:17]+".000000000Z_")
	if err != nil {
		return 0, err
	}

	return t.Unix(), nil
}

// the first line of SS test is in format "K: cid PollTime LocalAddress LocalPort ... other_web100_variables_separated_by_space"
func ParseKHeader(header string) ([]string, error) {
	var var_names []string
	web100_vars := strings.Split(header, " ")
	if web100_vars[0] != "K:" {
		return var_names, errors.New("Corrupted header")
	}

	data, err := web100.Asset("tcp-kis.txt")
	if err != nil {
		panic("tcp-kis.txt not found")
	}
	b := bytes.NewBuffer(data)

	mapping, err := web100.ParseWeb100Definitions(b)

	for index, name := range web100_vars {
		if index == 0 {
			continue
		}

		if mapping[name] != "" {
			var_names = append(var_names, mapping[name])
		} else {
			var_names = append(var_names, name)
		}
	}
	return var_names, nil
}

func (ss *SSParser) TableName() string {
	return ss.inserter.TableBase()
}

func (ss *SSParser) FullTableName() string {
	return ss.inserter.FullTableName()
}

func (ss *SSParser) Flush() error {
	return ss.inserter.Flush()
}

// Prepare data into sidestream BigQeury schema and insert it.
func PackDataIntoSchema(ss_value map[string]string, log_time int64, testName string) (schema.SS, error) {
	local_port, err := strconv.Atoi(ss_value["LocalPort"])
	if err != nil {
		return schema.SS{}, err
	}
	remote_port, err := strconv.Atoi(ss_value["RemPort"])
	if err != nil {
		return schema.SS{}, err
	}

	conn_spec := &schema.Web100ConnectionSpecification{
		Local_ip:    ss_value["LocalAddress"],
		Local_af:    ParseIPFamily(ss_value["LocalAddress"]),
		Local_port:  int64(local_port),
		Remote_ip:   ss_value["RemAddress"],
		Remote_port: int64(remote_port),
	}
	snap, err := PopulateSnap(ss_value)
	if err != nil {
		return schema.SS{}, err
	}
	web100_log := &schema.Web100LogEntry{
		Log_time:        log_time,
		Version:         "unknown",
		Group_name:      "read",
		Connection_spec: *conn_spec,
		Snap:            snap,
	}

	ss_test := &schema.SS{
		Test_id:          testName,
		Log_time:         log_time,
		Type:             int64(1),
		Project:          int64(2),
		Web100_log_entry: *web100_log,
	}
	return *ss_test, nil
}

func ParseOneLine(snapshot string, var_names []string) (map[string]string, error) {
	value := strings.Split(snapshot, " ")
	ss_value := make(map[string]string)
	if value[0] != "C:" || len(value) != len(var_names)+1 {
		log.Printf("corrupted content:")
		log.Printf(snapshot)
		return ss_value, errors.New("corrupted content")
	}

	for index, val := range value[1:] {
		// Match value with var_name
		ss_value[var_names[index]] = val
	}
	return ss_value, nil
}

func PopulateSnap(ss_value map[string]string) (schema.Web100Snap, error) {
	var snap = &schema.Web100Snap{}
	for key := range ss_value {
		// Skip cid and PollTime. They are SideStream-specific fields, not web100 variables.
		if key == "cid" || key == "PollTime" {
			continue
		}
		// We do special handling for this variable
		if key == "StartTimeUsec" {
			// TODO: func CalculateStartTimeStamp() to get correct StartTimeStamp value.
			continue
		}
		x := reflect.ValueOf(snap).Elem().FieldByName(key)

		switch x.Type().String() {
		case "int64":
			value, err := strconv.Atoi(ss_value[key])
			if err != nil {
				return *snap, err
			}
			x.SetInt(int64(value))
		case "string":
			x.Set(reflect.ValueOf(ss_value[key]))
		case "bool":
			if ss_value[key] == "0" {
				x.Set(reflect.ValueOf(false))
			} else if ss_value[key] == "1" {
				x.Set(reflect.ValueOf(true))
			} else {
				return *snap, errors.New("Cannot parse field " + key + " into a valie bool value.")
			}
		}
	}
	// TODO: check whether snap has valid LocalAddress, RemAddress. Return error if not.
	return *snap, nil
}

func (ss *SSParser) ParseAndInsert(meta map[string]bigquery.Value, testName string, rawContent []byte) error {
	metrics.WorkerState.WithLabelValues("ss").Inc()
	defer metrics.WorkerState.WithLabelValues("ss").Dec()
	if strings.Contains(testName, ".tra") {
		// Ignore the trace file for sidestream test.
		return nil
	}
	log_time, err := ExtractLogtimeFromFilename(testName)
	if err != nil {
		return err
	}
	var var_names []string
	testContent := strings.Split(string(rawContent[:]), "\n")
	if len(testContent) < 2 {
		return errors.New("empty test file.")
	}
	var_names, err = ParseKHeader(testContent[0])
	if err != nil {
		metrics.ErrorCount.WithLabelValues(
			ss.TableName(), "ss", "corrupted header").Inc()
		return err
	}
	for _, oneLine := range testContent[1:] {
		oneLine := strings.TrimSuffix(oneLine, "\n")

		if len(oneLine) == 0 {
			continue
		}
		ss_value, err := ParseOneLine(oneLine, var_names)
		if err != nil {
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "corrupted content").Inc()
			return err
		}
		if ValidateIP(ss_value["LocalAddress"]) != nil {
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "Invalid server IP").Inc()
			return errors.New("Invalid server IP address: " + ss_value["LocalAddress"])
		}
		if ValidateIP(ss_value["RemAddress"]) != nil {
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "Invalid client IP").Inc()
			return errors.New("Invalid client IP address: " + ss_value["RemAddress"])
		}
		ss_test, err := PackDataIntoSchema(ss_value, log_time, testName)
		if err != nil {
			metrics.ErrorCount.WithLabelValues(
				ss.TableName(), "ss", "corrupted data").Inc()
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "corrupted data").Inc()
			log.Printf("cannot pack data into sidestream schema: %v\n", err)
			return err
		}
		err = ss.inserter.InsertRow(ss_test)
		if err != nil {
			metrics.ErrorCount.WithLabelValues(
				ss.TableName(), "ss", "insert-err").Inc()
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "insert-err").Inc()
			log.Printf("insert-err: %v\n", err)
			continue
		}
		metrics.TestCount.WithLabelValues(ss.TableName(), "ss", "ok").Inc()

	}
	return nil
}
<commit_msg>Update err messages with more info<commit_after>// Parse Sidestream tests.
package parser

import (
	"bytes"
	"cloud.google.com/go/bigquery"
	"errors"
	"log"
	"path/filepath"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/m-lab/etl/etl"
	"github.com/m-lab/etl/metrics"
	"github.com/m-lab/etl/schema"
	"github.com/m-lab/etl/web100"
)

type SSParser struct {
	inserter etl.Inserter
	etl.RowStats
}

func NewSSParser(ins etl.Inserter) *SSParser {
	return &SSParser{ins, ins}
}

// The legacy filename is like  "20170203T00:00:00Z_ALL0.web100"
// The current filename is like "20170315T01:00:00Z_173.205.3.39_0.web100"
// Return time stamp if the filename is in right format
func ExtractLogtimeFromFilename(fileName string) (int64, error) {
	testName := filepath.Base(fileName)
	if len(testName) < 19 || !strings.Contains(testName, ".web100") {
		log.Println(testName)
		return 0, errors.New("Invalid sidestream filename")
	}

	t, err := time.Parse("20060102T15:04:05.999999999Z_", testName[0:17]+".000000000Z_")
	if err != nil {
		return 0, err
	}

	return t.Unix(), nil
}

// the first line of SS test is in format "K: cid PollTime LocalAddress LocalPort ... other_web100_variables_separated_by_space"
func ParseKHeader(header string) ([]string, error) {
	var var_names []string
	web100_vars := strings.Split(header, " ")
	if web100_vars[0] != "K:" {
		return var_names, errors.New("Corrupted header")
	}

	data, err := web100.Asset("tcp-kis.txt")
	if err != nil {
		panic("tcp-kis.txt not found")
	}
	b := bytes.NewBuffer(data)

	mapping, err := web100.ParseWeb100Definitions(b)

	for index, name := range web100_vars {
		if index == 0 {
			continue
		}

		if mapping[name] != "" {
			var_names = append(var_names, mapping[name])
		} else {
			var_names = append(var_names, name)
		}
	}
	return var_names, nil
}

func (ss *SSParser) TableName() string {
	return ss.inserter.TableBase()
}

func (ss *SSParser) FullTableName() string {
	return ss.inserter.FullTableName()
}

func (ss *SSParser) Flush() error {
	return ss.inserter.Flush()
}

// Prepare data into sidestream BigQeury schema and insert it.
func PackDataIntoSchema(ss_value map[string]string, log_time int64, testName string) (schema.SS, error) {
	local_port, err := strconv.Atoi(ss_value["LocalPort"])
	if err != nil {
		return schema.SS{}, err
	}
	remote_port, err := strconv.Atoi(ss_value["RemPort"])
	if err != nil {
		return schema.SS{}, err
	}

	conn_spec := &schema.Web100ConnectionSpecification{
		Local_ip:    ss_value["LocalAddress"],
		Local_af:    ParseIPFamily(ss_value["LocalAddress"]),
		Local_port:  int64(local_port),
		Remote_ip:   ss_value["RemAddress"],
		Remote_port: int64(remote_port),
	}
	snap, err := PopulateSnap(ss_value)
	if err != nil {
		return schema.SS{}, err
	}
	web100_log := &schema.Web100LogEntry{
		Log_time:        log_time,
		Version:         "unknown",
		Group_name:      "read",
		Connection_spec: *conn_spec,
		Snap:            snap,
	}

	ss_test := &schema.SS{
		Test_id:          testName,
		Log_time:         log_time,
		Type:             int64(1),
		Project:          int64(2),
		Web100_log_entry: *web100_log,
	}
	return *ss_test, nil
}

func ParseOneLine(snapshot string, var_names []string) (map[string]string, error) {
	value := strings.Split(snapshot, " ")
	ss_value := make(map[string]string)
	if value[0] != "C:" || len(value) != len(var_names)+1 {
		log.Printf("corrupted content:")
		log.Printf(snapshot)
		return ss_value, errors.New("corrupted content")
	}

	for index, val := range value[1:] {
		// Match value with var_name
		ss_value[var_names[index]] = val
	}
	return ss_value, nil
}

func PopulateSnap(ss_value map[string]string) (schema.Web100Snap, error) {
	var snap = &schema.Web100Snap{}
	for key := range ss_value {
		// Skip cid and PollTime. They are SideStream-specific fields, not web100 variables.
		if key == "cid" || key == "PollTime" {
			continue
		}
		// We do special handling for this variable
		if key == "StartTimeUsec" {
			// TODO: func CalculateStartTimeStamp() to get correct StartTimeStamp value.
			continue
		}
		x := reflect.ValueOf(snap).Elem().FieldByName(key)

		switch x.Type().String() {
		case "int64":
			value, err := strconv.Atoi(ss_value[key])
			if err != nil {
				return *snap, err
			}
			x.SetInt(int64(value))
		case "string":
			x.Set(reflect.ValueOf(ss_value[key]))
		case "bool":
			if ss_value[key] == "0" {
				x.Set(reflect.ValueOf(false))
			} else if ss_value[key] == "1" {
				x.Set(reflect.ValueOf(true))
			} else {
				return *snap, errors.New("Cannot parse field " + key + " into a valie bool value.")
			}
		}
	}
	// TODO: check whether snap has valid LocalAddress, RemAddress. Return error if not.
	return *snap, nil
}

func (ss *SSParser) ParseAndInsert(meta map[string]bigquery.Value, testName string, rawContent []byte) error {
	metrics.WorkerState.WithLabelValues("ss").Inc()
	defer metrics.WorkerState.WithLabelValues("ss").Dec()
	if strings.Contains(testName, ".tra") {
		// Ignore the trace file for sidestream test.
		return nil
	}
	log_time, err := ExtractLogtimeFromFilename(testName)
	if err != nil {
		return err
	}
	var var_names []string
	testContent := strings.Split(string(rawContent[:]), "\n")
	if len(testContent) < 2 {
		return errors.New("empty test file.")
	}
	var_names, err = ParseKHeader(testContent[0])
	if err != nil {
		metrics.ErrorCount.WithLabelValues(
			ss.TableName(), "ss", "corrupted header").Inc()
		return err
	}
	for _, oneLine := range testContent[1:] {
		oneLine := strings.TrimSuffix(oneLine, "\n")

		if len(oneLine) == 0 {
			continue
		}
		ss_value, err := ParseOneLine(oneLine, var_names)
		if err != nil {
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "corrupted content").Inc()
			return err
		}
		if err := ValidateIP(ss_value["LocalAddress"]), err != nil {
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "Invalid server IP").Inc()
			return errors.New("Invalid server IP address: " + ss_value["LocalAddress"] +
					  " with error: " + err.Error())
		}
		if err := ValidateIP(ss_value["RemAddress"]), err != nil {
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "Invalid client IP").Inc()
			return errors.New("Invalid client IP address: " + ss_value["RemAddress"] +
					  " with error: " + err.Error())
		}
		ss_test, err := PackDataIntoSchema(ss_value, log_time, testName)
		if err != nil {
			metrics.ErrorCount.WithLabelValues(
				ss.TableName(), "ss", "corrupted data").Inc()
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "corrupted data").Inc()
			log.Printf("cannot pack data into sidestream schema: %v\n", err)
			return err
		}
		err = ss.inserter.InsertRow(ss_test)
		if err != nil {
			metrics.ErrorCount.WithLabelValues(
				ss.TableName(), "ss", "insert-err").Inc()
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "insert-err").Inc()
			log.Printf("insert-err: %v\n", err)
			continue
		}
		metrics.TestCount.WithLabelValues(ss.TableName(), "ss", "ok").Inc()

	}
	return nil
}
<|endoftext|>


<commit_before>package gotwilio

import (
	"encoding/json"
	"fmt"
	"github.com/google/go-querystring/query"
	"net/http"
)

// Conference represents a Twilio Voice conference call
type Conference struct {
	Sid          string `json:"sid"`
	FriendlyName string `json:"friendly_name"`
	Status       string `json:"status"`
	Region       string `json:"region"`
}

// ConferenceOptions are used for updating Conferences
type ConferenceOptions struct {
	Status         string `url:"Status,omitempty"`
	AnnounceURL    string `url:"AnnounceUrl,omitempty"`
	AnnounceMethod string `url:"AnnounceMethod,omitempty"`
}

// ConferenceParticipant represents a Participant in responses from the Twilio API
type ConferenceParticipant struct {
	CallSid                string `json:"call_sid"`
	ConferenceSid          string `json:"conference_sid"`
	Muted                  bool   `json:"muted"`
	Hold                   bool   `json:"hold"`
	Status                 string `json:"status"`
	StartConferenceOnEnter bool   `json:"start_conference_on_enter"`
	EndConferenceOnExit    bool   `json:"end_conference_on_exit"`
	Coaching               bool   `json:"coaching"`
	CallSidToCoach         string `json:"call_sid_to_coach"`
}

// ConferenceParticipantOptions are used for creating and updating Conference Participants.
type ConferenceParticipantOptions struct {
	From                                    string  `url:"From,omitempty"`
	To                                      string  `url:"To,omitempty"`
	StatusCallback                          string  `url:"StatusCallback,omitempty"`
	StatusCallbackMethod                    string  `url:"StatusCallbackMethod,omitempty"`
	StatusCallbackEvent                     string  `url:"statusCallbackEvent,omitempty"`
	Timeout                                 int     `url:"Timeout"`
	Record                                  *bool   `url:"Record,omitempty"`
	Beep                                    *bool   `url:"Beep,omitempty"`
	Muted                                   *bool   `url:"Muted,omitempty"`
	Hold                                    *bool   `url:"Hold,omitempty"`
	HoldURL                                 *string `url:"HoldURL,omitempty"`
	HoldMethod                              *string `url:"HoldMethod,omitempty"`
	AnnounceURL                             *string `url:"AnnounceURL,omitempty"`
	AnnounceMethod                          *string `url:"AnnounceMethod,omitempty"`
	StartConferenceOnEnter                  *bool   `url:"StartConferenceOnEnter,omitempty"`
	EndConferenceOnExit                     *bool   `url:"EndConferenceOnExit,omitempty"`
	WaitURL                                 string  `url:"WaitURL,omitempty"`
	WaitMethod                              string  `url:"WaitMethod,omitempty"`
	EarlyMedia                              *bool   `url:"EarlyMedia,omitempty"`
	MaxParticipants                         int     `url:"MaxParticipants"`
	ConferenceRecord                        string  `url:"ConferenceRecord,omitempty"`
	ConferenceTrim                          string  `url:"ConferenceTrim,omitempty"`
	ConferenceStatusCallback                string  `url:"ConferenceStatusCallback,omitempty"`
	ConferenceStatusCallbackMethod          string  `url:"ConferenceStatusCallbackMethod,omitempty"`
	ConferenceStatusCallbackEvent           string  `url:"ConferenceStatusCallbackEvent,omitempty"`
	RecordingChannels                       string  `url:"RecordingChannels,omitempty"`
	RecordingStatusCallback                 string  `url:"RecordingStatusCallback,omitempty"`
	RecordingStatusCallbackMethod           string  `url:"RecordingStatusCallbackMethod,omitempty"`
	RecordingStatusCallbackEvent            string  `url:"RecordingStatusCallbackEvent,omitempty"`
	SipAuthUsername                         string  `url:"SipAuthUsername,omitempty"`
	SipAuthPassword                         string  `url:"SipAuthPassword,omitempty"`
	Region                                  string  `url:"Region,omitempty"`
	ConferenceRecordingStatusCallback       string  `url:"ConferenceRecordingStatusCallback,omitempty"`
	ConferenceRecordingStatusCallbackMethod string  `url:"ConferenceRecordingStatusCallbackMethod,omitempty"`
	Coaching                                *bool   `url:"Coaching,omitempty"`
	CallSidToCoach                          string  `url:"CallSidToCoach,omitempty"`
}

// GetConference fetches details for a single conference instance
// https://www.twilio.com/docs/voice/api/conference-resource#fetch-a-conference-resource
func (twilio *Twilio) GetConference(conferenceSid string) (*Conference, *Exception, error) {
	res, err := twilio.get(twilio.buildUrl(fmt.Sprintf("Conferences/%s.json", conferenceSid)))
	if err != nil {
		return nil, nil, err
	}

	decoder := json.NewDecoder(res.Body)

	// handle NULL response
	if res.StatusCode != http.StatusOK {
		exception := new(Exception)
		err = decoder.Decode(exception)
		return nil, exception, err
	}

	conf := new(Conference)
	err = decoder.Decode(conf)
	return conf, nil, err
}

// UpdateConference to end it or play an announcement
// https://www.twilio.com/docs/voice/api/conference-resource#update-a-conference-resource
func (twilio *Twilio) UpdateConference(conferenceSid string, options *ConferenceOptions) (*Conference, *Exception, error) {
	form, err := query.Values(options)
	if err != nil {
		return nil, nil, err
	}

	res, err := twilio.post(form, twilio.buildUrl(fmt.Sprintf("Conferences/%s.json", conferenceSid)))
	if err != nil {
		return nil, nil, err
	}

	decoder := json.NewDecoder(res.Body)

	if res.StatusCode != http.StatusOK {
		exception := new(Exception)
		err = decoder.Decode(exception)
		return nil, exception, err
	}

	c := new(Conference)
	err = decoder.Decode(c)
	return c, nil, err
}

// GetConferenceParticipant fetches details for a conference participant resource
// https://www.twilio.com/docs/voice/api/conference-participant-resource#fetch-a-participant-resource
func (twilio *Twilio) GetConferenceParticipant(conferenceSid, callSid string) (*ConferenceParticipant, *Exception, error) {
	res, err := twilio.get(twilio.buildUrl(fmt.Sprintf("Conferences/%s/Participants/%s.json", conferenceSid, callSid)))
	if err != nil {
		return nil, nil, err
	}

	decoder := json.NewDecoder(res.Body)

	// handle NULL response
	if res.StatusCode != http.StatusOK {
		exception := new(Exception)
		err = decoder.Decode(exception)
		return nil, exception, err
	}

	conf := new(ConferenceParticipant)
	err = decoder.Decode(conf)
	return conf, nil, err
}

// AddConferenceParticipant adds a Participant to a conference by dialing out a new call
// https://www.twilio.com/docs/voice/api/conference-participant-resource#create-a-participant-agent-conference-only
func (twilio *Twilio) AddConferenceParticipant(conferenceSid string, participant *ConferenceParticipantOptions) (*ConferenceParticipant, *Exception, error) {
	form, err := query.Values(participant)
	if err != nil {
		return nil, nil, err
	}

	res, err := twilio.post(form, twilio.buildUrl(fmt.Sprintf("Conferences/%s/Participants.json", conferenceSid)))
	if err != nil {
		return nil, nil, err
	}

	decoder := json.NewDecoder(res.Body)

	if res.StatusCode != http.StatusCreated {
		exception := new(Exception)
		err = decoder.Decode(exception)
		return nil, exception, err
	}

	conf := new(ConferenceParticipant)
	err = decoder.Decode(conf)
	return conf, nil, err
}

// UpdateConferenceParticipant
// https://www.twilio.com/docs/voice/api/conference-participant-resource#create-a-participant-agent-conference-only
func (twilio *Twilio) UpdateConferenceParticipant(conferenceSid string, callSid string, participant *ConferenceParticipantOptions) (*ConferenceParticipant, *Exception, error) {
	form, err := query.Values(participant)
	if err != nil {
		return nil, nil, err
	}

	res, err := twilio.post(form, twilio.buildUrl(fmt.Sprintf("Conferences/%s/Participants/%s.json", conferenceSid, callSid)))
	if err != nil {
		return nil, nil, err
	}

	decoder := json.NewDecoder(res.Body)

	if res.StatusCode != http.StatusOK {
		exception := new(Exception)
		err = decoder.Decode(exception)
		return nil, exception, err
	}

	p := new(ConferenceParticipant)
	err = decoder.Decode(p)
	return p, nil, err
}

// DeleteConferenceParticipant
func (twilio *Twilio) DeleteConferenceParticipant(conferenceSid string, callSid string) (*Exception, error) {
	res, err := twilio.delete(twilio.buildUrl(fmt.Sprintf("Conferences/%s/Participants/%s.json", conferenceSid, callSid)))
	if err != nil {
		return nil, err
	}

	if res.StatusCode != http.StatusOK {
		decoder := json.NewDecoder(res.Body)
		exception := new(Exception)
		err = decoder.Decode(exception)
		return exception, err
	}

	return nil, err
}
<commit_msg>get conference participants (#70)<commit_after>package gotwilio

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/google/go-querystring/query"
)

// Conference represents a Twilio Voice conference call
type Conference struct {
	Sid          string `json:"sid"`
	FriendlyName string `json:"friendly_name"`
	Status       string `json:"status"`
	Region       string `json:"region"`
}

// ConferenceOptions are used for updating Conferences
type ConferenceOptions struct {
	Status         string `url:"Status,omitempty"`
	AnnounceURL    string `url:"AnnounceUrl,omitempty"`
	AnnounceMethod string `url:"AnnounceMethod,omitempty"`
}

// ConferenceParticipant represents a Participant in responses from the Twilio API
type ConferenceParticipant struct {
	CallSid                string `json:"call_sid"`
	ConferenceSid          string `json:"conference_sid"`
	Muted                  bool   `json:"muted"`
	Hold                   bool   `json:"hold"`
	Status                 string `json:"status"`
	StartConferenceOnEnter bool   `json:"start_conference_on_enter"`
	EndConferenceOnExit    bool   `json:"end_conference_on_exit"`
	Coaching               bool   `json:"coaching"`
	CallSidToCoach         string `json:"call_sid_to_coach"`
}

// ConferenceParticipantOptions are used for creating and updating Conference Participants.
type ConferenceParticipantOptions struct {
	From                                    string  `url:"From,omitempty"`
	To                                      string  `url:"To,omitempty"`
	StatusCallback                          string  `url:"StatusCallback,omitempty"`
	StatusCallbackMethod                    string  `url:"StatusCallbackMethod,omitempty"`
	StatusCallbackEvent                     string  `url:"statusCallbackEvent,omitempty"`
	Timeout                                 int     `url:"Timeout"`
	Record                                  *bool   `url:"Record,omitempty"`
	Beep                                    *bool   `url:"Beep,omitempty"`
	Muted                                   *bool   `url:"Muted,omitempty"`
	Hold                                    *bool   `url:"Hold,omitempty"`
	HoldURL                                 *string `url:"HoldURL,omitempty"`
	HoldMethod                              *string `url:"HoldMethod,omitempty"`
	AnnounceURL                             *string `url:"AnnounceURL,omitempty"`
	AnnounceMethod                          *string `url:"AnnounceMethod,omitempty"`
	StartConferenceOnEnter                  *bool   `url:"StartConferenceOnEnter,omitempty"`
	EndConferenceOnExit                     *bool   `url:"EndConferenceOnExit,omitempty"`
	WaitURL                                 string  `url:"WaitURL,omitempty"`
	WaitMethod                              string  `url:"WaitMethod,omitempty"`
	EarlyMedia                              *bool   `url:"EarlyMedia,omitempty"`
	MaxParticipants                         int     `url:"MaxParticipants"`
	ConferenceRecord                        string  `url:"ConferenceRecord,omitempty"`
	ConferenceTrim                          string  `url:"ConferenceTrim,omitempty"`
	ConferenceStatusCallback                string  `url:"ConferenceStatusCallback,omitempty"`
	ConferenceStatusCallbackMethod          string  `url:"ConferenceStatusCallbackMethod,omitempty"`
	ConferenceStatusCallbackEvent           string  `url:"ConferenceStatusCallbackEvent,omitempty"`
	RecordingChannels                       string  `url:"RecordingChannels,omitempty"`
	RecordingStatusCallback                 string  `url:"RecordingStatusCallback,omitempty"`
	RecordingStatusCallbackMethod           string  `url:"RecordingStatusCallbackMethod,omitempty"`
	RecordingStatusCallbackEvent            string  `url:"RecordingStatusCallbackEvent,omitempty"`
	SipAuthUsername                         string  `url:"SipAuthUsername,omitempty"`
	SipAuthPassword                         string  `url:"SipAuthPassword,omitempty"`
	Region                                  string  `url:"Region,omitempty"`
	ConferenceRecordingStatusCallback       string  `url:"ConferenceRecordingStatusCallback,omitempty"`
	ConferenceRecordingStatusCallbackMethod string  `url:"ConferenceRecordingStatusCallbackMethod,omitempty"`
	Coaching                                *bool   `url:"Coaching,omitempty"`
	CallSidToCoach                          string  `url:"CallSidToCoach,omitempty"`
}

// GetConference fetches details for a single conference instance
// https://www.twilio.com/docs/voice/api/conference-resource#fetch-a-conference-resource
func (twilio *Twilio) GetConference(conferenceSid string) (*Conference, *Exception, error) {
	res, err := twilio.get(twilio.buildUrl(fmt.Sprintf("Conferences/%s.json", conferenceSid)))
	if err != nil {
		return nil, nil, err
	}

	decoder := json.NewDecoder(res.Body)

	// handle NULL response
	if res.StatusCode != http.StatusOK {
		exception := new(Exception)
		err = decoder.Decode(exception)
		return nil, exception, err
	}

	conf := new(Conference)
	err = decoder.Decode(conf)
	return conf, nil, err
}

// UpdateConference to end it or play an announcement
// https://www.twilio.com/docs/voice/api/conference-resource#update-a-conference-resource
func (twilio *Twilio) UpdateConference(conferenceSid string, options *ConferenceOptions) (*Conference, *Exception, error) {
	form, err := query.Values(options)
	if err != nil {
		return nil, nil, err
	}

	res, err := twilio.post(form, twilio.buildUrl(fmt.Sprintf("Conferences/%s.json", conferenceSid)))
	if err != nil {
		return nil, nil, err
	}

	decoder := json.NewDecoder(res.Body)

	if res.StatusCode != http.StatusOK {
		exception := new(Exception)
		err = decoder.Decode(exception)
		return nil, exception, err
	}

	c := new(Conference)
	err = decoder.Decode(c)
	return c, nil, err
}

type getParticipantsResponse struct {
	Participants []*ConferenceParticipant `json:"participants"`
}

// GetConferenceParticipants fetches details for all conference participants resource
// https://www.twilio.com/docs/voice/api/conference-participant-resource#read-multiple-participant-resources
func (twilio *Twilio) GetConferenceParticipants(conferenceSid string) ([]*ConferenceParticipant, *Exception, error) {
	res, err := twilio.get(twilio.buildUrl(fmt.Sprintf("Conferences/%s/Participants.json", conferenceSid)))
	if err != nil {
		return nil, nil, err
	}

	decoder := json.NewDecoder(res.Body)

	// handle NULL response
	if res.StatusCode != http.StatusOK {
		exception := new(Exception)
		err = decoder.Decode(exception)
		return nil, exception, err
	}

	conf := new(getParticipantsResponse)
	err = decoder.Decode(conf)

	return conf.Participants, nil, err
}

// GetConferenceParticipant fetches details for a conference participant resource
// https://www.twilio.com/docs/voice/api/conference-participant-resource#fetch-a-participant-resource
func (twilio *Twilio) GetConferenceParticipant(conferenceSid, callSid string) (*ConferenceParticipant, *Exception, error) {
	res, err := twilio.get(twilio.buildUrl(fmt.Sprintf("Conferences/%s/Participants/%s.json", conferenceSid, callSid)))
	if err != nil {
		return nil, nil, err
	}

	decoder := json.NewDecoder(res.Body)

	// handle NULL response
	if res.StatusCode != http.StatusOK {
		exception := new(Exception)
		err = decoder.Decode(exception)
		return nil, exception, err
	}

	conf := new(ConferenceParticipant)
	err = decoder.Decode(conf)
	return conf, nil, err
}

// AddConferenceParticipant adds a Participant to a conference by dialing out a new call
// https://www.twilio.com/docs/voice/api/conference-participant-resource#create-a-participant-agent-conference-only
func (twilio *Twilio) AddConferenceParticipant(conferenceSid string, participant *ConferenceParticipantOptions) (*ConferenceParticipant, *Exception, error) {
	form, err := query.Values(participant)
	if err != nil {
		return nil, nil, err
	}

	res, err := twilio.post(form, twilio.buildUrl(fmt.Sprintf("Conferences/%s/Participants.json", conferenceSid)))
	if err != nil {
		return nil, nil, err
	}

	decoder := json.NewDecoder(res.Body)

	if res.StatusCode != http.StatusCreated {
		exception := new(Exception)
		err = decoder.Decode(exception)
		return nil, exception, err
	}

	conf := new(ConferenceParticipant)
	err = decoder.Decode(conf)
	return conf, nil, err
}

// UpdateConferenceParticipant
// https://www.twilio.com/docs/voice/api/conference-participant-resource#create-a-participant-agent-conference-only
func (twilio *Twilio) UpdateConferenceParticipant(conferenceSid string, callSid string, participant *ConferenceParticipantOptions) (*ConferenceParticipant, *Exception, error) {
	form, err := query.Values(participant)
	if err != nil {
		return nil, nil, err
	}

	res, err := twilio.post(form, twilio.buildUrl(fmt.Sprintf("Conferences/%s/Participants/%s.json", conferenceSid, callSid)))
	if err != nil {
		return nil, nil, err
	}

	decoder := json.NewDecoder(res.Body)

	if res.StatusCode != http.StatusOK {
		exception := new(Exception)
		err = decoder.Decode(exception)
		return nil, exception, err
	}

	p := new(ConferenceParticipant)
	err = decoder.Decode(p)
	return p, nil, err
}

// DeleteConferenceParticipant
func (twilio *Twilio) DeleteConferenceParticipant(conferenceSid string, callSid string) (*Exception, error) {
	res, err := twilio.delete(twilio.buildUrl(fmt.Sprintf("Conferences/%s/Participants/%s.json", conferenceSid, callSid)))
	if err != nil {
		return nil, err
	}

	if res.StatusCode != http.StatusOK {
		decoder := json.NewDecoder(res.Body)
		exception := new(Exception)
		err = decoder.Decode(exception)
		return exception, err
	}

	return nil, err
}
<|endoftext|>


<commit_before>// Copyright 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This test suite includes test cases to verify basic vDVS functionality
// in docker swarm mode.

// +build runonce

package e2e

import (
	"log"

	. "gopkg.in/check.v1"

	constant "github.com/vmware/docker-volume-vsphere/tests/constants/dockercli"
	"github.com/vmware/docker-volume-vsphere/tests/constants/properties"
	"github.com/vmware/docker-volume-vsphere/tests/utils/dockercli"
	"github.com/vmware/docker-volume-vsphere/tests/utils/esx"
	"github.com/vmware/docker-volume-vsphere/tests/utils/inputparams"
	"github.com/vmware/docker-volume-vsphere/tests/utils/misc"
	"github.com/vmware/docker-volume-vsphere/tests/utils/verification"
)

type SwarmTestSuite struct {
	esxName     string
	master      string
	worker1     string
	worker2     string
	swarmNodes  []string
	volumeName  string
	serviceName string
}

func (s *SwarmTestSuite) SetUpSuite(c *C) {
	s.esxName = inputparams.GetEsxIP()
	s.master = inputparams.GetSwarmManager1()
	s.worker1 = inputparams.GetSwarmWorker1()
	s.worker2 = inputparams.GetSwarmWorker2()
	s.swarmNodes = inputparams.GetSwarmNodes()

	// Verify if swarm cluster is already initialized
	out, err := dockercli.ListNodes(s.master)
	c.Assert(err, IsNil, Commentf(out))
}

func (s *SwarmTestSuite) SetUpTest(c *C) {
	s.volumeName = inputparams.GetUniqueVolumeName("swarm_test")
	s.serviceName = inputparams.GetUniqueServiceName("swarm_test")

	// Create the volume
	out, err := dockercli.CreateVolume(s.master, s.volumeName)
	c.Assert(err, IsNil, Commentf(out))

	status := verification.VerifyDetachedStatus(s.volumeName, s.master, s.esxName)
	c.Assert(status, Equals, true, Commentf("Volume %s is not detached", s.volumeName))
}

func (s *SwarmTestSuite) TearDownTest(c *C) {
	// Clean up the volume
	out, err := dockercli.DeleteVolume(s.master, s.volumeName)
	c.Assert(err, IsNil, Commentf(out))
}

var _ = Suite(&SwarmTestSuite{})

// Test vDVS usage during failover across different swarm nodes
//
// Test steps:
// 1. Create a service with volume mounted
// 2. Verify the service is up and running on one node
// 3. Verify one container is spawned
// 4. Verify the volume is in attached status
// 5. Write data to the volume
// 6. Shutdown the node on which the service is running
// 7. Verify the service is restarted on a different node
// 8. Verify the volume is in attached status
// 9. Verify the data from this node
// 10. Remove the service
// 11. Verify the service is gone
// 12. Verify the volume is in detached status
func (s *SwarmTestSuite) TestFailoverAcrossSwarmNodes(c *C) {
	misc.LogTestStart(c.TestName())

	const (
		testData = "Hello World!"
		testFile = "hello.txt"
		volPath  = "/vol"
	)

	// Create a swarm service that will be scheduled in the worker nodes only and will restart on failure automatically
	fullVolumeName := verification.GetFullVolumeName(s.master, s.volumeName)
	opts := "--mount type=volume,source=" + fullVolumeName + ",target=" + volPath + ",volume-driver=" + constant.VDVSPluginName + "--constraint node.role==worker --restart-condition on-failure" + constant.TestContainer
	out, err := dockercli.CreateService(s.master, s.serviceName, opts)
	c.Assert(err, IsNil, Commentf(out))

	status := verification.IsDockerServiceRunning(s.master, s.serviceName, 1)
	c.Assert(status, Equals, true, Commentf("Service %s is not running", s.serviceName))

	status, host := verification.IsDockerContainerRunning(s.swarmNodes, s.serviceName, 1)
	c.Assert(status, Equals, true, Commentf("Container %s is not running", s.serviceName))

	status = verification.VerifyAttachedStatus(s.volumeName, host, s.esxName)
	c.Assert(status, Equals, true, Commentf("Volume %s is not attached", s.volumeName))

	containerName, err := dockercli.GetContainerName(host, s.serviceName)
	log.Printf("ContainerName: [%s]\n", containerName)
	c.Assert(err, IsNil, Commentf("Failed to retrieve container name: %s", containerName))

	out, err = dockercli.WriteToContainer(host, containerName, volPath, testFile, testData)
	c.Assert(err, IsNil, Commentf(out))

	out, err = dockercli.ReadFromContainer(host, containerName, volPath, testFile)
	c.Assert(err, IsNil, Commentf(out))
	c.Assert(out, Equals, testData)

	// Power off the running worker node
	hostName := esx.RetrieveVMNameFromIP(host)
	esx.PowerOffVM(hostName)

	isStatusChanged := esx.WaitForExpectedState(esx.GetVMPowerState, hostName, properties.PowerOffState)
	c.Assert(isStatusChanged, Equals, true, Commentf("VM [%s] should be powered off state", hostName))

	status = verification.IsDockerServiceRunning(s.master, s.serviceName, 1)
	c.Assert(status, Equals, true, Commentf("Service %s is not running", s.serviceName))

	// Only 2 worker nodes - running node is down, the container should be failed over to the other node
	var otherWorker string
	if host == s.worker1 {
		otherWorker = s.worker2
	} else {
		otherWorker = s.worker1
	}
	status, host = verification.IsDockerContainerRunning([]string{otherWorker}, s.serviceName, 1)
	c.Assert(status, Equals, true, Commentf("Container %s is not running", s.serviceName))

	status = verification.VerifyAttachedStatus(s.volumeName, host, s.esxName)
	c.Assert(status, Equals, true, Commentf("Volume %s is not attached", s.volumeName))

	containerName, err = dockercli.GetContainerName(host, s.serviceName)
	c.Assert(err, IsNil, Commentf("Failed to retrieve container name: %s", containerName))

	// TODO: this verification does not pass during auto run - need further investigation
	// out, err = dockercli.ReadFromContainer(host, containerName, volPath, testFile)
	// c.Assert(err, IsNil, Commentf(out))
	// c.Assert(out, Equals, testData)

	// Power on the worker node
	esx.PowerOnVM(hostName)
	isStatusChanged = esx.WaitForExpectedState(esx.GetVMPowerState, hostName, properties.PowerOnState)
	c.Assert(isStatusChanged, Equals, true, Commentf("VM [%s] should be powered on state", hostName))

	out, err = dockercli.RemoveService(s.master, s.serviceName)
	c.Assert(err, IsNil, Commentf(out))

	out, err = dockercli.ListService(s.master, s.serviceName)
	c.Assert(err, NotNil, Commentf("Expected error does not happen"))

	status = verification.VerifyDetachedStatus(s.volumeName, s.master, s.esxName)
	c.Assert(status, Equals, true, Commentf("Volume %s is still attached", s.volumeName))

	misc.LogTestEnd(c.TestName())
}

// Test vDVS usage during failover across different service replicas
//
// Note: Swarm scaled replica feature doesn't support stateful
// applications. So at one time the volume can be attached to
// only one container.
//
// Test steps:
// 1. Create a docker service with replicas setting to 1
// 2. Verify the service is up and running with one node
// 3. Verify one container is spawned
// 5. Verify the volume is in attached status
// 6. Scale the service to set replica numbers to 2
// 7. Verify the service is up and running with two nodes
// 8. Verify 2 containers are spawned
// 9. Stop one node of the service
// 10. Verify the service is still running with two nodes
// 11. Verify there are still 2 containers up and running
// 12. Verify the volume is in attached status
// 13. Delete the volume - expect fail
// 14. Remove the service
// 15. Verify the service is gone
// 16. Verify the volume is in detached status
func (s *SwarmTestSuite) TestFailoverAcrossReplicas(c *C) {
	misc.LogTestStart(c.TestName())

	fullVolumeName := verification.GetFullVolumeName(s.master, s.volumeName)
	opts := "--replicas 1 --mount type=volume,source=" + fullVolumeName + ",target=/vol,volume-driver=" + constant.VDVSPluginName + constant.TestContainer
	out, err := dockercli.CreateService(s.master, s.serviceName, opts)
	c.Assert(err, IsNil, Commentf(out))

	status := verification.IsDockerServiceRunning(s.master, s.serviceName, 1)
	c.Assert(status, Equals, true, Commentf("Service %s is not running", s.serviceName))

	status, host := verification.IsDockerContainerRunning(s.swarmNodes, s.serviceName, 1)
	c.Assert(status, Equals, true, Commentf("Container %s is not running", s.serviceName))

	status = verification.VerifyAttachedStatus(s.volumeName, host, s.esxName)
	c.Assert(status, Equals, true, Commentf("Volume %s is not attached", s.volumeName))

	out, err = dockercli.ScaleService(s.master, s.serviceName, 2)
	c.Assert(err, IsNil, Commentf(out))

	status = verification.IsDockerServiceRunning(s.master, s.serviceName, 2)
	c.Assert(status, Equals, true, Commentf("Service %s is not running", s.serviceName))

	status, host = verification.IsDockerContainerRunning(s.swarmNodes, s.serviceName, 2)
	c.Assert(status, Equals, true, Commentf("Container %s is not running on any hosts", s.serviceName))

	containerName, err := dockercli.GetContainerName(host, s.serviceName+".1")
	c.Assert(err, IsNil, Commentf("Failed to retrieve container name: %s", containerName))
	out, err = dockercli.StopService(host, containerName)
	c.Assert(err, IsNil, Commentf(out))

	status = verification.IsDockerServiceRunning(s.master, s.serviceName, 2)
	c.Assert(status, Equals, true, Commentf("Service %s is not running", s.serviceName))

	status, host = verification.IsDockerContainerRunning(s.swarmNodes, s.serviceName, 2)
	c.Assert(status, Equals, true, Commentf("Container %s is not running", s.serviceName))

	status = verification.VerifyAttachedStatus(s.volumeName, host, s.esxName)
	c.Assert(status, Equals, true, Commentf("Volume %s is not attached", s.volumeName))

	out, err = dockercli.DeleteVolume(s.master, s.volumeName)
	c.Assert(err, NotNil, Commentf("Expected error does not happen"))

	out, err = dockercli.RemoveService(s.master, s.serviceName)
	c.Assert(err, IsNil, Commentf(out))

	out, err = dockercli.ListService(s.master, s.serviceName)
	c.Assert(err, NotNil, Commentf("Expected error does not happen"))

	status = verification.VerifyDetachedStatus(s.volumeName, s.master, s.esxName)
	c.Assert(status, Equals, true, Commentf("Volume %s is still attached", s.volumeName))

	misc.LogTestEnd(c.TestName())
}
<commit_msg>Check if vDVS is running after vm power-on (#1519)<commit_after>// Copyright 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This test suite includes test cases to verify basic vDVS functionality
// in docker swarm mode.

// +build runonce

package e2e

import (
	"log"

	. "gopkg.in/check.v1"

	constant "github.com/vmware/docker-volume-vsphere/tests/constants/dockercli"
	"github.com/vmware/docker-volume-vsphere/tests/constants/properties"
	"github.com/vmware/docker-volume-vsphere/tests/utils/dockercli"
	"github.com/vmware/docker-volume-vsphere/tests/utils/esx"
	"github.com/vmware/docker-volume-vsphere/tests/utils/inputparams"
	"github.com/vmware/docker-volume-vsphere/tests/utils/misc"
	"github.com/vmware/docker-volume-vsphere/tests/utils/verification"
)

type SwarmTestSuite struct {
	esxName     string
	master      string
	worker1     string
	worker2     string
	swarmNodes  []string
	volumeName  string
	serviceName string
}

func (s *SwarmTestSuite) SetUpSuite(c *C) {
	s.esxName = inputparams.GetEsxIP()
	s.master = inputparams.GetSwarmManager1()
	s.worker1 = inputparams.GetSwarmWorker1()
	s.worker2 = inputparams.GetSwarmWorker2()
	s.swarmNodes = inputparams.GetSwarmNodes()

	// Verify if swarm cluster is already initialized
	out, err := dockercli.ListNodes(s.master)
	c.Assert(err, IsNil, Commentf(out))
}

func (s *SwarmTestSuite) SetUpTest(c *C) {
	s.volumeName = inputparams.GetUniqueVolumeName("swarm_test")
	s.serviceName = inputparams.GetUniqueServiceName("swarm_test")

	// Create the volume
	out, err := dockercli.CreateVolume(s.master, s.volumeName)
	c.Assert(err, IsNil, Commentf(out))

	status := verification.VerifyDetachedStatus(s.volumeName, s.master, s.esxName)
	c.Assert(status, Equals, true, Commentf("Volume %s is not detached", s.volumeName))
}

func (s *SwarmTestSuite) TearDownTest(c *C) {
	// Clean up the volume
	out, err := dockercli.DeleteVolume(s.master, s.volumeName)
	c.Assert(err, IsNil, Commentf(out))
}

var _ = Suite(&SwarmTestSuite{})

// Test vDVS usage during failover across different swarm nodes
//
// Test steps:
// 1. Create a service with volume mounted
// 2. Verify the service is up and running on one node
// 3. Verify one container is spawned
// 4. Verify the volume is in attached status
// 5. Write data to the volume
// 6. Shutdown the node on which the service is running
// 7. Verify the service is restarted on a different node
// 8. Verify the volume is in attached status
// 9. Verify the data from this node
// 10. Remove the service
// 11. Verify the service is gone
// 12. Verify the volume is in detached status
func (s *SwarmTestSuite) TestFailoverAcrossSwarmNodes(c *C) {
	misc.LogTestStart(c.TestName())

	const (
		testData = "Hello World!"
		testFile = "hello.txt"
		volPath  = "/vol"
	)

	// Create a swarm service that will be scheduled in the worker nodes only and will restart on failure automatically
	fullVolumeName := verification.GetFullVolumeName(s.master, s.volumeName)
	opts := "--mount type=volume,source=" + fullVolumeName + ",target=" + volPath + ",volume-driver=" + constant.VDVSPluginName + "--constraint node.role==worker --restart-condition on-failure" + constant.TestContainer
	out, err := dockercli.CreateService(s.master, s.serviceName, opts)
	c.Assert(err, IsNil, Commentf(out))

	status := verification.IsDockerServiceRunning(s.master, s.serviceName, 1)
	c.Assert(status, Equals, true, Commentf("Service %s is not running", s.serviceName))

	status, host := verification.IsDockerContainerRunning(s.swarmNodes, s.serviceName, 1)
	c.Assert(status, Equals, true, Commentf("Container %s is not running", s.serviceName))

	status = verification.VerifyAttachedStatus(s.volumeName, host, s.esxName)
	c.Assert(status, Equals, true, Commentf("Volume %s is not attached", s.volumeName))

	containerName, err := dockercli.GetContainerName(host, s.serviceName)
	log.Printf("ContainerName: [%s]\n", containerName)
	c.Assert(err, IsNil, Commentf("Failed to retrieve container name: %s", containerName))

	out, err = dockercli.WriteToContainer(host, containerName, volPath, testFile, testData)
	c.Assert(err, IsNil, Commentf(out))

	out, err = dockercli.ReadFromContainer(host, containerName, volPath, testFile)
	c.Assert(err, IsNil, Commentf(out))
	c.Assert(out, Equals, testData)

	// Power off the running worker node
	hostName := esx.RetrieveVMNameFromIP(host)
	esx.PowerOffVM(hostName)

	isStatusChanged := esx.WaitForExpectedState(esx.GetVMPowerState, hostName, properties.PowerOffState)
	c.Assert(isStatusChanged, Equals, true, Commentf("VM [%s] should be powered off state", hostName))

	status = verification.IsDockerServiceRunning(s.master, s.serviceName, 1)
	c.Assert(status, Equals, true, Commentf("Service %s is not running", s.serviceName))

	// Only 2 worker nodes - running node is down, the container should be failed over to the other node
	var otherWorker string
	if host == s.worker1 {
		otherWorker = s.worker2
	} else {
		otherWorker = s.worker1
	}
	status, host = verification.IsDockerContainerRunning([]string{otherWorker}, s.serviceName, 1)
	c.Assert(status, Equals, true, Commentf("Container %s is not running", s.serviceName))

	status = verification.VerifyAttachedStatus(s.volumeName, host, s.esxName)
	c.Assert(status, Equals, true, Commentf("Volume %s is not attached", s.volumeName))

	containerName, err = dockercli.GetContainerName(host, s.serviceName)
	c.Assert(err, IsNil, Commentf("Failed to retrieve container name: %s", containerName))

	// TODO: this verification does not pass during auto run - need further investigation
	// out, err = dockercli.ReadFromContainer(host, containerName, volPath, testFile)
	// c.Assert(err, IsNil, Commentf(out))
	// c.Assert(out, Equals, testData)

	// Power on the worker node
	esx.PowerOnVM(hostName)
	isStatusChanged = esx.WaitForExpectedState(esx.GetVMPowerState, hostName, properties.PowerOnState)
	c.Assert(isStatusChanged, Equals, true, Commentf("VM [%s] should be powered on state", hostName))
	c.Assert(verification.IsVDVSIsRunning(host), Equals, true, Commentf("vDVS [%s] is not running", hostName))

	out, err = dockercli.RemoveService(s.master, s.serviceName)
	c.Assert(err, IsNil, Commentf(out))

	out, err = dockercli.ListService(s.master, s.serviceName)
	c.Assert(err, NotNil, Commentf("Expected error does not happen"))

	status = verification.VerifyDetachedStatus(s.volumeName, s.master, s.esxName)
	c.Assert(status, Equals, true, Commentf("Volume %s is still attached", s.volumeName))

	misc.LogTestEnd(c.TestName())
}

// Test vDVS usage during failover across different service replicas
//
// Note: Swarm scaled replica feature doesn't support stateful
// applications. So at one time the volume can be attached to
// only one container.
//
// Test steps:
// 1. Create a docker service with replicas setting to 1
// 2. Verify the service is up and running with one node
// 3. Verify one container is spawned
// 5. Verify the volume is in attached status
// 6. Scale the service to set replica numbers to 2
// 7. Verify the service is up and running with two nodes
// 8. Verify 2 containers are spawned
// 9. Stop one node of the service
// 10. Verify the service is still running with two nodes
// 11. Verify there are still 2 containers up and running
// 12. Verify the volume is in attached status
// 13. Delete the volume - expect fail
// 14. Remove the service
// 15. Verify the service is gone
// 16. Verify the volume is in detached status
func (s *SwarmTestSuite) TestFailoverAcrossReplicas(c *C) {
	misc.LogTestStart(c.TestName())

	fullVolumeName := verification.GetFullVolumeName(s.master, s.volumeName)
	opts := "--replicas 1 --mount type=volume,source=" + fullVolumeName + ",target=/vol,volume-driver=" + constant.VDVSPluginName + constant.TestContainer
	out, err := dockercli.CreateService(s.master, s.serviceName, opts)
	c.Assert(err, IsNil, Commentf(out))

	status := verification.IsDockerServiceRunning(s.master, s.serviceName, 1)
	c.Assert(status, Equals, true, Commentf("Service %s is not running", s.serviceName))

	status, host := verification.IsDockerContainerRunning(s.swarmNodes, s.serviceName, 1)
	c.Assert(status, Equals, true, Commentf("Container %s is not running", s.serviceName))

	status = verification.VerifyAttachedStatus(s.volumeName, host, s.esxName)
	c.Assert(status, Equals, true, Commentf("Volume %s is not attached", s.volumeName))

	out, err = dockercli.ScaleService(s.master, s.serviceName, 2)
	c.Assert(err, IsNil, Commentf(out))

	status = verification.IsDockerServiceRunning(s.master, s.serviceName, 2)
	c.Assert(status, Equals, true, Commentf("Service %s is not running", s.serviceName))

	status, host = verification.IsDockerContainerRunning(s.swarmNodes, s.serviceName, 2)
	c.Assert(status, Equals, true, Commentf("Container %s is not running on any hosts", s.serviceName))

	containerName, err := dockercli.GetContainerName(host, s.serviceName+".1")
	c.Assert(err, IsNil, Commentf("Failed to retrieve container name: %s", containerName))
	out, err = dockercli.StopService(host, containerName)
	c.Assert(err, IsNil, Commentf(out))

	status = verification.IsDockerServiceRunning(s.master, s.serviceName, 2)
	c.Assert(status, Equals, true, Commentf("Service %s is not running", s.serviceName))

	status, host = verification.IsDockerContainerRunning(s.swarmNodes, s.serviceName, 2)
	c.Assert(status, Equals, true, Commentf("Container %s is not running", s.serviceName))

	status = verification.VerifyAttachedStatus(s.volumeName, host, s.esxName)
	c.Assert(status, Equals, true, Commentf("Volume %s is not attached", s.volumeName))

	out, err = dockercli.DeleteVolume(s.master, s.volumeName)
	c.Assert(err, NotNil, Commentf("Expected error does not happen"))

	out, err = dockercli.RemoveService(s.master, s.serviceName)
	c.Assert(err, IsNil, Commentf(out))

	out, err = dockercli.ListService(s.master, s.serviceName)
	c.Assert(err, NotNil, Commentf("Expected error does not happen"))

	status = verification.VerifyDetachedStatus(s.volumeName, s.master, s.esxName)
	c.Assert(status, Equals, true, Commentf("Volume %s is still attached", s.volumeName))

	misc.LogTestEnd(c.TestName())
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuse

import (
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"runtime"
	"sync"
	"syscall"

	"golang.org/x/net/context"

	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/internal/buffer"
	"github.com/jacobsa/fuse/internal/fusekernel"
	"github.com/jacobsa/fuse/internal/fuseshim"
)

// Ask the Linux kernel for larger read requests.
//
// As of 2015-03-26, the behavior in the kernel is:
//
//  *  (http://goo.gl/bQ1f1i, http://goo.gl/HwBrR6) Set the local variable
//     ra_pages to be init_response->max_readahead divided by the page size.
//
//  *  (http://goo.gl/gcIsSh, http://goo.gl/LKV2vA) Set
//     backing_dev_info::ra_pages to the min of that value and what was sent
//     in the request's max_readahead field.
//
//  *  (http://goo.gl/u2SqzH) Use backing_dev_info::ra_pages when deciding
//     how much to read ahead.
//
//  *  (http://goo.gl/JnhbdL) Don't read ahead at all if that field is zero.
//
// Reading a page at a time is a drag. Ask for a larger size.
const maxReadahead = 1 << 20

// A connection to the fuse kernel process.
type Connection struct {
	debugLogger *log.Logger
	errorLogger *log.Logger

	// The device through which we're talking to the kernel, and the protocol
	// version that we're using to talk to it.
	dev      *os.File
	protocol fusekernel.Protocol

	// The context from which all op contexts inherit.
	parentCtx context.Context

	// For logging purposes only.
	nextOpID uint32

	mu sync.Mutex

	// A map from fuse "unique" request ID (*not* the op ID for logging used
	// above) to a function that cancel's its associated context.
	//
	// GUARDED_BY(mu)
	cancelFuncs map[uint64]func()
}

// Create a connection wrapping the supplied file descriptor connected to the
// kernel. You must eventually call c.close().
//
// The loggers may be nil.
func newConnection(
	parentCtx context.Context,
	debugLogger *log.Logger,
	errorLogger *log.Logger,
	dev *os.File) (c *Connection, err error) {
	// Create an initialized a wrapped fuseshim connection.
	wrapped := &fuseshim.Conn{
		Dev: dev,
	}

	err = fuseshim.InitMount(wrapped, maxReadahead, 0)
	if err != nil {
		err = fmt.Errorf("fuseshim.InitMount: %v", err)
		return
	}

	// Create an object wrapping it.
	c = &Connection{
		debugLogger: debugLogger,
		errorLogger: errorLogger,
		dev:         wrapped.Dev,
		protocol:    wrapped.Protocol(),
		parentCtx:   parentCtx,
		cancelFuncs: make(map[uint64]func()),
	}

	return
}

// Log information for an operation with the given ID. calldepth is the depth
// to use when recovering file:line information with runtime.Caller.
func (c *Connection) debugLog(
	opID uint32,
	calldepth int,
	format string,
	v ...interface{}) {
	if c.debugLogger == nil {
		return
	}

	// Get file:line info.
	var file string
	var line int
	var ok bool

	_, file, line, ok = runtime.Caller(calldepth)
	if !ok {
		file = "???"
	}

	fileLine := fmt.Sprintf("%v:%v", path.Base(file), line)

	// Format the actual message to be printed.
	msg := fmt.Sprintf(
		"Op 0x%08x %24s] %v",
		opID,
		fileLine,
		fmt.Sprintf(format, v...))

	// Print it.
	c.debugLogger.Println(msg)
}

// LOCKS_EXCLUDED(c.mu)
func (c *Connection) recordCancelFunc(
	fuseID uint64,
	f func()) {
	c.mu.Lock()
	defer c.mu.Unlock()

	if _, ok := c.cancelFuncs[fuseID]; ok {
		panic(fmt.Sprintf("Already have cancel func for request %v", fuseID))
	}

	c.cancelFuncs[fuseID] = f
}

// Set up state for an op that is about to be returned to the user, given its
// underlying fuse opcode and request ID.
//
// Return a context that should be used for the op.
//
// LOCKS_EXCLUDED(c.mu)
func (c *Connection) beginOp(
	opCode uint32,
	fuseID uint64) (ctx context.Context) {
	// Start with the parent context.
	ctx = c.parentCtx

	// Set up a cancellation function.
	//
	// Special case: On Darwin, osxfuse aggressively reuses "unique" request IDs.
	// This matters for Forget requests, which have no reply associated and
	// therefore have IDs that are immediately eligible for reuse. For these, we
	// should not record any state keyed on their ID.
	//
	// Cf. https://github.com/osxfuse/osxfuse/issues/208
	if opCode != fusekernel.OpForget {
		var cancel func()
		ctx, cancel = context.WithCancel(ctx)
		c.recordCancelFunc(fuseID, cancel)
	}

	return
}

// Clean up all state associated with an op to which the user has responded,
// given its underlying fuse opcode and request ID. This must be called before
// a response is sent to the kernel, to avoid a race where the request's ID
// might be reused by osxfuse.
//
// LOCKS_EXCLUDED(c.mu)
func (c *Connection) finishOp(
	opCode uint32,
	fuseID uint64) {
	c.mu.Lock()
	defer c.mu.Unlock()

	// Even though the op is finished, context.WithCancel requires us to arrange
	// for the cancellation function to be invoked. We also must remove it from
	// our map.
	//
	// Special case: we don't do this for Forget requests. See the note in
	// beginOp above.
	if opCode != fusekernel.OpForget {
		cancel, ok := c.cancelFuncs[fuseID]
		if !ok {
			panic(fmt.Sprintf("Unknown request ID in finishOp: %v", fuseID))
		}

		cancel()
		delete(c.cancelFuncs, fuseID)
	}
}

// LOCKS_EXCLUDED(c.mu)
func (c *Connection) handleInterrupt(fuseID uint64) {
	c.mu.Lock()
	defer c.mu.Unlock()

	// NOTE(jacobsa): fuse.txt in the Linux kernel documentation
	// (https://goo.gl/H55Dnr) defines the kernel <-> userspace protocol for
	// interrupts.
	//
	// In particular, my reading of it is that an interrupt request cannot be
	// delivered to userspace before the original request. The part about the
	// race and EAGAIN appears to be aimed at userspace programs that
	// concurrently process requests (cf. http://goo.gl/BES2rs).
	//
	// So in this method if we can't find the ID to be interrupted, it means that
	// the request has already been replied to.
	//
	// Cf. https://github.com/osxfuse/osxfuse/issues/208
	// Cf. http://comments.gmane.org/gmane.comp.file-systems.fuse.devel/14675
	cancel, ok := c.cancelFuncs[fuseID]
	if !ok {
		return
	}

	cancel()
}

func (c *Connection) allocateInMessage() (m *buffer.InMessage) {
	// TODO(jacobsa): Use a freelist.
	m = new(buffer.InMessage)
	return
}

func (c *Connection) destroyInMessage(m *buffer.InMessage) {
	// TODO(jacobsa): Use a freelist.
}

// Read the next message from the kernel. The message must later be destroyed
// using destroyInMessage.
func (c *Connection) readMessage() (m *buffer.InMessage, err error) {
	// Allocate a message.
	m = c.allocateInMessage()

	// Loop past transient errors.
	for {
		// Attempt a reaed.
		err = m.Init(c.dev)

		// Special cases:
		//
		//  *  ENODEV means fuse has hung up.
		//
		//  *  EINTR means we should try again. (This seems to happen often on
		//     OS X, cf. http://golang.org/issue/11180)
		//
		if pe, ok := err.(*os.PathError); ok {
			switch pe.Err {
			case syscall.ENODEV:
				err = io.EOF

			case syscall.EINTR:
				err = nil
				continue
			}
		}

		if err != nil {
			c.destroyInMessage(m)
			m = nil
			return
		}

		return
	}
}

// Write the supplied message to the kernel.
func (c *Connection) writeMessage(msg []byte) (err error) {
	// Avoid the retry loop in os.File.Write.
	n, err := syscall.Write(int(c.dev.Fd()), msg)
	if err != nil {
		return
	}

	if n != len(msg) {
		err = fmt.Errorf("Wrote %d bytes; expected %d", n, len(msg))
		return
	}

	return
}

// Read the next op from the kernel process. Return io.EOF if the kernel has
// closed the connection.
//
// This function delivers ops in exactly the order they are received from
// /dev/fuse. It must not be called multiple times concurrently.
//
// LOCKS_EXCLUDED(c.mu)
func (c *Connection) ReadOp() (op fuseops.Op, err error) {
	// Keep going until we find a request we know how to convert.
	for {
		// Read the next message from the kernel.
		var m *buffer.InMessage
		m, err = c.readMessage()
		if err != nil {
			return
		}

		// Choose an ID for this operation for the purposes of logging.
		opID := c.nextOpID
		c.nextOpID++

		// Set up op dependencies.
		opCtx := c.beginOp(m.Header().Opcode, m.Header().Unique)

		var debugLogForOp func(int, string, ...interface{})
		if c.debugLogger != nil {
			debugLogForOp = func(calldepth int, format string, v ...interface{}) {
				c.debugLog(opID, calldepth+1, format, v...)
			}
		}

		sendReply := func(
			op fuseops.Op,
			fuseID uint64,
			replyMsg []byte,
			opErr error) (err error) {
			// Make sure we destroy the message, as required by readMessage.
			defer c.destroyInMessage(m)

			// Clean up state for this op.
			c.finishOp(m.Header().Opcode, m.Header().Unique)

			// Debug logging
			if c.debugLogger != nil {
				if opErr == nil {
					op.Logf("-> OK: %s", op.DebugString())
				} else {
					op.Logf("-> error: %v", opErr)
				}
			}

			// Error logging
			if opErr != nil && c.errorLogger != nil {
				c.errorLogger.Printf("(%s) error: %v", op.ShortDesc(), opErr)
			}

			// Send the reply to the kernel.
			err = c.writeMessage(replyMsg)
			if err != nil {
				err = fmt.Errorf("writeMessage: %v", err)
				return
			}

			return
		}

		// Convert the message to an Op.
		op, err = fuseops.Convert(
			opCtx,
			m,
			c.protocol,
			debugLogForOp,
			c.errorLogger,
			sendReply)

		if err != nil {
			err = fmt.Errorf("fuseops.Convert: %v", err)
			return
		}

		// Log the receipt of the operation.
		c.debugLog(opID, 1, "<- %v", op.ShortDesc())

		// Special case: responding to statfs is required to make mounting work on
		// OS X. We don't currently expose the capability for the file system to
		// intercept this.
		if _, ok := op.(*fuseops.InternalStatFSOp); ok {
			op.Respond(nil)
			continue
		}

		// Special case: handle interrupt requests.
		if interruptOp, ok := op.(*fuseops.InternalInterruptOp); ok {
			c.handleInterrupt(interruptOp.FuseID)
			continue
		}

		return
	}
}

// Close the connection. Must not be called until operations that were read
// from the connection have been responded to.
func (c *Connection) close() (err error) {
	// Posix doesn't say that close can be called concurrently with read or
	// write, but luckily we exclude the possibility of a race by requiring the
	// user to respond to all ops first.
	err = c.dev.Close()
	return
}
<commit_msg>Connection.Init<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuse

import (
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"runtime"
	"sync"
	"syscall"

	"golang.org/x/net/context"

	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/internal/buffer"
	"github.com/jacobsa/fuse/internal/fusekernel"
)

// Ask the Linux kernel for larger read requests.
//
// As of 2015-03-26, the behavior in the kernel is:
//
//  *  (http://goo.gl/bQ1f1i, http://goo.gl/HwBrR6) Set the local variable
//     ra_pages to be init_response->max_readahead divided by the page size.
//
//  *  (http://goo.gl/gcIsSh, http://goo.gl/LKV2vA) Set
//     backing_dev_info::ra_pages to the min of that value and what was sent
//     in the request's max_readahead field.
//
//  *  (http://goo.gl/u2SqzH) Use backing_dev_info::ra_pages when deciding
//     how much to read ahead.
//
//  *  (http://goo.gl/JnhbdL) Don't read ahead at all if that field is zero.
//
// Reading a page at a time is a drag. Ask for a larger size.
const maxReadahead = 1 << 20

// A connection to the fuse kernel process.
type Connection struct {
	debugLogger *log.Logger
	errorLogger *log.Logger

	// The device through which we're talking to the kernel, and the protocol
	// version that we're using to talk to it.
	dev      *os.File
	protocol fusekernel.Protocol

	// The context from which all op contexts inherit.
	parentCtx context.Context

	// For logging purposes only.
	nextOpID uint32

	mu sync.Mutex

	// A map from fuse "unique" request ID (*not* the op ID for logging used
	// above) to a function that cancel's its associated context.
	//
	// GUARDED_BY(mu)
	cancelFuncs map[uint64]func()
}

// Create a connection wrapping the supplied file descriptor connected to the
// kernel. You must eventually call c.close().
//
// The loggers may be nil.
func newConnection(
	parentCtx context.Context,
	debugLogger *log.Logger,
	errorLogger *log.Logger,
	dev *os.File) (c *Connection, err error) {
	c = &Connection{
		debugLogger: debugLogger,
		errorLogger: errorLogger,
		dev:         dev,
		parentCtx:   parentCtx,
		cancelFuncs: make(map[uint64]func()),
	}

	// Initialize.
	err = c.Init()
	if err != nil {
		c.close()
		err = fmt.Errorf("Init: %v", err)
		return
	}

	return
}

// Do the work necessary to cause the mount process to complete.
func (c *Connection) Init() (err error) {
	// Read the init op.
	op, err := c.ReadOp()
	if err != nil {
		err = fmt.Errorf("Reading init op: %v", err)
		return
	}

	initOp, ok := op.(*fuseops.InitOp)
	if !ok {
		err = fmt.Errorf("Expected *fuseops.InitOp, got %T", op)
		return
	}

	// Make sure the protocol version spoken by the kernel is new enough.
	min := fusekernel.Protocol{
		fusekernel.ProtoVersionMinMajor,
		fusekernel.ProtoVersionMinMinor,
	}

	if initOp.Kernel.LT(min) {
		initOp.Respond(syscall.EPROTO)
		err = fmt.Errorf("Version too old: %v", initOp.Kernel)
		return
	}

	// Downgrade our protocol if necessary.
	c.protocol = fusekernel.Protocol{
		fusekernel.ProtoVersionMaxMajor,
		fusekernel.ProtoVersionMaxMinor,
	}

	if initOp.Kernel.LT(c.protocol) {
		c.protocol = r.Kernel
	}

	// Respond to the init op.
	initOp.Library = c.protocol
	initOp.MaxReadahead = maxReadahead
	initOp.MaxWrite = buffer.MaxWriteSize
	initOp.Flags = fusekernel.InitBigWrites
	initOp.Respond(nil)

	return
}

// Log information for an operation with the given ID. calldepth is the depth
// to use when recovering file:line information with runtime.Caller.
func (c *Connection) debugLog(
	opID uint32,
	calldepth int,
	format string,
	v ...interface{}) {
	if c.debugLogger == nil {
		return
	}

	// Get file:line info.
	var file string
	var line int
	var ok bool

	_, file, line, ok = runtime.Caller(calldepth)
	if !ok {
		file = "???"
	}

	fileLine := fmt.Sprintf("%v:%v", path.Base(file), line)

	// Format the actual message to be printed.
	msg := fmt.Sprintf(
		"Op 0x%08x %24s] %v",
		opID,
		fileLine,
		fmt.Sprintf(format, v...))

	// Print it.
	c.debugLogger.Println(msg)
}

// LOCKS_EXCLUDED(c.mu)
func (c *Connection) recordCancelFunc(
	fuseID uint64,
	f func()) {
	c.mu.Lock()
	defer c.mu.Unlock()

	if _, ok := c.cancelFuncs[fuseID]; ok {
		panic(fmt.Sprintf("Already have cancel func for request %v", fuseID))
	}

	c.cancelFuncs[fuseID] = f
}

// Set up state for an op that is about to be returned to the user, given its
// underlying fuse opcode and request ID.
//
// Return a context that should be used for the op.
//
// LOCKS_EXCLUDED(c.mu)
func (c *Connection) beginOp(
	opCode uint32,
	fuseID uint64) (ctx context.Context) {
	// Start with the parent context.
	ctx = c.parentCtx

	// Set up a cancellation function.
	//
	// Special case: On Darwin, osxfuse aggressively reuses "unique" request IDs.
	// This matters for Forget requests, which have no reply associated and
	// therefore have IDs that are immediately eligible for reuse. For these, we
	// should not record any state keyed on their ID.
	//
	// Cf. https://github.com/osxfuse/osxfuse/issues/208
	if opCode != fusekernel.OpForget {
		var cancel func()
		ctx, cancel = context.WithCancel(ctx)
		c.recordCancelFunc(fuseID, cancel)
	}

	return
}

// Clean up all state associated with an op to which the user has responded,
// given its underlying fuse opcode and request ID. This must be called before
// a response is sent to the kernel, to avoid a race where the request's ID
// might be reused by osxfuse.
//
// LOCKS_EXCLUDED(c.mu)
func (c *Connection) finishOp(
	opCode uint32,
	fuseID uint64) {
	c.mu.Lock()
	defer c.mu.Unlock()

	// Even though the op is finished, context.WithCancel requires us to arrange
	// for the cancellation function to be invoked. We also must remove it from
	// our map.
	//
	// Special case: we don't do this for Forget requests. See the note in
	// beginOp above.
	if opCode != fusekernel.OpForget {
		cancel, ok := c.cancelFuncs[fuseID]
		if !ok {
			panic(fmt.Sprintf("Unknown request ID in finishOp: %v", fuseID))
		}

		cancel()
		delete(c.cancelFuncs, fuseID)
	}
}

// LOCKS_EXCLUDED(c.mu)
func (c *Connection) handleInterrupt(fuseID uint64) {
	c.mu.Lock()
	defer c.mu.Unlock()

	// NOTE(jacobsa): fuse.txt in the Linux kernel documentation
	// (https://goo.gl/H55Dnr) defines the kernel <-> userspace protocol for
	// interrupts.
	//
	// In particular, my reading of it is that an interrupt request cannot be
	// delivered to userspace before the original request. The part about the
	// race and EAGAIN appears to be aimed at userspace programs that
	// concurrently process requests (cf. http://goo.gl/BES2rs).
	//
	// So in this method if we can't find the ID to be interrupted, it means that
	// the request has already been replied to.
	//
	// Cf. https://github.com/osxfuse/osxfuse/issues/208
	// Cf. http://comments.gmane.org/gmane.comp.file-systems.fuse.devel/14675
	cancel, ok := c.cancelFuncs[fuseID]
	if !ok {
		return
	}

	cancel()
}

func (c *Connection) allocateInMessage() (m *buffer.InMessage) {
	// TODO(jacobsa): Use a freelist.
	m = new(buffer.InMessage)
	return
}

func (c *Connection) destroyInMessage(m *buffer.InMessage) {
	// TODO(jacobsa): Use a freelist.
}

// Read the next message from the kernel. The message must later be destroyed
// using destroyInMessage.
func (c *Connection) readMessage() (m *buffer.InMessage, err error) {
	// Allocate a message.
	m = c.allocateInMessage()

	// Loop past transient errors.
	for {
		// Attempt a reaed.
		err = m.Init(c.dev)

		// Special cases:
		//
		//  *  ENODEV means fuse has hung up.
		//
		//  *  EINTR means we should try again. (This seems to happen often on
		//     OS X, cf. http://golang.org/issue/11180)
		//
		if pe, ok := err.(*os.PathError); ok {
			switch pe.Err {
			case syscall.ENODEV:
				err = io.EOF

			case syscall.EINTR:
				err = nil
				continue
			}
		}

		if err != nil {
			c.destroyInMessage(m)
			m = nil
			return
		}

		return
	}
}

// Write the supplied message to the kernel.
func (c *Connection) writeMessage(msg []byte) (err error) {
	// Avoid the retry loop in os.File.Write.
	n, err := syscall.Write(int(c.dev.Fd()), msg)
	if err != nil {
		return
	}

	if n != len(msg) {
		err = fmt.Errorf("Wrote %d bytes; expected %d", n, len(msg))
		return
	}

	return
}

// Read the next op from the kernel process. Return io.EOF if the kernel has
// closed the connection.
//
// This function delivers ops in exactly the order they are received from
// /dev/fuse. It must not be called multiple times concurrently.
//
// LOCKS_EXCLUDED(c.mu)
func (c *Connection) ReadOp() (op fuseops.Op, err error) {
	// Keep going until we find a request we know how to convert.
	for {
		// Read the next message from the kernel.
		var m *buffer.InMessage
		m, err = c.readMessage()
		if err != nil {
			return
		}

		// Choose an ID for this operation for the purposes of logging.
		opID := c.nextOpID
		c.nextOpID++

		// Set up op dependencies.
		opCtx := c.beginOp(m.Header().Opcode, m.Header().Unique)

		var debugLogForOp func(int, string, ...interface{})
		if c.debugLogger != nil {
			debugLogForOp = func(calldepth int, format string, v ...interface{}) {
				c.debugLog(opID, calldepth+1, format, v...)
			}
		}

		sendReply := func(
			op fuseops.Op,
			fuseID uint64,
			replyMsg []byte,
			opErr error) (err error) {
			// Make sure we destroy the message, as required by readMessage.
			defer c.destroyInMessage(m)

			// Clean up state for this op.
			c.finishOp(m.Header().Opcode, m.Header().Unique)

			// Debug logging
			if c.debugLogger != nil {
				if opErr == nil {
					op.Logf("-> OK: %s", op.DebugString())
				} else {
					op.Logf("-> error: %v", opErr)
				}
			}

			// Error logging
			if opErr != nil && c.errorLogger != nil {
				c.errorLogger.Printf("(%s) error: %v", op.ShortDesc(), opErr)
			}

			// Send the reply to the kernel.
			err = c.writeMessage(replyMsg)
			if err != nil {
				err = fmt.Errorf("writeMessage: %v", err)
				return
			}

			return
		}

		// Convert the message to an Op.
		op, err = fuseops.Convert(
			opCtx,
			m,
			c.protocol,
			debugLogForOp,
			c.errorLogger,
			sendReply)

		if err != nil {
			err = fmt.Errorf("fuseops.Convert: %v", err)
			return
		}

		// Log the receipt of the operation.
		c.debugLog(opID, 1, "<- %v", op.ShortDesc())

		// Special case: responding to statfs is required to make mounting work on
		// OS X. We don't currently expose the capability for the file system to
		// intercept this.
		if _, ok := op.(*fuseops.InternalStatFSOp); ok {
			op.Respond(nil)
			continue
		}

		// Special case: handle interrupt requests.
		if interruptOp, ok := op.(*fuseops.InternalInterruptOp); ok {
			c.handleInterrupt(interruptOp.FuseID)
			continue
		}

		return
	}
}

// Close the connection. Must not be called until operations that were read
// from the connection have been responded to.
func (c *Connection) close() (err error) {
	// Posix doesn't say that close can be called concurrently with read or
	// write, but luckily we exclude the possibility of a race by requiring the
	// user to respond to all ops first.
	err = c.dev.Close()
	return
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Arista Networks, Inc.
// Use of this source code is governed by the Apache License 2.0
// that can be found in the COPYING file.

// Package path contains methods for dealing with elemental paths.
package path

import (
	"bytes"
	"strings"

	"github.com/aristanetworks/goarista/key"
)

// Path represents a path decomposed into elements where each
// element is a key.Key. A Path can be interpreted as either
// absolute or relative depending on how it is used.
type Path []key.Key

// New constructs a Path from a variable number of elements.
// Each element may either be a key.Key or a value that can
// be wrapped by a key.Key.
func New(elements ...interface{}) Path {
	path := make(Path, len(elements))
	copyElements(path, elements...)
	return path
}

// Append appends a variable number of elements to a Path.
// Each element may either be a key.Key or a value that can
// be wrapped by a key.Key. Note that calling Append on a
// single Path returns that same Path, whereas in all other
// cases a new Path is returned.
func Append(path Path, elements ...interface{}) Path {
	if len(elements) == 0 {
		return path
	}
	n := len(path)
	p := make(Path, n+len(elements))
	copy(p, path)
	copyElements(p[n:], elements...)
	return p
}

// Join joins a variable number of Paths together. Each path
// in the joining is treated as a subpath of its predecessor.
// Calling Join with no arguments returns nil.
func Join(paths ...Path) Path {
	if len(paths) == 0 {
		return nil
	}
	n := 0
	for _, path := range paths {
		n += len(path)
	}
	joined := make(Path, 0, n)
	for _, path := range paths {
		if len(path) != 0 {
			joined = append(joined, path...)
		}
	}
	return joined
}

// Base returns the last element of the Path. If the Path is
// empty, Base returns nil.
func Base(path Path) key.Key {
	if len(path) > 0 {
		return path[len(path)-1]
	}
	return nil
}

// Clone returns a new Path with the same elements as in the
// provided Path.
func Clone(path Path) Path {
	p := make(Path, len(path))
	copy(p, path)
	return p
}

// Equal returns whether Path a and Path b are the same
// length and whether each element in b corresponds to the
// same element in a.
func Equal(a, b Path) bool {
	return len(a) == len(b) && hasPrefix(a, b)
}

// HasPrefix returns whether Path b is at most the length
// of Path a and whether each element in b corresponds to
// the same element in a from the first element.
func HasPrefix(a, b Path) bool {
	return len(a) >= len(b) && hasPrefix(a, b)
}

// Match returns whether Path a and Path b are the same
// length and whether each element in b corresponds to the
// same element or a wildcard in a.
func Match(a, b Path) bool {
	return len(a) == len(b) && matchPrefix(a, b)
}

// MatchPrefix returns whether Path b is at most the length
// of Path a and whether each element in b corresponds to
// the same element or a wildcard in a from the first
// element.
func MatchPrefix(a, b Path) bool {
	return len(a) >= len(b) && matchPrefix(a, b)
}

// FromString constructs a Path from the elements resulting
// from a split of the input string by "/". Strings that do
// not lead with a '/' are accepted but not reconstructable
// with Path.String.
func FromString(str string) Path {
	if str == "" {
		return Path{}
	} else if str[0] == '/' {
		str = str[1:]
	}
	elements := strings.Split(str, "/")
	path := make(Path, len(elements))
	for i, element := range elements {
		path[i] = key.New(element)
	}
	return path
}

// String returns the Path as an absolute path string.
func (p Path) String() string {
	if len(p) == 0 {
		return "/"
	}
	var buf bytes.Buffer
	for _, element := range p {
		buf.WriteByte('/')
		buf.WriteString(element.String())
	}
	return buf.String()
}

func copyElements(path Path, elements ...interface{}) {
	for i, element := range elements {
		switch val := element.(type) {
		case key.Key:
			path[i] = val
		default:
			path[i] = key.New(val)
		}
	}
}

func hasPrefix(a, b Path) bool {
	for i := range b {
		if !b[i].Equal(a[i]) {
			return false
		}
	}
	return true
}

func matchPrefix(a, b Path) bool {
	for i := range b {
		if !a[i].Equal(Wildcard) && !b[i].Equal(a[i]) {
			return false
		}
	}
	return true
}
<commit_msg>path: Update Join (minor optimization)<commit_after>// Copyright (c) 2017 Arista Networks, Inc.
// Use of this source code is governed by the Apache License 2.0
// that can be found in the COPYING file.

// Package path contains methods for dealing with elemental paths.
package path

import (
	"bytes"
	"strings"

	"github.com/aristanetworks/goarista/key"
)

// Path represents a path decomposed into elements where each
// element is a key.Key. A Path can be interpreted as either
// absolute or relative depending on how it is used.
type Path []key.Key

// New constructs a Path from a variable number of elements.
// Each element may either be a key.Key or a value that can
// be wrapped by a key.Key.
func New(elements ...interface{}) Path {
	path := make(Path, len(elements))
	copyElements(path, elements...)
	return path
}

// Append appends a variable number of elements to a Path.
// Each element may either be a key.Key or a value that can
// be wrapped by a key.Key. Note that calling Append on a
// single Path returns that same Path, whereas in all other
// cases a new Path is returned.
func Append(path Path, elements ...interface{}) Path {
	if len(elements) == 0 {
		return path
	}
	n := len(path)
	p := make(Path, n+len(elements))
	copy(p, path)
	copyElements(p[n:], elements...)
	return p
}

// Join joins a variable number of Paths together. Each path
// in the joining is treated as a subpath of its predecessor.
// Calling Join with no arguments returns nil.
func Join(paths ...Path) Path {
	if len(paths) == 0 {
		return nil
	}
	n := 0
	for _, path := range paths {
		n += len(path)
	}
	joined, i := make(Path, n), 0
	for _, path := range paths {
		if n = len(path); n > 0 {
			copy(joined[i:i+n], path)
			i += n
		}
	}
	return joined
}

// Base returns the last element of the Path. If the Path is
// empty, Base returns nil.
func Base(path Path) key.Key {
	if len(path) > 0 {
		return path[len(path)-1]
	}
	return nil
}

// Clone returns a new Path with the same elements as in the
// provided Path.
func Clone(path Path) Path {
	p := make(Path, len(path))
	copy(p, path)
	return p
}

// Equal returns whether Path a and Path b are the same
// length and whether each element in b corresponds to the
// same element in a.
func Equal(a, b Path) bool {
	return len(a) == len(b) && hasPrefix(a, b)
}

// HasPrefix returns whether Path b is at most the length
// of Path a and whether each element in b corresponds to
// the same element in a from the first element.
func HasPrefix(a, b Path) bool {
	return len(a) >= len(b) && hasPrefix(a, b)
}

// Match returns whether Path a and Path b are the same
// length and whether each element in b corresponds to the
// same element or a wildcard in a.
func Match(a, b Path) bool {
	return len(a) == len(b) && matchPrefix(a, b)
}

// MatchPrefix returns whether Path b is at most the length
// of Path a and whether each element in b corresponds to
// the same element or a wildcard in a from the first
// element.
func MatchPrefix(a, b Path) bool {
	return len(a) >= len(b) && matchPrefix(a, b)
}

// FromString constructs a Path from the elements resulting
// from a split of the input string by "/". Strings that do
// not lead with a '/' are accepted but not reconstructable
// with Path.String.
func FromString(str string) Path {
	if str == "" {
		return Path{}
	} else if str[0] == '/' {
		str = str[1:]
	}
	elements := strings.Split(str, "/")
	path := make(Path, len(elements))
	for i, element := range elements {
		path[i] = key.New(element)
	}
	return path
}

// String returns the Path as an absolute path string.
func (p Path) String() string {
	if len(p) == 0 {
		return "/"
	}
	var buf bytes.Buffer
	for _, element := range p {
		buf.WriteByte('/')
		buf.WriteString(element.String())
	}
	return buf.String()
}

func copyElements(path Path, elements ...interface{}) {
	for i, element := range elements {
		switch val := element.(type) {
		case key.Key:
			path[i] = val
		default:
			path[i] = key.New(val)
		}
	}
}

func hasPrefix(a, b Path) bool {
	for i := range b {
		if !b[i].Equal(a[i]) {
			return false
		}
	}
	return true
}

func matchPrefix(a, b Path) bool {
	for i := range b {
		if !a[i].Equal(Wildcard) && !b[i].Equal(a[i]) {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"os"
	"runtime"
	"strings"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	_ "net/http/pprof"

	"github.com/hyperledger/fabric/common/flogging"
	"github.com/hyperledger/fabric/core/config"
	"github.com/hyperledger/fabric/peer/chaincode"
	"github.com/hyperledger/fabric/peer/channel"
	"github.com/hyperledger/fabric/peer/clilogging"
	"github.com/hyperledger/fabric/peer/common"
	"github.com/hyperledger/fabric/peer/node"
	"github.com/hyperledger/fabric/peer/version"
)

var logger = flogging.MustGetLogger("main")
var logOutput = os.Stderr

// Constants go here.
const cmdRoot = "core"

// The main command describes the service and
// defaults to printing the help message.
var mainCmd = &cobra.Command{
	Use: "peer",
	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
		// check for CORE_LOGGING_LEVEL environment variable, which should override
		// all other log settings. otherwise, this will use the value for from
		// core.yaml
		loggingSpec := viper.GetString("logging.level")
		flogging.InitFromSpec(loggingSpec)

		return nil
	},
	Run: func(cmd *cobra.Command, args []string) {
		if versionFlag {
			fmt.Print(version.GetInfo())
		} else {
			cmd.HelpFunc()(cmd, args)
		}
	},
}

// Peer command version flag
var versionFlag bool

func main() {
	// For environment variables.
	viper.SetEnvPrefix(cmdRoot)
	viper.AutomaticEnv()
	replacer := strings.NewReplacer(".", "_")
	viper.SetEnvKeyReplacer(replacer)

	// Define command-line flags that are valid for all peer commands and
	// subcommands.
	mainFlags := mainCmd.PersistentFlags()
	mainFlags.BoolVarP(&versionFlag, "version", "v", false, "Display current version of fabric peer server")

	mainFlags.String("logging-level", "", "Default logging level and overrides, see core.yaml for full syntax")
	viper.BindPFlag("logging_level", mainFlags.Lookup("logging-level"))
	testCoverProfile := ""
	mainFlags.StringVarP(&testCoverProfile, "test.coverprofile", "", "coverage.cov", "Done")

	err := common.InitConfig(cmdRoot)
	if err != nil { // Handle errors reading the config file
		panic(fmt.Errorf("Fatal error when initializing %s config : %s\n", cmdRoot, err))
	}

	mainCmd.AddCommand(version.Cmd())
	mainCmd.AddCommand(node.Cmd())
	mainCmd.AddCommand(chaincode.Cmd(nil))
	mainCmd.AddCommand(clilogging.Cmd(nil))
	mainCmd.AddCommand(channel.Cmd(nil))

	runtime.GOMAXPROCS(viper.GetInt("peer.gomaxprocs"))

	// setup system-wide logging backend based on settings from core.yaml
	flogging.InitBackend(flogging.SetFormat(viper.GetString("logging.format")), logOutput)

	// Init the MSP
	var mspMgrConfigDir = config.GetPath("peer.mspConfigPath")
	var mspID = viper.GetString("peer.localMspId")
	err = common.InitCrypto(mspMgrConfigDir, mspID)
	if err != nil { // Handle errors reading the config file
		logger.Errorf("Cannot run peer because %s", err.Error())
		os.Exit(1)
	}
	// On failure Cobra prints the usage message and error string, so we only
	// need to exit with a non-0 status
	if mainCmd.Execute() != nil {
		os.Exit(1)
	}
	logger.Info("Exiting.....")
}
<commit_msg>[FAB-7220] Remove test.coverprofile flag<commit_after>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"os"
	"runtime"
	"strings"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	_ "net/http/pprof"

	"github.com/hyperledger/fabric/common/flogging"
	"github.com/hyperledger/fabric/core/config"
	"github.com/hyperledger/fabric/peer/chaincode"
	"github.com/hyperledger/fabric/peer/channel"
	"github.com/hyperledger/fabric/peer/clilogging"
	"github.com/hyperledger/fabric/peer/common"
	"github.com/hyperledger/fabric/peer/node"
	"github.com/hyperledger/fabric/peer/version"
)

var logger = flogging.MustGetLogger("main")
var logOutput = os.Stderr

// Constants go here.
const cmdRoot = "core"

// The main command describes the service and
// defaults to printing the help message.
var mainCmd = &cobra.Command{
	Use: "peer",
	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
		// check for CORE_LOGGING_LEVEL environment variable, which should override
		// all other log settings. otherwise, this will use the value for from
		// core.yaml
		loggingSpec := viper.GetString("logging.level")
		flogging.InitFromSpec(loggingSpec)

		return nil
	},
	Run: func(cmd *cobra.Command, args []string) {
		if versionFlag {
			fmt.Print(version.GetInfo())
		} else {
			cmd.HelpFunc()(cmd, args)
		}
	},
}

// Peer command version flag
var versionFlag bool

func main() {
	// For environment variables.
	viper.SetEnvPrefix(cmdRoot)
	viper.AutomaticEnv()
	replacer := strings.NewReplacer(".", "_")
	viper.SetEnvKeyReplacer(replacer)

	// Define command-line flags that are valid for all peer commands and
	// subcommands.
	mainFlags := mainCmd.PersistentFlags()
	mainFlags.BoolVarP(&versionFlag, "version", "v", false, "Display current version of fabric peer server")

	mainFlags.String("logging-level", "", "Default logging level and overrides, see core.yaml for full syntax")
	viper.BindPFlag("logging_level", mainFlags.Lookup("logging-level"))

	err := common.InitConfig(cmdRoot)
	if err != nil { // Handle errors reading the config file
		panic(fmt.Errorf("Fatal error when initializing %s config : %s\n", cmdRoot, err))
	}

	mainCmd.AddCommand(version.Cmd())
	mainCmd.AddCommand(node.Cmd())
	mainCmd.AddCommand(chaincode.Cmd(nil))
	mainCmd.AddCommand(clilogging.Cmd(nil))
	mainCmd.AddCommand(channel.Cmd(nil))

	runtime.GOMAXPROCS(viper.GetInt("peer.gomaxprocs"))

	// setup system-wide logging backend based on settings from core.yaml
	flogging.InitBackend(flogging.SetFormat(viper.GetString("logging.format")), logOutput)

	// Init the MSP
	var mspMgrConfigDir = config.GetPath("peer.mspConfigPath")
	var mspID = viper.GetString("peer.localMspId")
	err = common.InitCrypto(mspMgrConfigDir, mspID)
	if err != nil { // Handle errors reading the config file
		logger.Errorf("Cannot run peer because %s", err.Error())
		os.Exit(1)
	}
	// On failure Cobra prints the usage message and error string, so we only
	// need to exit with a non-0 status
	if mainCmd.Execute() != nil {
		os.Exit(1)
	}
	logger.Info("Exiting.....")
}
<|endoftext|>


<commit_before>package compresstest

import (
	"io"
	"math/rand"
	"os"
	"testing"

	"time"

	"compress/flate"

	"github.com/xiaonanln/goworld/engine/compress/gwsnappy"
)

type randomFailWriter struct {
	io.Writer
}

func (rfw randomFailWriter) Write(b []byte) (int, error) {
	fv := rand.Float32()
	if fv < 0.1 {
		println("ErrShortWrite", fv)
		return 0, io.ErrShortWrite
	}
	return rfw.Writer.Write(b)
}

type Flusher interface {
	Flush() error
}

func TestFlateWriterSize(t *testing.T) {
	var err error
	f, err := os.Open("input.txt")
	if err != nil {
		panic(err)
	}

	all := []interface{}{}
	for i := 0; i < 5000; i++ {
		all = append(all, gwsnappy.NewReader(f))
		all = append(all, gwsnappy.NewBufferedWriter(f))
		all = append(all, flate.NewReader(f))
		w, _ := flate.NewWriter(f, flate.BestSpeed)
		all = append(all, w)
	}
	println("created")
	time.Sleep(time.Second * 10)
}

//func TestFlateCompressor(t *testing.T) {
//	var err error
//	f, err := os.Open("input.txt")
//	if err != nil {
//		panic(err)
//	}
//	of, err := os.OpenFile("output-flate.txt", os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
//	if err != nil {
//		panic(err)
//	}
//
//	cw, err := flate.NewWriter(of, 1)
//	if err != nil {
//		panic(err)
//	}
//	testCompressor(t, f, cw)
//}
//
//func TestSnappyCompressor(t *testing.T) {
//	var err error
//	f, err := os.Open("input.txt")
//	if err != nil {
//		panic(err)
//	}
//	of, err := os.OpenFile("output-snappy.txt", os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
//	if err != nil {
//		panic(err)
//	}
//
//	testCompressor(t, f, snappy.NewWriter(randomFailWriter{of}))
//}

//func TestLz4Compressor(t *testing.T) {
//	var err error
//	f, err := os.Open("input.txt")
//	if err != nil {
//		panic(err)
//	}
//	of, err := os.OpenFile("output-lz4.txt", os.O_CREATE|os.O_WRONLY, 0644)
//	if err != nil {
//		panic(err)
//	}
//
//	testCompressor(t, f, lz4.NewWriter(randomFailWriter{of}))
//}

func testCompressor(t *testing.T, r io.Reader, w io.Writer) {
	buf := make([]byte, 1024)
	for {
		n, err := r.Read(buf)
		if err != nil {
			if err == io.EOF {
				break
			}
			panic(err)
		}

		if n > 0 {
			wbuf := buf[:n]
			for len(wbuf) > 0 {
				n, err := w.Write(wbuf)
				wbuf = wbuf[n:]
				if err != nil {
					println(n, err)
					w.(Flusher).Flush()
				}
			}
		}

	}
	w.(Flusher).Flush()
}
<commit_msg>remove tests<commit_after><|endoftext|>