<commit_before>package ov

import (
	"encoding/json"
	"fmt"
	"github.com/HewlettPackard/oneview-golang/rest"
	"github.com/HewlettPackard/oneview-golang/utils"
	"github.com/docker/machine/libmachine/log"
)

// The Marshal() will omit the bool attibutes below if they are false.
// Please remove the omitempty option and use it as and when required.

type StorageVolume struct {
	Category                  string                    `json:"category,omitempty"`
	Created                   string                    `json:"created,omitempty"`
	Description               utils.Nstring             `json:"description,omitempty"`
	ETAG                      string                    `json:"eTag,omitempty"`
	Name                      string                    `json:"name,omitempty"`
	State                     string                    `json:"state,omitempty"`
	Status                    string                    `json:"status,omitempty"`
	Type                      string                    `json:"type,omitempty"`
	URI                       utils.Nstring             `json:"uri,omitempty"`
	DeviceVolumeName          string                    `json:"deviceVolumeName,omitempty"`
	RequestingRefresh         bool                      `json:"requestingRefresh,omitempty"`
	AllocatedCapacity         string                    `json:"allocatedCapacity,omitempty"`
	InitialScopeUris          []utils.Nstring           `json:"initialScopeUris,omitempty"`
	DeviceSpecificAttributes  *DeviceSpecificAttributes `json:"deviceSpecificAttributes,omitempty"`
	VolumeTemplateUri         utils.Nstring             `json:"volumeTemplateUri,omitempty"`
	IsShareable               *bool                     `json:"isShareable,omitempty"`
	StoragePoolUri            utils.Nstring             `json:"storagePoolUri,omitempty"`
	StorageSystemUri          utils.Nstring             `json:"storageSystemUri,omitempty"`
	ProvisionedCapacity       string                    `json:"provisionedCapacity,omitempty"`
	Properties                *Properties               `json:"properties,omitempty"`
	TemplateURI               utils.Nstring             `json:"templateURI,omitempty"`
	IsPermanent               *bool                     `json:"isPermanent,omitempty"`
	ProvisioningTypeForUpdate string                    `json:"provisioningType,omitempty"`
	TemplateVersion           string                    `json:"templateVersion,omitempty"`
	//	Wwn										string				`json:""`

	/*
	   Category              string        `json:"category,omitempty"`              //  "type": "StorageVolume",
	   Created               string        `json:"created,omitempty"`               //  "created": "2016-06-20T22:48:14.422Z"
	   Description           string        `json:"description,omitempty"`           //  "description": "Integration test volume 2",
	   ETAG                  string        `json:"eTag,omitempty"`                  //  "eTag": "2016-06-20T22:48:17.704Z",            string        `json:"modified,omitempty"`              // "modified": "20150831T154835.250Z",
	   Name                  string        `json:"name,omitempty"`                  //  "name": "Volume_2",
	   State                 string        `json:"state,omitempty"`                 //  "state": "Configured",
	   Status                string        `json:"status,omitempty"`                //  "status": "OK",
	   Type                  string        `json:"type,omitempty"`                  //  "type": "StorageVolume",
	   URI                   utils.Nstring `json:"uri,omitempty"`                   //  "uri": "/rest/storage-volumes/527801AC-B6B6-4A63-8510-D32906C9C57B"
	   Shareable							string				`json:"shareable,omitempty"`                                //  "shareable": false,
	   AllocatedCapacity			string				`json:""`																 //  "allocatedCapacity": "1073741824",
	   DeviceType						string				`json:""`																 //  "deviceType": "SSD",
	   DeviceVolumeName			string				`json:""`                                //  "deviceVolumeName": "Volume_2",
	   IsPermanent						string 				`json:""`                                //  "isPermanent": true,
	   RaidLevel							string				`json:""`                								 //  "raidLevel": "RAID5",
	   RefreshState					string				`json:""`                                //  "refreshState": "NotRefreshing",
	   RevertToSnapshotUri		string				`json:""`                                //  "revertToSnapshotUri": null,
	   SnapshotPoolUri				string				`json:""`                                //  "snapshotPoolUri": "/rest/storage-pools/000D692C-74B3-44AC-B297-3741AABE29F8",
	   Snapshots							string				`json:""`                                //  "snapshots": null,
	   StateReason						string				`json:""`                                //  "stateReason": "None",
	   StoragePoolUri				string				`json:"storagePoolUri,omitempty"`        //  "storagePoolUri": "/rest/storage-pools/000D692C-74B3-44AC-B297-3741AABE29F8",
	   StorageSystemUri			string				`json:"storageSystemUri,omitempty"`      //  "storageSystemUri": "/rest/storage-systems/TXQ1000307",
	   ProvisionType					string 				`json:"provisionType,omitempty"`         //  "provisionType": "Full",
	   ProvisionedCapacity		string				`json:"provisionedCapacity,omitempty"`   //  "provisionedCapacity": "1073741824",
	   ProvisioningParameters []ProvisioningParameters		`json:"provisioningParameters,omitempty"`
	   //	Wwn										string				`json:""`                                //  "wwn": "DC:57:93:56:32:00:10:00:30:71:46:64:62:89:52:11",

	*/
}

type ProvisioningParameters struct {
	StoragePoolUri    utils.Nstring `json:"storagePoolUri,omitempty"`
	ProvisionType     string        `json:"provisionType,omitempty"`
	RequestedCapacity string        `json:"requestedCapacity,omitempty"`
	Shareable         bool          `json:"shareable,omitempty"`
}

type Properties struct {
	Name                string        `json:"name,omitempty"`
	Description         utils.Nstring `json:"description,omitempty"`
	Storagepool         utils.Nstring `json:"storagePool,omitempty"`
	Size                int           `json:"size,omitempty"`
	ProvisioningType    string        `json:"provisioningType,omitempty"`
	DataTransferLimit   int           `json:"dataTransferLimit,omitempty"`
	IsDeduplicated      bool          `json:"isDeduplicated,omitempty"`
	IsEncrypted         bool          `json:"isEncrypted,omitempty"`
	IsPinned            bool          `json:"isPinned,omitempty"`
	IsCompressed        bool          `json:"isCompressed,omitempty"`
	IsShareable         bool          `json:"isShareable,omitempty"`
	DataProtectionLevel string        `json:"dataProtectionLevel,omitempty"`
}

type DeviceSpecificAttributes struct {
	ApplicationCategory           string        `json:"applicationCategory,omitempty"`
	BlockSize                     int           `json:"blockSize,omitempty"`
	DataTransferLimit             int           `json:"dataTransferLimit,omitempty"`
	FolderId                      string        `json:"folderId,omitempty"`
	FolderName                    string        `json:"folderName,omitempty"`
	IopsLimit                     int           `json:"iopsLimit,omitempty"`
	IsDeduplicated                bool          `json:"isDeduplicated,omitempty"`
	IsEncrypted                   bool          `json:"isEncrypted,omitempty"`
	IsPinned                      bool          `json:"isPinned,omitempty"`
	PreviouslyDeduplicated        bool          `json:"previouslyDeduplicated,omitempty"`
	IsCompressed                  bool          `json:"isCompressed,omitempty"`
	Transport                     string        `json:"transport,omitempty"`
	Iqn                           string        `json:"iqn,omitempty"`
	NumberOfReplicas              int           `json:"numberOfReplicas,omitempty"`
	DataProtectionLevel           string        `json:"dataProtectionLevel,omitempty"`
	Id                            int           `json:"id,omitempty"`
	Uri                           utils.Nstring `json:"uri,omitempty"`
	CopyState                     string        `json:"copyState,omitempty"`
	SnapshotPoolUri               utils.Nstring `json:"snapshotPoolUri,omitempty"`
	IsAdaptiveOptimizationEnabled bool          `json:"isAdaptiveOptimizationEnabled,omitempty"`
}

type StorageVolumesList struct {
	Total       int             `json:"total,omitempty"`       // "total": 1,
	Count       int             `json:"count,omitempty"`       // "count": 1,
	Start       int             `json:"start,omitempty"`       // "start": 0,
	PrevPageURI utils.Nstring   `json:"prevPageUri,omitempty"` // "prevPageUri": null,
	NextPageURI utils.Nstring   `json:"nextPageUri,omitempty"` // "nextPageUri": null,
	URI         utils.Nstring   `json:"uri,omitempty"`         // "uri": "/rest/server-profiles?filter=connectionTemplateUri%20matches%7769cae0-b680-435b-9b87-9b864c81657fsort=name:asc"
	Members     []StorageVolume `json:"members,omitempty"`     // "members":[]
}

func (c *OVClient) GetStorageVolumeByName(name string) (StorageVolume, error) {
	var (
		sVol StorageVolume
	)
	sVols, err := c.GetStorageVolumes(fmt.Sprintf("name matches '%s'", name), "name:asc")
	if sVols.Total > 0 {
		return sVols.Members[0], err
	} else {
		return sVol, err
	}
}

func (c *OVClient) GetStorageVolumes(filter string, sort string) (StorageVolumesList, error) {
	var (
		uri   = "/rest/storage-volumes"
		q     map[string]interface{}
		sVols StorageVolumesList
	)
	q = make(map[string]interface{})
	if len(filter) > 0 {
		q["filter"] = filter
	}

	if sort != "" {
		q["sort"] = sort
	}

	// refresh login
	c.RefreshLogin()
	c.SetAuthHeaderOptions(c.GetAuthHeaderMap())
	// Setup query
	if len(q) > 0 {
		c.SetQueryString(q)
	}

	data, err := c.RestAPICall(rest.GET, uri, nil)
	if err != nil {
		return sVols, err
	}

	log.Debugf("GetStorageVolumes %s", data)
	if err := json.Unmarshal([]byte(data), &sVols); err != nil {
		return sVols, err
	}
	return sVols, nil
}

func (c *OVClient) CreateStorageVolume(sVol StorageVolume) error {
	log.Infof("Initializing creation of storage volume for %s.", sVol.Name)
	var (
		uri = "/rest/storage-volumes"
		t   *Task
	)
	// refresh login
	c.RefreshLogin()
	c.SetAuthHeaderOptions(c.GetAuthHeaderMap())

	t = t.NewProfileTask(c)
	t.ResetTask()
	log.Debugf("REST : %s \n %+v\n", uri, sVol)
	log.Debugf("task -> %+v", t)

	data, err := c.RestAPICall(rest.POST, uri, sVol)
	if err != nil {
		t.TaskIsDone = true
		log.Errorf("Error submitting new storage volume request: %s", err)
		return err
	}

	log.Debugf("Response New StorageVolume %s", data)
	if err := json.Unmarshal([]byte(data), &t); err != nil {
		t.TaskIsDone = true
		log.Errorf("Error with task un-marshal: %s", err)
		return err
	}

	err = t.Wait()
	if err != nil {
		return err
	}

	return nil
}

func (c *OVClient) DeleteStorageVolume(name string) error {
	var (
		sVol StorageVolume
		err  error
		t    *Task
		uri  string
	)

	sVol, err = c.GetStorageVolumeByName(name)
	if err != nil {
		return err
	}
	if sVol.Name != "" {
		t = t.NewProfileTask(c)
		t.ResetTask()
		log.Debugf("REST : %s \n %+v\n", sVol.URI, sVol)
		log.Debugf("task -> %+v", t)
		uri = sVol.URI.String()
		if uri == "" {
			log.Warn("Unable to post delete, no uri found.")
			t.TaskIsDone = true
			return err
		}
		data, err := c.RestAPICall(rest.DELETE, uri, nil)
		if err != nil {
			log.Errorf("Error submitting delete storage volume request: %s", err)
			t.TaskIsDone = true
			return err
		}

		log.Debugf("Response delete storage volume %s", data)
		if err := json.Unmarshal([]byte(data), &t); err != nil {
			t.TaskIsDone = true
			log.Errorf("Error with task un-marshal: %s", err)
			return err
		}
		err = t.Wait()
		if err != nil {
			return err
		}
		return nil
	} else {
		log.Infof("StorageVolume could not be found to delete, %s, skipping delete ...", name)
	}
	return nil
}

func (c *OVClient) UpdateStorageVolume(sVol StorageVolume) error {
	log.Infof("Initializing update of storage volume for %s.", sVol.Name)
	var (
		uri = sVol.URI.String()
		t   *Task
	)
	// refresh login
	c.RefreshLogin()
	c.SetAuthHeaderOptions(c.GetAuthHeaderMap())

	t = t.NewProfileTask(c)
	t.ResetTask()
	log.Debugf("REST : %s \n %+v\n", uri, sVol)
	log.Debugf("task -> %+v", t)
	data, err := c.RestAPICall(rest.PUT, uri, sVol)
	if err != nil {
		t.TaskIsDone = true
		log.Errorf("Error submitting update StorageVolume request: %s", err)
		return err
	}
	log.Debugf("Response update StorageVolume %s", data)
	if err := json.Unmarshal([]byte(data), &t); err != nil {
		t.TaskIsDone = true
		log.Errorf("Error with task un-marshal: %s", err)
		return err
	}

	err = t.Wait()
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Datatype change for Id in DeviceSpecificattributes<commit_after>package ov

import (
	"encoding/json"
	"fmt"

	"github.com/HewlettPackard/oneview-golang/rest"
	"github.com/HewlettPackard/oneview-golang/utils"
	"github.com/docker/machine/libmachine/log"
)

// The Marshal() will omit the bool attibutes below if they are false.
// Please remove the omitempty option and use it as and when required.

type StorageVolume struct {
	Category                  string                    `json:"category,omitempty"`
	Created                   string                    `json:"created,omitempty"`
	Description               utils.Nstring             `json:"description,omitempty"`
	ETAG                      string                    `json:"eTag,omitempty"`
	Name                      string                    `json:"name,omitempty"`
	State                     string                    `json:"state,omitempty"`
	Status                    string                    `json:"status,omitempty"`
	Type                      string                    `json:"type,omitempty"`
	URI                       utils.Nstring             `json:"uri,omitempty"`
	DeviceVolumeName          string                    `json:"deviceVolumeName,omitempty"`
	RequestingRefresh         bool                      `json:"requestingRefresh,omitempty"`
	AllocatedCapacity         string                    `json:"allocatedCapacity,omitempty"`
	InitialScopeUris          []utils.Nstring           `json:"initialScopeUris,omitempty"`
	DeviceSpecificAttributes  *DeviceSpecificAttributes `json:"deviceSpecificAttributes,omitempty"`
	VolumeTemplateUri         utils.Nstring             `json:"volumeTemplateUri,omitempty"`
	IsShareable               *bool                     `json:"isShareable,omitempty"`
	StoragePoolUri            utils.Nstring             `json:"storagePoolUri,omitempty"`
	StorageSystemUri          utils.Nstring             `json:"storageSystemUri,omitempty"`
	ProvisionedCapacity       string                    `json:"provisionedCapacity,omitempty"`
	Properties                *Properties               `json:"properties,omitempty"`
	TemplateURI               utils.Nstring             `json:"templateURI,omitempty"`
	IsPermanent               *bool                     `json:"isPermanent,omitempty"`
	ProvisioningTypeForUpdate string                    `json:"provisioningType,omitempty"`
	TemplateVersion           string                    `json:"templateVersion,omitempty"`
	//	Wwn										string				`json:""`

	/*
	   Category              string        `json:"category,omitempty"`              //  "type": "StorageVolume",
	   Created               string        `json:"created,omitempty"`               //  "created": "2016-06-20T22:48:14.422Z"
	   Description           string        `json:"description,omitempty"`           //  "description": "Integration test volume 2",
	   ETAG                  string        `json:"eTag,omitempty"`                  //  "eTag": "2016-06-20T22:48:17.704Z",            string        `json:"modified,omitempty"`              // "modified": "20150831T154835.250Z",
	   Name                  string        `json:"name,omitempty"`                  //  "name": "Volume_2",
	   State                 string        `json:"state,omitempty"`                 //  "state": "Configured",
	   Status                string        `json:"status,omitempty"`                //  "status": "OK",
	   Type                  string        `json:"type,omitempty"`                  //  "type": "StorageVolume",
	   URI                   utils.Nstring `json:"uri,omitempty"`                   //  "uri": "/rest/storage-volumes/527801AC-B6B6-4A63-8510-D32906C9C57B"
	   Shareable							string				`json:"shareable,omitempty"`                                //  "shareable": false,
	   AllocatedCapacity			string				`json:""`																 //  "allocatedCapacity": "1073741824",
	   DeviceType						string				`json:""`																 //  "deviceType": "SSD",
	   DeviceVolumeName			string				`json:""`                                //  "deviceVolumeName": "Volume_2",
	   IsPermanent						string 				`json:""`                                //  "isPermanent": true,
	   RaidLevel							string				`json:""`                								 //  "raidLevel": "RAID5",
	   RefreshState					string				`json:""`                                //  "refreshState": "NotRefreshing",
	   RevertToSnapshotUri		string				`json:""`                                //  "revertToSnapshotUri": null,
	   SnapshotPoolUri				string				`json:""`                                //  "snapshotPoolUri": "/rest/storage-pools/000D692C-74B3-44AC-B297-3741AABE29F8",
	   Snapshots							string				`json:""`                                //  "snapshots": null,
	   StateReason						string				`json:""`                                //  "stateReason": "None",
	   StoragePoolUri				string				`json:"storagePoolUri,omitempty"`        //  "storagePoolUri": "/rest/storage-pools/000D692C-74B3-44AC-B297-3741AABE29F8",
	   StorageSystemUri			string				`json:"storageSystemUri,omitempty"`      //  "storageSystemUri": "/rest/storage-systems/TXQ1000307",
	   ProvisionType					string 				`json:"provisionType,omitempty"`         //  "provisionType": "Full",
	   ProvisionedCapacity		string				`json:"provisionedCapacity,omitempty"`   //  "provisionedCapacity": "1073741824",
	   ProvisioningParameters []ProvisioningParameters		`json:"provisioningParameters,omitempty"`
	   //	Wwn										string				`json:""`                                //  "wwn": "DC:57:93:56:32:00:10:00:30:71:46:64:62:89:52:11",

	*/
}

type ProvisioningParameters struct {
	StoragePoolUri    utils.Nstring `json:"storagePoolUri,omitempty"`
	ProvisionType     string        `json:"provisionType,omitempty"`
	RequestedCapacity string        `json:"requestedCapacity,omitempty"`
	Shareable         bool          `json:"shareable,omitempty"`
}

type Properties struct {
	Name                string        `json:"name,omitempty"`
	Description         utils.Nstring `json:"description,omitempty"`
	Storagepool         utils.Nstring `json:"storagePool,omitempty"`
	Size                int           `json:"size,omitempty"`
	ProvisioningType    string        `json:"provisioningType,omitempty"`
	DataTransferLimit   int           `json:"dataTransferLimit,omitempty"`
	IsDeduplicated      bool          `json:"isDeduplicated,omitempty"`
	IsEncrypted         bool          `json:"isEncrypted,omitempty"`
	IsPinned            bool          `json:"isPinned,omitempty"`
	IsCompressed        bool          `json:"isCompressed,omitempty"`
	IsShareable         bool          `json:"isShareable,omitempty"`
	DataProtectionLevel string        `json:"dataProtectionLevel,omitempty"`
}

type DeviceSpecificAttributes struct {
	ApplicationCategory           string        `json:"applicationCategory,omitempty"`
	BlockSize                     int           `json:"blockSize,omitempty"`
	DataTransferLimit             int           `json:"dataTransferLimit,omitempty"`
	FolderId                      string        `json:"folderId,omitempty"`
	FolderName                    string        `json:"folderName,omitempty"`
	IopsLimit                     int           `json:"iopsLimit,omitempty"`
	IsDeduplicated                bool          `json:"isDeduplicated,omitempty"`
	IsEncrypted                   bool          `json:"isEncrypted,omitempty"`
	IsPinned                      bool          `json:"isPinned,omitempty"`
	PreviouslyDeduplicated        bool          `json:"previouslyDeduplicated,omitempty"`
	IsCompressed                  bool          `json:"isCompressed,omitempty"`
	Transport                     string        `json:"transport,omitempty"`
	Iqn                           string        `json:"iqn,omitempty"`
	NumberOfReplicas              int           `json:"numberOfReplicas,omitempty"`
	DataProtectionLevel           string        `json:"dataProtectionLevel,omitempty"`
	Id                            string        `json:"id,omitempty"`
	Uri                           utils.Nstring `json:"uri,omitempty"`
	CopyState                     string        `json:"copyState,omitempty"`
	SnapshotPoolUri               utils.Nstring `json:"snapshotPoolUri,omitempty"`
	IsAdaptiveOptimizationEnabled bool          `json:"isAdaptiveOptimizationEnabled,omitempty"`
}

type StorageVolumesList struct {
	Total       int             `json:"total,omitempty"`       // "total": 1,
	Count       int             `json:"count,omitempty"`       // "count": 1,
	Start       int             `json:"start,omitempty"`       // "start": 0,
	PrevPageURI utils.Nstring   `json:"prevPageUri,omitempty"` // "prevPageUri": null,
	NextPageURI utils.Nstring   `json:"nextPageUri,omitempty"` // "nextPageUri": null,
	URI         utils.Nstring   `json:"uri,omitempty"`         // "uri": "/rest/server-profiles?filter=connectionTemplateUri%20matches%7769cae0-b680-435b-9b87-9b864c81657fsort=name:asc"
	Members     []StorageVolume `json:"members,omitempty"`     // "members":[]
}

func (c *OVClient) GetStorageVolumeByName(name string) (StorageVolume, error) {
	var (
		sVol StorageVolume
	)
	sVols, err := c.GetStorageVolumes(fmt.Sprintf("name matches '%s'", name), "name:asc")
	if sVols.Total > 0 {
		return sVols.Members[0], err
	} else {
		return sVol, err
	}
}

func (c *OVClient) GetStorageVolumes(filter string, sort string) (StorageVolumesList, error) {
	var (
		uri   = "/rest/storage-volumes"
		q     map[string]interface{}
		sVols StorageVolumesList
	)
	q = make(map[string]interface{})
	if len(filter) > 0 {
		q["filter"] = filter
	}

	if sort != "" {
		q["sort"] = sort
	}

	// refresh login
	c.RefreshLogin()
	c.SetAuthHeaderOptions(c.GetAuthHeaderMap())
	// Setup query
	if len(q) > 0 {
		c.SetQueryString(q)
	}

	data, err := c.RestAPICall(rest.GET, uri, nil)
	if err != nil {
		return sVols, err
	}

	log.Debugf("GetStorageVolumes %s", data)
	if err := json.Unmarshal([]byte(data), &sVols); err != nil {
		return sVols, err
	}
	return sVols, nil
}

func (c *OVClient) CreateStorageVolume(sVol StorageVolume) error {
	log.Infof("Initializing creation of storage volume for %s.", sVol.Name)
	var (
		uri = "/rest/storage-volumes"
		t   *Task
	)
	// refresh login
	c.RefreshLogin()
	c.SetAuthHeaderOptions(c.GetAuthHeaderMap())

	t = t.NewProfileTask(c)
	t.ResetTask()
	log.Debugf("REST : %s \n %+v\n", uri, sVol)
	log.Debugf("task -> %+v", t)

	data, err := c.RestAPICall(rest.POST, uri, sVol)
	if err != nil {
		t.TaskIsDone = true
		log.Errorf("Error submitting new storage volume request: %s", err)
		return err
	}

	log.Debugf("Response New StorageVolume %s", data)
	if err := json.Unmarshal([]byte(data), &t); err != nil {
		t.TaskIsDone = true
		log.Errorf("Error with task un-marshal: %s", err)
		return err
	}

	err = t.Wait()
	if err != nil {
		return err
	}

	return nil
}

func (c *OVClient) DeleteStorageVolume(name string) error {
	var (
		sVol StorageVolume
		err  error
		t    *Task
		uri  string
	)

	sVol, err = c.GetStorageVolumeByName(name)
	if err != nil {
		return err
	}
	if sVol.Name != "" {
		t = t.NewProfileTask(c)
		t.ResetTask()
		log.Debugf("REST : %s \n %+v\n", sVol.URI, sVol)
		log.Debugf("task -> %+v", t)
		uri = sVol.URI.String()
		if uri == "" {
			log.Warn("Unable to post delete, no uri found.")
			t.TaskIsDone = true
			return err
		}
		data, err := c.RestAPICall(rest.DELETE, uri, nil)
		if err != nil {
			log.Errorf("Error submitting delete storage volume request: %s", err)
			t.TaskIsDone = true
			return err
		}

		log.Debugf("Response delete storage volume %s", data)
		if err := json.Unmarshal([]byte(data), &t); err != nil {
			t.TaskIsDone = true
			log.Errorf("Error with task un-marshal: %s", err)
			return err
		}
		err = t.Wait()
		if err != nil {
			return err
		}
		return nil
	} else {
		log.Infof("StorageVolume could not be found to delete, %s, skipping delete ...", name)
	}
	return nil
}

func (c *OVClient) UpdateStorageVolume(sVol StorageVolume) error {
	log.Infof("Initializing update of storage volume for %s.", sVol.Name)
	var (
		uri = sVol.URI.String()
		t   *Task
	)
	// refresh login
	c.RefreshLogin()
	c.SetAuthHeaderOptions(c.GetAuthHeaderMap())

	t = t.NewProfileTask(c)
	t.ResetTask()
	log.Debugf("REST : %s \n %+v\n", uri, sVol)
	log.Debugf("task -> %+v", t)
	data, err := c.RestAPICall(rest.PUT, uri, sVol)
	if err != nil {
		t.TaskIsDone = true
		log.Errorf("Error submitting update StorageVolume request: %s", err)
		return err
	}
	log.Debugf("Response update StorageVolume %s", data)
	if err := json.Unmarshal([]byte(data), &t); err != nil {
		t.TaskIsDone = true
		log.Errorf("Error with task un-marshal: %s", err)
		return err
	}

	err = t.Wait()
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package util

import (
	"context"
	"fmt"
	"time"

	"k8s.io/kubernetes/test/e2e/framework/volume"

	kapiv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/wait"
	e2e "k8s.io/kubernetes/test/e2e/framework"
	e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
)

// SetupK8SNFSServerAndVolume sets up an nfs server pod with count number of persistent volumes
func SetupK8SNFSServerAndVolume(oc *CLI, count int) (*kapiv1.Pod, []*kapiv1.PersistentVolume, error) {
	e2e.Logf("Adding privileged scc from system:serviceaccount:%s:default", oc.Namespace())
	_, err := oc.AsAdmin().Run("adm").Args("policy", "add-scc-to-user", "privileged", fmt.Sprintf("system:serviceaccount:%s:default", oc.Namespace())).Output()
	if err != nil {
		return nil, nil, err
	}

	e2e.Logf(fmt.Sprintf("Creating NFS server"))
	config := volume.TestConfig{
		Namespace: oc.Namespace(),
		Prefix:    "nfs",
		// this image is an extension of k8s.gcr.io/volume-nfs:0.8 that adds
		// additional nfs mounts to allow for openshift extended tests with
		// replicas and shared state (mongo, postgresql, mysql, etc.); defined
		// in repo https://github.com/gmontero/nfs-server
		ServerImage:   "docker.io/gmontero/nfs-server:latest",
		ServerPorts:   []int{2049},
		ServerVolumes: map[string]string{"": "/exports"},
	}
	pod, ip := volume.CreateStorageServer(oc.AsAdmin().KubeFramework().ClientSet, config)
	e2e.Logf("Waiting for pod running")
	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
		phase, err := oc.AsAdmin().Run("get").Args("pods", pod.Name, "--template", "{{.status.phase}}").Output()
		if err != nil {
			return false, nil
		}
		if phase != "Running" {
			return false, nil
		}
		return true, nil
	})

	pvs := []*kapiv1.PersistentVolume{}
	volLabel := labels.Set{e2epv.VolumeSelectorKey: oc.Namespace()}
	for i := 0; i < count; i++ {
		e2e.Logf(fmt.Sprintf("Creating persistent volume %d", i))
		pvConfig := e2epv.PersistentVolumeConfig{
			NamePrefix: "nfs-",
			Labels:     volLabel,
			PVSource: kapiv1.PersistentVolumeSource{
				NFS: &kapiv1.NFSVolumeSource{
					Server:   ip,
					Path:     fmt.Sprintf("/exports/data-%d", i),
					ReadOnly: false,
				},
			},
		}
		pvTemplate := e2epv.MakePersistentVolume(pvConfig)
		pv, err := oc.AdminKubeClient().CoreV1().PersistentVolumes().Create(context.Background(), pvTemplate, metav1.CreateOptions{})
		if err != nil {
			e2e.Logf("error creating persistent volume %#v", err)
		}
		e2e.Logf("Created persistent volume %#v", pv)
		pvs = append(pvs, pv)
	}
	return pod, pvs, err
}
<commit_msg>move jenkins e2e nfs setup from docker.io to quay.io image<commit_after>package util

import (
	"context"
	"fmt"
	"time"

	"k8s.io/kubernetes/test/e2e/framework/volume"

	kapiv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/wait"
	e2e "k8s.io/kubernetes/test/e2e/framework"
	e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
)

// SetupK8SNFSServerAndVolume sets up an nfs server pod with count number of persistent volumes
func SetupK8SNFSServerAndVolume(oc *CLI, count int) (*kapiv1.Pod, []*kapiv1.PersistentVolume, error) {
	e2e.Logf("Adding privileged scc from system:serviceaccount:%s:default", oc.Namespace())
	_, err := oc.AsAdmin().Run("adm").Args("policy", "add-scc-to-user", "privileged", fmt.Sprintf("system:serviceaccount:%s:default", oc.Namespace())).Output()
	if err != nil {
		return nil, nil, err
	}

	e2e.Logf(fmt.Sprintf("Creating NFS server"))
	config := volume.TestConfig{
		Namespace: oc.Namespace(),
		Prefix:    "nfs",
		// this image is an extension of k8s.gcr.io/volume-nfs:0.8 that adds
		// additional nfs mounts to allow for openshift extended tests with
		// replicas and shared state (formerly mongo, postgresql, mysql, etc., now only jenkins); defined
		// in repo https://github.com/redhat-developer/nfs-server
		ServerImage:   "quay.io/redhat-developer/nfs-server:latest",
		ServerPorts:   []int{2049},
		ServerVolumes: map[string]string{"": "/exports"},
	}
	pod, ip := volume.CreateStorageServer(oc.AsAdmin().KubeFramework().ClientSet, config)
	e2e.Logf("Waiting for pod running")
	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
		phase, err := oc.AsAdmin().Run("get").Args("pods", pod.Name, "--template", "{{.status.phase}}").Output()
		if err != nil {
			return false, nil
		}
		if phase != "Running" {
			return false, nil
		}
		return true, nil
	})

	pvs := []*kapiv1.PersistentVolume{}
	volLabel := labels.Set{e2epv.VolumeSelectorKey: oc.Namespace()}
	for i := 0; i < count; i++ {
		e2e.Logf(fmt.Sprintf("Creating persistent volume %d", i))
		pvConfig := e2epv.PersistentVolumeConfig{
			NamePrefix: "nfs-",
			Labels:     volLabel,
			PVSource: kapiv1.PersistentVolumeSource{
				NFS: &kapiv1.NFSVolumeSource{
					Server:   ip,
					Path:     fmt.Sprintf("/exports/data-%d", i),
					ReadOnly: false,
				},
			},
		}
		pvTemplate := e2epv.MakePersistentVolume(pvConfig)
		pv, err := oc.AdminKubeClient().CoreV1().PersistentVolumes().Create(context.Background(), pvTemplate, metav1.CreateOptions{})
		if err != nil {
			e2e.Logf("error creating persistent volume %#v", err)
		}
		e2e.Logf("Created persistent volume %#v", pv)
		pvs = append(pvs, pv)
	}
	return pod, pvs, err
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"log"
	"os"
	"path/filepath"
	"regexp"
)

var (
	sectionSeparator = regexp.MustCompile(`^=====*$`)
	sectionHeader    = regexp.MustCompile(`^\[[A-Z]+.*\]$`)
	bracketsToIgnore = regexp.MustCompile(` \{[0-3]\}$`)
	manuelFilePath   string
	faxbotFilePath   string
	outputFilePath   string
)

func init() {
	const (
		manuelUsage = "path to the file of output from missingManuel.ash"
		faxbotUsage = "path to the file of monsters available in the faxbot"
		outputUsage = "path to the missing manuel output file"
	)
	var (
		defaultDir    = filepath.Clean(`C:\Users\admin\Desktop\kolmafia\samples`)
		manuelDefault = filepath.Join(defaultDir, "monster manuel.txt")
		faxbotDefault = filepath.Join(defaultDir, "faxbot.txt")
		outputDefault = filepath.Join(defaultDir, "filtered_faxbot.txt")
	)

	flag.StringVar(&manuelFilePath, "manuel", manuelDefault, manuelUsage)
	flag.StringVar(&manuelFilePath, "m", manuelDefault, manuelUsage+" (shorthand)")
	flag.StringVar(&faxbotFilePath, "faxbot", faxbotDefault, faxbotUsage)
	flag.StringVar(&faxbotFilePath, "f", faxbotDefault, faxbotUsage+" (shorthand)")
	flag.StringVar(&outputFilePath, "output", outputDefault, outputUsage)
	flag.StringVar(&outputFilePath, "o", outputDefault, outputUsage+" (shorthand)")
}

func main() {
	flag.Parse()
	filterManuel(manuelFilePath, faxbotFilePath, outputFilePath)
}

func filterManuel(manuelPath, faxbotPath, outputPath string) {
	//todo: read and prep manuel and faxbox in parallel
	manuelFile, err := os.Open(manuelPath)
	if err != nil {
		log.Fatal(err)
	}
	defer manuelFile.Close()
	manuel := bufio.NewScanner(manuelFile)

	faxbotFile, err := os.Open(faxbotPath)
	if err != nil {
		log.Fatal(err)
	}
	defer faxbotFile.Close()
	faxbot := bufio.NewScanner(faxbotFile)
	faxbotMonsters := make(map[string]struct{})

	for faxbot.Scan() {
		faxbotMonsters[faxbot.Text()] = struct{}{}
	}

	outputFile, err := os.Create(outputPath)
	if err != nil {
		log.Fatal(err)
	}
	defer outputFile.Close()
	output := bufio.NewWriter(outputFile)
	defer output.Flush()
	newLine := ""
	for manuel.Scan() {
		line := manuel.Text()
		if shouldCopy(line, faxbotMonsters) {
			_, err := output.WriteString(newLine + line)
			if err != nil {
				log.Fatal(err)
			}
			newLine = "\r\n"
		}
	}
}

func shouldCopy(s string, allowed map[string]struct{}) bool {
	if isSectionSeparator(s) {
		return true
	}
	if isSectionHeader(s) {
		return true
	}
	if isInAllowedSet(s, allowed) {
		return true
	}
	return false
}

func isSectionSeparator(s string) bool {
	return sectionSeparator.MatchString(s)
}

func isSectionHeader(s string) bool {
	return sectionHeader.MatchString(s)
}

func isInAllowedSet(s string, allowed map[string]struct{}) bool {
	stringWithBracketsRemoved := bracketsToIgnore.ReplaceAllString(s, "")
	_, ok := allowed[stringWithBracketsRemoved]
	return ok
}
<commit_msg>adjusted some cli help message text<commit_after>package main

import (
	"bufio"
	"flag"
	"log"
	"os"
	"path/filepath"
	"regexp"
)

var (
	sectionSeparator = regexp.MustCompile(`^=====*$`)
	sectionHeader    = regexp.MustCompile(`^\[[A-Z]+.*\]$`)
	bracketsToIgnore = regexp.MustCompile(` \{[0-3]\}$`)
	manuelFilePath   string
	faxbotFilePath   string
	outputFilePath   string
)

func init() {
	const (
		manuelUsage = "path to the missingManuel.ash data file"
		faxbotUsage = "path to the faxbot data file"
		outputUsage = "path to where the output data should be written"
	)
	var (
		defaultDir    = filepath.Clean(`C:\Users\admin\Desktop\kolmafia\samples`)
		manuelDefault = filepath.Join(defaultDir, "monster manuel.txt")
		faxbotDefault = filepath.Join(defaultDir, "faxbot.txt")
		outputDefault = filepath.Join(defaultDir, "filtered_faxbot.txt")
	)

	flag.StringVar(&manuelFilePath, "manuel", manuelDefault, manuelUsage)
	flag.StringVar(&manuelFilePath, "m", manuelDefault, manuelUsage+" (shorthand)")
	flag.StringVar(&faxbotFilePath, "faxbot", faxbotDefault, faxbotUsage)
	flag.StringVar(&faxbotFilePath, "f", faxbotDefault, faxbotUsage+" (shorthand)")
	flag.StringVar(&outputFilePath, "output", outputDefault, outputUsage)
	flag.StringVar(&outputFilePath, "o", outputDefault, outputUsage+" (shorthand)")
}

func main() {
	flag.Parse()
	filterManuel(manuelFilePath, faxbotFilePath, outputFilePath)
}

func filterManuel(manuelPath, faxbotPath, outputPath string) {
	//todo: read and prep manuel and faxbox in parallel
	manuelFile, err := os.Open(manuelPath)
	if err != nil {
		log.Fatal(err)
	}
	defer manuelFile.Close()
	manuel := bufio.NewScanner(manuelFile)

	faxbotFile, err := os.Open(faxbotPath)
	if err != nil {
		log.Fatal(err)
	}
	defer faxbotFile.Close()
	faxbot := bufio.NewScanner(faxbotFile)
	faxbotMonsters := make(map[string]struct{})

	for faxbot.Scan() {
		faxbotMonsters[faxbot.Text()] = struct{}{}
	}

	outputFile, err := os.Create(outputPath)
	if err != nil {
		log.Fatal(err)
	}
	defer outputFile.Close()
	output := bufio.NewWriter(outputFile)
	defer output.Flush()
	newLine := ""
	for manuel.Scan() {
		line := manuel.Text()
		if shouldCopy(line, faxbotMonsters) {
			_, err := output.WriteString(newLine + line)
			if err != nil {
				log.Fatal(err)
			}
			newLine = "\r\n"
		}
	}
}

func shouldCopy(s string, allowed map[string]struct{}) bool {
	if isSectionSeparator(s) {
		return true
	}
	if isSectionHeader(s) {
		return true
	}
	if isInAllowedSet(s, allowed) {
		return true
	}
	return false
}

func isSectionSeparator(s string) bool {
	return sectionSeparator.MatchString(s)
}

func isSectionHeader(s string) bool {
	return sectionHeader.MatchString(s)
}

func isInAllowedSet(s string, allowed map[string]struct{}) bool {
	stringWithBracketsRemoved := bracketsToIgnore.ReplaceAllString(s, "")
	_, ok := allowed[stringWithBracketsRemoved]
	return ok
}
<|endoftext|>


<commit_before>// +build e2e

/*
Copyright 2019 The Tekton Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"strings"
	"testing"

	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1"
	tb "github.com/tektoncd/pipeline/test/builder"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	knativetest "knative.dev/pkg/test"
)

const (
	gitSourceResourceName  = "git-source-resource"
	gitTestTaskName        = "git-check-task"
	gitTestPipelineName    = "git-check-pipeline"
	gitTestPipelineRunName = "git-check-pipeline-run"
)

// TestGitPipelineRun is an integration test that will verify the source code is either fetched or pulled
// successfully under different revision inputs (branch, commitid, tag, ref)
func TestGitPipelineRun(t *testing.T) {
	t.Parallel()

	revisions := []string{"master", "c15aced0e5aaee6456fbe6f7a7e95e0b5b3b2b2f", "c15aced", "release-0.1", "v0.1.0", "refs/pull/347/head"}

	for _, revision := range revisions {

		c, namespace := setup(t)
		knativetest.CleanupOnInterrupt(func() { tearDown(t, c, namespace) }, t.Logf)
		defer tearDown(t, c, namespace)

		t.Logf("Creating Git PipelineResource %s", gitSourceResourceName)
		if _, err := c.PipelineResourceClient.Create(getGitPipelineResource(namespace, revision)); err != nil {
			t.Fatalf("Failed to create Pipeline Resource `%s`: %s", gitSourceResourceName, err)
		}

		t.Logf("Creating Task %s", gitTestTaskName)
		if _, err := c.TaskClient.Create(getGitCheckTask(namespace, t)); err != nil {
			t.Fatalf("Failed to create Task `%s`: %s", gitTestTaskName, err)
		}

		t.Logf("Creating Pipeline %s", gitTestPipelineName)
		if _, err := c.PipelineClient.Create(getGitCheckPipeline(namespace)); err != nil {
			t.Fatalf("Failed to create Pipeline `%s`: %s", gitTestPipelineName, err)
		}

		t.Logf("Creating PipelineRun %s", gitTestPipelineRunName)
		if _, err := c.PipelineRunClient.Create(getGitCheckPipelineRun(namespace)); err != nil {
			t.Fatalf("Failed to create Pipeline `%s`: %s", gitTestPipelineRunName, err)
		}

		if err := WaitForPipelineRunState(c, gitTestPipelineRunName, timeout, PipelineRunSucceed(gitTestPipelineRunName), "PipelineRunCompleted"); err != nil {
			t.Errorf("Error waiting for PipelineRun %s to finish: %s", gitTestPipelineRunName, err)
			t.Fatalf("PipelineRun execution failed")
		}
	}
}

// TestGitPipelineRunFail is a test to ensure that the code extraction from github fails as expected when
// an invalid revision is passed on the pipelineresource.
func TestGitPipelineRunFail(t *testing.T) {
	t.Parallel()

	c, namespace := setup(t)
	knativetest.CleanupOnInterrupt(func() { tearDown(t, c, namespace) }, t.Logf)
	defer tearDown(t, c, namespace)

	t.Logf("Creating Git PipelineResource %s", gitSourceResourceName)
	if _, err := c.PipelineResourceClient.Create(getGitPipelineResource(namespace, "Idontexistrabbitmonkeydonkey")); err != nil {
		t.Fatalf("Failed to create Pipeline Resource `%s`: %s", gitSourceResourceName, err)
	}

	t.Logf("Creating Task %s", gitTestTaskName)
	if _, err := c.TaskClient.Create(getGitCheckTask(namespace, t)); err != nil {
		t.Fatalf("Failed to create Task `%s`: %s", gitTestTaskName, err)
	}

	t.Logf("Creating Pipeline %s", gitTestPipelineName)
	if _, err := c.PipelineClient.Create(getGitCheckPipeline(namespace)); err != nil {
		t.Fatalf("Failed to create Pipeline `%s`: %s", gitTestPipelineName, err)
	}

	t.Logf("Creating PipelineRun %s", gitTestPipelineRunName)
	if _, err := c.PipelineRunClient.Create(getGitCheckPipelineRun(namespace)); err != nil {
		t.Fatalf("Failed to create Pipeline `%s`: %s", gitTestPipelineRunName, err)
	}

	if err := WaitForPipelineRunState(c, gitTestPipelineRunName, timeout, PipelineRunSucceed(gitTestPipelineRunName), "PipelineRunCompleted"); err != nil {
		taskruns, err := c.TaskRunClient.List(metav1.ListOptions{})
		if err != nil {
			t.Errorf("Error getting TaskRun list for PipelineRun %s %s", gitTestPipelineRunName, err)
		}
		for _, tr := range taskruns.Items {
			if tr.Status.PodName != "" {
				p, err := c.KubeClient.Kube.CoreV1().Pods(namespace).Get(tr.Status.PodName, metav1.GetOptions{})
				if err != nil {
					t.Fatalf("Error getting pod `%s` in namespace `%s`", tr.Status.PodName, namespace)
				}

				for _, stat := range p.Status.ContainerStatuses {
					if strings.HasPrefix(stat.Name, "step-git-source-"+gitSourceResourceName) {
						if stat.State.Terminated != nil {
							req := c.KubeClient.Kube.CoreV1().Pods(namespace).GetLogs(p.Name, &corev1.PodLogOptions{Container: stat.Name})
							logContent, err := req.Do().Raw()
							if err != nil {
								t.Fatalf("Error getting pod logs for pod `%s` and container `%s` in namespace `%s`", tr.Status.PodName, stat.Name, namespace)
							}
							// Check for failure messages from fetch and pull in the log file
							if strings.Contains(string(logContent), "couldn't find remote ref Idontexistrabbitmonkeydonkey") &&
								strings.Contains(string(logContent), "pathspec 'Idontexistrabbitmonkeydonkey' did not match any file(s) known to git") {
								t.Logf("Found exepected errors when retrieving non-existent git revision")
							} else {
								t.Logf("Container `%s` log File: %s", stat.Name, logContent)
								t.Fatalf("The git code extraction did not fail as expected.  Expected errors not found in log file.")
							}
						}
					}
				}
			}
		}

	} else {
		t.Fatalf("PipelineRun succeeded when should have failed")
	}
}

func getGitPipelineResource(namespace, revision string) *v1alpha1.PipelineResource {
	return tb.PipelineResource(gitSourceResourceName, namespace, tb.PipelineResourceSpec(
		v1alpha1.PipelineResourceTypeGit,
		tb.PipelineResourceSpecParam("Url", "https://github.com/tektoncd/pipeline"),
		tb.PipelineResourceSpecParam("Revision", revision),
	))
}

func getGitCheckTask(namespace string, t *testing.T) *v1alpha1.Task {
	return tb.Task(gitTestTaskName, namespace, tb.TaskSpec(
		tb.TaskInputs(tb.InputsResource("gitsource", v1alpha1.PipelineResourceTypeGit)),
		tb.Step("git", "alpine/git", tb.StepArgs("--git-dir=/workspace/gitsource/.git", "show")),
	))
}

func getGitCheckPipeline(namespace string) *v1alpha1.Pipeline {
	return tb.Pipeline(gitTestPipelineName, namespace, tb.PipelineSpec(
		tb.PipelineDeclaredResource("git-repo", "git"),
		tb.PipelineTask("git-check", gitTestTaskName,
			tb.PipelineTaskInputResource("gitsource", "git-repo"),
		),
	))
}

func getGitCheckPipelineRun(namespace string) *v1alpha1.PipelineRun {
	return tb.PipelineRun(gitTestPipelineRunName, namespace, tb.PipelineRunSpec(
		gitTestPipelineName,
		tb.PipelineRunResourceBinding("git-repo", tb.PipelineResourceBindingRef(gitSourceResourceName)),
	))
}
<commit_msg>Avoid cases when comparing in TestGitPipelineRun<commit_after>// +build e2e

/*
Copyright 2019 The Tekton Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"strings"
	"testing"

	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1"
	tb "github.com/tektoncd/pipeline/test/builder"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	knativetest "knative.dev/pkg/test"
)

const (
	gitSourceResourceName  = "git-source-resource"
	gitTestTaskName        = "git-check-task"
	gitTestPipelineName    = "git-check-pipeline"
	gitTestPipelineRunName = "git-check-pipeline-run"
)

// TestGitPipelineRun is an integration test that will verify the source code is either fetched or pulled
// successfully under different revision inputs (branch, commitid, tag, ref)
func TestGitPipelineRun(t *testing.T) {
	t.Parallel()

	revisions := []string{"master", "c15aced0e5aaee6456fbe6f7a7e95e0b5b3b2b2f", "c15aced", "release-0.1", "v0.1.0", "refs/pull/347/head"}

	for _, revision := range revisions {

		c, namespace := setup(t)
		knativetest.CleanupOnInterrupt(func() { tearDown(t, c, namespace) }, t.Logf)
		defer tearDown(t, c, namespace)

		t.Logf("Creating Git PipelineResource %s", gitSourceResourceName)
		if _, err := c.PipelineResourceClient.Create(getGitPipelineResource(namespace, revision)); err != nil {
			t.Fatalf("Failed to create Pipeline Resource `%s`: %s", gitSourceResourceName, err)
		}

		t.Logf("Creating Task %s", gitTestTaskName)
		if _, err := c.TaskClient.Create(getGitCheckTask(namespace, t)); err != nil {
			t.Fatalf("Failed to create Task `%s`: %s", gitTestTaskName, err)
		}

		t.Logf("Creating Pipeline %s", gitTestPipelineName)
		if _, err := c.PipelineClient.Create(getGitCheckPipeline(namespace)); err != nil {
			t.Fatalf("Failed to create Pipeline `%s`: %s", gitTestPipelineName, err)
		}

		t.Logf("Creating PipelineRun %s", gitTestPipelineRunName)
		if _, err := c.PipelineRunClient.Create(getGitCheckPipelineRun(namespace)); err != nil {
			t.Fatalf("Failed to create Pipeline `%s`: %s", gitTestPipelineRunName, err)
		}

		if err := WaitForPipelineRunState(c, gitTestPipelineRunName, timeout, PipelineRunSucceed(gitTestPipelineRunName), "PipelineRunCompleted"); err != nil {
			t.Errorf("Error waiting for PipelineRun %s to finish: %s", gitTestPipelineRunName, err)
			t.Fatalf("PipelineRun execution failed")
		}
	}
}

// TestGitPipelineRunFail is a test to ensure that the code extraction from github fails as expected when
// an invalid revision is passed on the pipelineresource.
func TestGitPipelineRunFail(t *testing.T) {
	t.Parallel()

	c, namespace := setup(t)
	knativetest.CleanupOnInterrupt(func() { tearDown(t, c, namespace) }, t.Logf)
	defer tearDown(t, c, namespace)

	t.Logf("Creating Git PipelineResource %s", gitSourceResourceName)
	if _, err := c.PipelineResourceClient.Create(getGitPipelineResource(namespace, "Idontexistrabbitmonkeydonkey")); err != nil {
		t.Fatalf("Failed to create Pipeline Resource `%s`: %s", gitSourceResourceName, err)
	}

	t.Logf("Creating Task %s", gitTestTaskName)
	if _, err := c.TaskClient.Create(getGitCheckTask(namespace, t)); err != nil {
		t.Fatalf("Failed to create Task `%s`: %s", gitTestTaskName, err)
	}

	t.Logf("Creating Pipeline %s", gitTestPipelineName)
	if _, err := c.PipelineClient.Create(getGitCheckPipeline(namespace)); err != nil {
		t.Fatalf("Failed to create Pipeline `%s`: %s", gitTestPipelineName, err)
	}

	t.Logf("Creating PipelineRun %s", gitTestPipelineRunName)
	if _, err := c.PipelineRunClient.Create(getGitCheckPipelineRun(namespace)); err != nil {
		t.Fatalf("Failed to create Pipeline `%s`: %s", gitTestPipelineRunName, err)
	}

	if err := WaitForPipelineRunState(c, gitTestPipelineRunName, timeout, PipelineRunSucceed(gitTestPipelineRunName), "PipelineRunCompleted"); err != nil {
		taskruns, err := c.TaskRunClient.List(metav1.ListOptions{})
		if err != nil {
			t.Errorf("Error getting TaskRun list for PipelineRun %s %s", gitTestPipelineRunName, err)
		}
		for _, tr := range taskruns.Items {
			if tr.Status.PodName != "" {
				p, err := c.KubeClient.Kube.CoreV1().Pods(namespace).Get(tr.Status.PodName, metav1.GetOptions{})
				if err != nil {
					t.Fatalf("Error getting pod `%s` in namespace `%s`", tr.Status.PodName, namespace)
				}

				for _, stat := range p.Status.ContainerStatuses {
					if strings.HasPrefix(stat.Name, "step-git-source-"+gitSourceResourceName) {
						if stat.State.Terminated != nil {
							req := c.KubeClient.Kube.CoreV1().Pods(namespace).GetLogs(p.Name, &corev1.PodLogOptions{Container: stat.Name})
							logContent, err := req.Do().Raw()
							if err != nil {
								t.Fatalf("Error getting pod logs for pod `%s` and container `%s` in namespace `%s`", tr.Status.PodName, stat.Name, namespace)
							}
							// Check for failure messages from fetch and pull in the log file
							if strings.Contains(strings.ToLower(string(logContent)), "couldn't find remote ref idontexistrabbitmonkeydonkey") &&
								strings.Contains(strings.ToLower(string(logContent)), "pathspec 'idontexistrabbitmonkeydonkey' did not match any file(s) known to git") {
								t.Logf("Found exepected errors when retrieving non-existent git revision")
							} else {
								t.Logf("Container `%s` log File: %s", stat.Name, logContent)
								t.Fatalf("The git code extraction did not fail as expected.  Expected errors not found in log file.")
							}
						}
					}
				}
			}
		}

	} else {
		t.Fatalf("PipelineRun succeeded when should have failed")
	}
}

func getGitPipelineResource(namespace, revision string) *v1alpha1.PipelineResource {
	return tb.PipelineResource(gitSourceResourceName, namespace, tb.PipelineResourceSpec(
		v1alpha1.PipelineResourceTypeGit,
		tb.PipelineResourceSpecParam("Url", "https://github.com/tektoncd/pipeline"),
		tb.PipelineResourceSpecParam("Revision", revision),
	))
}

func getGitCheckTask(namespace string, t *testing.T) *v1alpha1.Task {
	return tb.Task(gitTestTaskName, namespace, tb.TaskSpec(
		tb.TaskInputs(tb.InputsResource("gitsource", v1alpha1.PipelineResourceTypeGit)),
		tb.Step("git", "alpine/git", tb.StepArgs("--git-dir=/workspace/gitsource/.git", "show")),
	))
}

func getGitCheckPipeline(namespace string) *v1alpha1.Pipeline {
	return tb.Pipeline(gitTestPipelineName, namespace, tb.PipelineSpec(
		tb.PipelineDeclaredResource("git-repo", "git"),
		tb.PipelineTask("git-check", gitTestTaskName,
			tb.PipelineTaskInputResource("gitsource", "git-repo"),
		),
	))
}

func getGitCheckPipelineRun(namespace string) *v1alpha1.PipelineRun {
	return tb.PipelineRun(gitTestPipelineRunName, namespace, tb.PipelineRunSpec(
		gitTestPipelineName,
		tb.PipelineRunResourceBinding("git-repo", tb.PipelineResourceBindingRef(gitSourceResourceName)),
	))
}
<|endoftext|>


<commit_before>package ora

/*
#include <oci.h>
#include "version.h"
*/
import "C"
import (
	"sync"
	"unsafe"
)

type arrHlp struct {
	sync.Mutex
	curlen     C.ub4
	nullInds   []C.sb2
	alen       []C.ACTUAL_LENGTH_TYPE
	rcode      []C.ub2
	allocated  []bool
	isAssocArr bool
}

type ociDef struct {
	ocidef *C.OCIDefine
	rset   *Rset
	arrHlp
}

func (d *ociDef) defineByPos(position int, valuep unsafe.Pointer, valueSize int, dty int) error {
	d.ensureFetchLength(d.rset.stmt.Cfg().FetchLen())
	// If you omit the rlenp parameter of OCIDefineByPos(), returned values are blank-padded to the buffer length, and NULLs are returned as a string of blank characters. If rlenp is included, returned values are not blank-padded. Instead, their actual lengths are returned in the rlenp parameter.
	if r := C.OCIDEFINEBYPOS(
		d.rset.ocistmt,    //OCIStmt     *stmtp,
		&d.ocidef,         //OCIDefine   **defnpp,
		d.rset.env.ocierr, //OCIError    *errhp,
		C.ub4(position),   //ub4         position,
		valuep,            //void        *valuep,
		C.LENGTH_TYPE(valueSize),       //sb8         value_sz,
		C.ub2(dty),                     //ub2         dty,
		unsafe.Pointer(&d.nullInds[0]), //void        *indp,
		&d.alen[0],                     //ub4         *rlenp,
		&d.rcode[0],                    //ub2         *rcodep,
		C.OCI_DEFAULT,                  //ub4         mode );
	); r == C.OCI_ERROR {
		return d.rset.stmt.ses.srv.env.ociError()
	}
	if r := C.OCIDefineArrayOfStruct(
		d.ocidef,          //OCIDefine *defnp
		d.rset.env.ocierr, //OCIError *errhp,
		C.ub4(valueSize),  //ub4 pvskip,
		2,                 //ub4 indskip,
		C.ACTUAL_LENGTH_LENGTH, //ub4 rlskip,
		2, //ub4 rcskip
	); r == C.OCI_ERROR {
		return d.rset.env.ociError()
	}
	return nil
}

var (
	sb2Pool  = sync.Pool{New: func() interface{} { return []C.sb2{} }}
	ub2Pool  = sync.Pool{New: func() interface{} { return []C.ub2{} }}
	alenPool = sync.Pool{New: func() interface{} { return []C.ACTUAL_LENGTH_TYPE{} }}
)

func (d *arrHlp) ensureAllocatedLength(length int) {
	if cap(d.allocated) < length {
		d.allocated = make([]bool, length)
		return
	}
	d.allocated = d.allocated[:length]
	for i := range d.allocated {
		d.allocated[i] = false
	}
}

func (a *arrHlp) ensureFetchLength(length int) {
	if length <= 0 || length >= MaxFetchLen {
		length = MaxFetchLen
	}
	a.Lock()
	defer a.Unlock()
	if cap(a.nullInds) >= length {
		a.nullInds = a.nullInds[:length]
	} else {
		if a.nullInds = sb2Pool.Get().([]C.sb2); cap(a.nullInds) < length {
			a.nullInds = make([]C.sb2, length)
		} else {
			a.nullInds = a.nullInds[:length]
		}
	}
	if cap(a.alen) >= length {
		a.alen = a.alen[:length]
	} else {
		if a.alen = alenPool.Get().([]C.ACTUAL_LENGTH_TYPE); cap(a.alen) < length {
			a.alen = make([]C.ACTUAL_LENGTH_TYPE, length)
		} else {
			a.alen = a.alen[:length]
		}
	}
	if cap(a.rcode) >= length {
		a.rcode = a.rcode[:length]
	} else {
		if a.rcode = ub2Pool.Get().([]C.ub2); cap(a.rcode) < length {
			a.rcode = make([]C.ub2, length)
		} else {
			a.rcode = a.rcode[:length]
		}
	}
}

// ensureBindArrLength calculates the needed length and capacity,
// and sets up the helper arrays for binding PL/SQL Table.
//
// Returns whether and element is needed to be appended to the value slice.
func (a *arrHlp) ensureBindArrLength(
	length, capacity *int,
	isAssocArray bool,
) (iterations uint32, curlenp *C.ub4, needsAppend bool) {
	a.Lock()
	defer a.Unlock()
	a.curlen = C.ub4(*length) // the real length, not L!
	if isAssocArray {
		// for PL/SQL associative arrays
		curlenp = &a.curlen
		iterations = 1
		a.isAssocArr = true
		if *length == 0 {
			*length = 1
			if *capacity == 0 {
				needsAppend = true
				*capacity = 1
			}
		}
	} else {
		curlenp = nil
		iterations = uint32(*length)
		a.isAssocArr = false
	}
	L, C := *length, *capacity
	if cap(a.nullInds) >= C {
		a.nullInds = (a.nullInds)[:L]
	} else {
		if a.nullInds = sb2Pool.Get().([]C.sb2); cap(a.nullInds) < C {
			a.nullInds = make([]C.sb2, L, C)
		} else {
			a.nullInds = (a.nullInds)[:L]
		}
	}
	if cap(a.alen) >= C {
		a.alen = a.alen[:L]
	} else {
		if a.alen = alenPool.Get().([]C.ACTUAL_LENGTH_TYPE); cap(a.alen) < C {
			a.alen = make([]C.ACTUAL_LENGTH_TYPE, L, C)
		} else {
			a.alen = a.alen[:L]
		}
	}
	if cap(a.rcode) >= C {
		a.rcode = a.rcode[:L]
	} else {
		if a.rcode = ub2Pool.Get().([]C.ub2); cap(a.rcode) < C {
			a.rcode = make([]C.ub2, L, C)
		} else {
			a.rcode = a.rcode[:L]
		}
	}
	return iterations, curlenp, needsAppend
}

// IsAssocArr returns true if the bind uses PL/SQL Table.
func (a arrHlp) IsAssocArr() bool {
	return a.isAssocArr
}

// close nils the slices, except when this is a PL/SQL Table.
//
// The reason for this is that for PL/SQL Tables, after exe returns,
// the bound slices can be reused; otherwise, they are still in use for
// the subsequent iterations!
func (a *arrHlp) close() error {
	if a == nil {
		return nil
	}
	a.Lock()
	defer a.Unlock()
	if a.isAssocArr {
		return nil
	}
	if a.nullInds != nil {
		sb2Pool.Put(a.nullInds)
		a.nullInds = nil
	}
	if a.alen != nil {
		alenPool.Put(a.alen)
		a.alen = nil
	}
	if a.rcode != nil {
		ub2Pool.Put(a.rcode)
		a.rcode = nil
	}
	if a.allocated != nil {
		a.allocated = nil
	}
	return nil
}

func getMaxarrLen(C int, isAssocArray bool) C.ub4 {
	if !isAssocArray {
		return 0
	}
	return C.ub4(C)
}
<commit_msg>less locking in arrHlp<commit_after>package ora

/*
#include <oci.h>
#include "version.h"
*/
import "C"
import (
	"sync"
	"unsafe"
)

type arrHlp struct {
	sync.Mutex
	curlen     C.ub4
	nullInds   []C.sb2
	alen       []C.ACTUAL_LENGTH_TYPE
	rcode      []C.ub2
	allocated  []bool
	isAssocArr bool
}

type ociDef struct {
	ocidef *C.OCIDefine
	rset   *Rset
	arrHlp
}

func (d *ociDef) defineByPos(position int, valuep unsafe.Pointer, valueSize int, dty int) error {
	d.ensureFetchLength(d.rset.stmt.Cfg().FetchLen())
	// If you omit the rlenp parameter of OCIDefineByPos(), returned values are blank-padded to the buffer length, and NULLs are returned as a string of blank characters. If rlenp is included, returned values are not blank-padded. Instead, their actual lengths are returned in the rlenp parameter.
	if r := C.OCIDEFINEBYPOS(
		d.rset.ocistmt,    //OCIStmt     *stmtp,
		&d.ocidef,         //OCIDefine   **defnpp,
		d.rset.env.ocierr, //OCIError    *errhp,
		C.ub4(position),   //ub4         position,
		valuep,            //void        *valuep,
		C.LENGTH_TYPE(valueSize),       //sb8         value_sz,
		C.ub2(dty),                     //ub2         dty,
		unsafe.Pointer(&d.nullInds[0]), //void        *indp,
		&d.alen[0],                     //ub4         *rlenp,
		&d.rcode[0],                    //ub2         *rcodep,
		C.OCI_DEFAULT,                  //ub4         mode );
	); r == C.OCI_ERROR {
		return d.rset.stmt.ses.srv.env.ociError()
	}
	if r := C.OCIDefineArrayOfStruct(
		d.ocidef,          //OCIDefine *defnp
		d.rset.env.ocierr, //OCIError *errhp,
		C.ub4(valueSize),  //ub4 pvskip,
		2,                 //ub4 indskip,
		C.ACTUAL_LENGTH_LENGTH, //ub4 rlskip,
		2, //ub4 rcskip
	); r == C.OCI_ERROR {
		return d.rset.env.ociError()
	}
	return nil
}

var (
	sb2Pool  = sync.Pool{New: func() interface{} { return []C.sb2{} }}
	ub2Pool  = sync.Pool{New: func() interface{} { return []C.ub2{} }}
	alenPool = sync.Pool{New: func() interface{} { return []C.ACTUAL_LENGTH_TYPE{} }}
)

func (d *arrHlp) ensureAllocatedLength(length int) {
	if cap(d.allocated) < length {
		d.allocated = make([]bool, length)
		return
	}
	d.allocated = d.allocated[:length]
	for i := range d.allocated {
		d.allocated[i] = false
	}
}

func (a *arrHlp) ensureFetchLength(length int) {
	if length <= 0 || length >= MaxFetchLen {
		length = MaxFetchLen
	}
	if cap(a.nullInds) >= length {
		a.nullInds = a.nullInds[:length]
	} else {
		if a.nullInds = sb2Pool.Get().([]C.sb2); cap(a.nullInds) < length {
			a.nullInds = make([]C.sb2, length)
		} else {
			a.nullInds = a.nullInds[:length]
		}
	}
	if cap(a.alen) >= length {
		a.alen = a.alen[:length]
	} else {
		if a.alen = alenPool.Get().([]C.ACTUAL_LENGTH_TYPE); cap(a.alen) < length {
			a.alen = make([]C.ACTUAL_LENGTH_TYPE, length)
		} else {
			a.alen = a.alen[:length]
		}
	}
	if cap(a.rcode) >= length {
		a.rcode = a.rcode[:length]
	} else {
		if a.rcode = ub2Pool.Get().([]C.ub2); cap(a.rcode) < length {
			a.rcode = make([]C.ub2, length)
		} else {
			a.rcode = a.rcode[:length]
		}
	}
}

// ensureBindArrLength calculates the needed length and capacity,
// and sets up the helper arrays for binding PL/SQL Table.
//
// Returns whether and element is needed to be appended to the value slice.
func (a *arrHlp) ensureBindArrLength(
	length, capacity *int,
	isAssocArray bool,
) (iterations uint32, curlenp *C.ub4, needsAppend bool) {
	a.curlen = C.ub4(*length) // the real length, not L!
	if isAssocArray {
		// for PL/SQL associative arrays
		curlenp = &a.curlen
		iterations = 1
		a.isAssocArr = true
		if *length == 0 {
			*length = 1
			if *capacity == 0 {
				needsAppend = true
				*capacity = 1
			}
		}
	} else {
		curlenp = nil
		iterations = uint32(*length)
		a.isAssocArr = false
	}
	L, C := *length, *capacity
	if cap(a.nullInds) >= C {
		a.nullInds = (a.nullInds)[:L]
	} else {
		if a.nullInds = sb2Pool.Get().([]C.sb2); cap(a.nullInds) < C {
			a.nullInds = make([]C.sb2, L, C)
		} else {
			a.nullInds = (a.nullInds)[:L]
		}
	}
	if cap(a.alen) >= C {
		a.alen = a.alen[:L]
	} else {
		if a.alen = alenPool.Get().([]C.ACTUAL_LENGTH_TYPE); cap(a.alen) < C {
			a.alen = make([]C.ACTUAL_LENGTH_TYPE, L, C)
		} else {
			a.alen = a.alen[:L]
		}
	}
	if cap(a.rcode) >= C {
		a.rcode = a.rcode[:L]
	} else {
		if a.rcode = ub2Pool.Get().([]C.ub2); cap(a.rcode) < C {
			a.rcode = make([]C.ub2, L, C)
		} else {
			a.rcode = a.rcode[:L]
		}
	}
	return iterations, curlenp, needsAppend
}

// IsAssocArr returns true if the bind uses PL/SQL Table.
func (a arrHlp) IsAssocArr() bool {
	return a.isAssocArr
}

// close nils the slices, except when this is a PL/SQL Table.
//
// The reason for this is that for PL/SQL Tables, after exe returns,
// the bound slices can be reused; otherwise, they are still in use for
// the subsequent iterations!
func (a *arrHlp) close() error {
	if a == nil {
		return nil
	}
	if a.isAssocArr {
		return nil
	}
	if a.nullInds != nil {
		sb2Pool.Put(a.nullInds)
		a.nullInds = nil
	}
	if a.alen != nil {
		alenPool.Put(a.alen)
		a.alen = nil
	}
	if a.rcode != nil {
		ub2Pool.Put(a.rcode)
		a.rcode = nil
	}
	if a.allocated != nil {
		a.allocated = nil
	}
	return nil
}

func getMaxarrLen(C int, isAssocArray bool) C.ub4 {
	if !isAssocArray {
		return 0
	}
	return C.ub4(C)
}
<|endoftext|>


<commit_before>package bitfinex

// Available pairs
const (
	BTCUSD = "BTCUSD"
	LTCUSD = "LTCUSD"
	LTCBTC = "LTCBTC"
	ETHUSD = "ETHUSD"
	ETHBTC = "ETHBTC"
	ETCUSD = "ETCUSD"
	ETCBTC = "ETCBTC"
	BFXUSD = "BFXUSD"
	BFXBTC = "BFXBTC"
	ZECUSD = "ZECUSD"
	ZECBTC = "ZECBTC"
	XMRUSD = "XMRUSD"
	XMRBTC = "XMRBTC"
	RRTUSD = "RRTUSD"
	RRTBTC = "RRTBTC"
	XRPUSD = "XRPUSD"
	XRPBTC = "XRPBTC"
	EOSETH = "EOSETH"
	EOSUSD = "EOSUSD"
	EOSBTC = "EOSBTC"
	IOTUSD = "IOTUSD"
	IOTBTC = "IOTBTC"
	IOTETH = "IOTETH"
	BCCBTC = "BCCBTC"
	BCUBTC = "BCUBTC"
	BCCUSD = "BCCUSD"
	BCUUSD = "BCUUSD"
)
<commit_msg>v2/pairs: remove de-listed pairs and add more<commit_after>package bitfinex

// Some of the more available paris. See https://api.bitfinex.com/v1/symbols for
// an up to date list

const (
	BTCUSD = "BTCUSD"
	BTCFUSTF = "BTCF0:USTF0"
	BTCEUR = "BTCEUR"
	BTCJPY = "BTCJPY"
	LTCUSD = "LTCUSD"
	LTCBTC = "LTCBTC"
	ETHUSD = "ETHUSD"
	ETHFUSTF ="ETHF0:USTF0"
	ETHBTC = "ETHBTC"
	ETHEUR = "ETHEUR"
	ETHJPY = "ETHJPY"
	ETHGBP = "ETHGBP"
	ETHUST = "ETHUST"
	ETCUSD = "ETCUSD"
	ETCBTC = "ETCBTC"
	ZECUSD = "ZECUSD"
	ZECBTC = "ZECBTC"
	XMRUSD = "XMRUSD"
	XMRBTC = "XMRBTC"
	RRTUSD = "RRTUSD"
	RRTBTC = "RRTBTC"
	XRPUSD = "XRPUSD"
	XRPBTC = "XRPBTC"
	EOSETH = "EOSETH"
	EOSUSD = "EOSUSD"
	EOSBTC = "EOSBTC"
	EOSEUR = "EOSEUR"
	EOSJPY = "EOSJPY"
	EOSGBP = "EOSGBP"
	NEOUSD = "NEOUSD"
	NEOBTC = "NEOBTC"
	NEOETH = "NEOETH"
	TRXUSD = "TRXUSD"
	TRXBTC = "TRXBTC"
	TRXETH = "TRXETH"
	IOTUSD = "IOTUSD"
	IOTBTC = "IOTBTC"
	IOTETH = "IOTETH"
	IOTEUR = "IOTEUR"
	IOTJPY = "IOTJPY"
	IOTGBP = "IOTGBP"
	LEOUSD = "LEOUSD"
	LEOBTC = "LEOBTC"
	LEOUST = "LEOUST"
	LEOEOS = "LEOEOS"
	LEOETH = "LEOETH"
)
<|endoftext|>


<commit_before>package httpdoc

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"reflect"
	"testing"

	"github.com/golang/protobuf/proto"
	"github.com/tenntenn/gpath"
)

var (
	defaultUnmarshalFunc = json.Unmarshal

	defaultAssertFunc = func(t *testing.T, expected, actual interface{}, desc string) {
		if !reflect.DeepEqual(expected, actual) {
			tFatalf(t, "%s: got %#v(%T), want %#v(%T)", desc, actual, actual, expected, expected)
		}
	}

	defaultFatalFunc = func(t *testing.T, format string, args ...interface{}) {
		t.Fatalf(format, args...)
	}

	protoUnmarshalFunc = func(data []byte, v interface{}) error {
		unmashaler, ok := v.(proto.Unmarshaler)
		if !ok {
			return fmt.Errorf("failed to type assert to Unmashaler: %T must implement proto.Unmarshaler interface", v)
		}
		return unmashaler.Unmarshal(data)
	}
)

var tFatalf fatalFunc = defaultFatalFunc

type (
	assertFunc    func(t *testing.T, expected, actual interface{}, desc string)
	fatalFunc     func(t *testing.T, format string, args ...interface{})
	unmarshalFunc func(data []byte, v interface{}) error
)

// Validator takes test cases and checks whether recorded values are equal to the given expected values.
// If not, it fails in the given test context. If ok, it uses the result for documentation.
type Validator struct {
	record *record

	unmarshalFunc unmarshalFunc
	assertFunc    assertFunc

	requestParams  []Data
	requestHeaders []Data
	requestFields  []Data

	responseHeaders []Data
	responseFields  []Data
}

type record struct {
	requestParams  url.Values
	requestHeaders http.Header
	requestBody    []byte

	responseStatusCode int
	responseHeaders    http.Header
	responseBody       []byte
}

// TestCase is test case validator uses. Validator inspects and extract request & response value based on
// Target (e.g, when testing request params, target is parameter name. when testing response
// body, target is filed name) and asserts with Expected value.
//
// TestCase can be used like table-driven way.
//
//   validator.RequestParams(t, []httpdoc.TestCase{
//       {"token", "12345", "Request token"},
//       {"pretty", "true", "Pretty print response message"},
//	 })
//
type TestCase struct {
	Target      string
	Expected    interface{}
	Description string
	AssertFunc  assertFunc
}

func newValidator() *Validator {
	return &Validator{
		unmarshalFunc: defaultUnmarshalFunc,
		assertFunc:    defaultAssertFunc,
		record:        &record{},
	}
}

// ResponseStatusCode validates response status code is expected or not.
func (v *Validator) ResponseStatusCode(t *testing.T, expected int) {
	v.assertFunc(t, expected, v.record.responseStatusCode, "response status code")
}

// RequestParams validated request params are expected or not.
func (v *Validator) RequestParams(t *testing.T, cases []TestCase) {
	for _, tc := range cases {
		data := Data{
			Name:        tc.Target,
			Value:       tc.Expected,
			Description: tc.Description,
		}
		v.requestParams = append(v.requestParams, data)
		pickAssertFunc(&tc, v)(t, tc.Expected, v.record.requestParams.Get(tc.Target), tc.Description)
	}
}

// RequestHeaders validates request headers are expected or not.
func (v *Validator) RequestHeaders(t *testing.T, cases []TestCase) {
	for _, tc := range cases {
		data := Data{
			Name:        tc.Target,
			Value:       tc.Expected,
			Description: tc.Description,
		}
		v.requestHeaders = append(v.requestHeaders, data)

		actual := v.record.requestHeaders.Get(tc.Target)
		if actual == "" {
			h, ok := v.record.requestHeaders[tc.Target]
			if !ok || len(h) == 0 {
				tFatalf(t, "request header %q is not found", tc.Target)
				return
			}
			actual = h[0]
		}

		pickAssertFunc(&tc, v)(t, tc.Expected, actual, tc.Description)
	}
}

// ResponseHeaders validates response headers are expected or not.
func (v *Validator) ResponseHeaders(t *testing.T, cases []TestCase) {
	for _, tc := range cases {
		data := Data{
			Name:        tc.Target,
			Value:       tc.Expected,
			Description: tc.Description,
		}
		v.responseHeaders = append(v.responseHeaders, data)

		actual := v.record.responseHeaders.Get(tc.Target)
		if actual == "" {
			h, ok := v.record.responseHeaders[tc.Target]
			if !ok || len(h) == 0 {
				tFatalf(t, "request header %q is not found", tc.Target)
				return
			}
			actual = h[0]
		}
		pickAssertFunc(&tc, v)(t, tc.Expected, actual, tc.Description)
	}
}

// RequestBody validates request body's fileds are expected or not. The request body
// is unmarshaled to the given struct. To extract a filed to validate, this uses dot-seprated
// expression in TestCase.Target. For example, if you want to access `Email` value in the
// following struct use `Setting.Name` in Target.
//
//   type User struct {
//       Setting Setting
//   }
//
//   type Setting struct {
//       Email string
//   }
//
func (v *Validator) RequestBody(t *testing.T, cases []TestCase, request interface{}) {
	// Unmarshal request body into the given struct
	if err := v.unmarshalFunc(v.record.requestBody, request); err != nil {
		tFatalf(t, "Failed to unmarshal request body: %s", err)
		return
	}
	v.validateFields(t, cases, request, &v.requestFields)
}

// ResponseBody validates response body's fields are expected or not. The response body
// is unmarshaled to the given struct. To extract a filed to validate, this uses dot-seprated
// expression in TestCase.Target. For example, if you want to access `Email` value in the
// following struct use `Setting.Name` in Target.
//
//   type User struct {
//       Setting Setting
//   }
//
//   type Setting struct {
//       Email string
//   }
//
func (v *Validator) ResponseBody(t *testing.T, cases []TestCase, response interface{}) {
	// Unmarshal request body into the given struct
	if err := v.unmarshalFunc(v.record.responseBody, response); err != nil {
		tFatalf(t, "Failed to unmarshal response body: %s", err)
	}
	v.validateFields(t, cases, response, &v.responseFields)
}

func (vl *Validator) validateFields(t *testing.T, cases []TestCase, v interface{}, fields *[]Data) {
	for _, tc := range cases {
		data := Data{
			Name:        tc.Target,
			Value:       tc.Expected,
			Description: tc.Description,
		}
		*fields = append(*fields, data)
		actual, _ := gpath.At(v, tc.Target)
		pickAssertFunc(&tc, vl)(t, tc.Expected, actual, tc.Description)
	}
}

func pickAssertFunc(tc *TestCase, v *Validator) assertFunc {
	if tc.AssertFunc != nil {
		return tc.AssertFunc
	}
	return v.assertFunc
}
<commit_msg>added NewTestCase.<commit_after>package httpdoc

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"reflect"
	"testing"

	"github.com/golang/protobuf/proto"
	"github.com/tenntenn/gpath"
)

var (
	defaultUnmarshalFunc = json.Unmarshal

	defaultAssertFunc = func(t *testing.T, expected, actual interface{}, desc string) {
		if !reflect.DeepEqual(expected, actual) {
			tFatalf(t, "%s: got %#v(%T), want %#v(%T)", desc, actual, actual, expected, expected)
		}
	}

	defaultFatalFunc = func(t *testing.T, format string, args ...interface{}) {
		t.Fatalf(format, args...)
	}

	protoUnmarshalFunc = func(data []byte, v interface{}) error {
		unmashaler, ok := v.(proto.Unmarshaler)
		if !ok {
			return fmt.Errorf("failed to type assert to Unmashaler: %T must implement proto.Unmarshaler interface", v)
		}
		return unmashaler.Unmarshal(data)
	}
)

var tFatalf fatalFunc = defaultFatalFunc

type (
	assertFunc    func(t *testing.T, expected, actual interface{}, desc string)
	fatalFunc     func(t *testing.T, format string, args ...interface{})
	unmarshalFunc func(data []byte, v interface{}) error
)

// Validator takes test cases and checks whether recorded values are equal to the given expected values.
// If not, it fails in the given test context. If ok, it uses the result for documentation.
type Validator struct {
	record *record

	unmarshalFunc unmarshalFunc
	assertFunc    assertFunc

	requestParams  []Data
	requestHeaders []Data
	requestFields  []Data

	responseHeaders []Data
	responseFields  []Data
}

type record struct {
	requestParams  url.Values
	requestHeaders http.Header
	requestBody    []byte

	responseStatusCode int
	responseHeaders    http.Header
	responseBody       []byte
}

// TestCase is test case validator uses. Validator inspects and extract request & response value based on
// Target (e.g, when testing request params, target is parameter name. when testing response
// body, target is filed name) and asserts with Expected value.
//
// TestCase can be used like table-driven way.
//
//   validator.RequestParams(t, []httpdoc.TestCase{
//       {"token", "12345", "Request token"},
//       {"pretty", "true", "Pretty print response message"},
//	 })
//
type TestCase struct {
	Target      string
	Expected    interface{}
	Description string
	AssertFunc  assertFunc
}

// NewTestCase returns new TestCase.
func NewTestCase(target string, expected interface{}, description string) TestCase {
	return TestCase{Target: target, Expected: expected, Description: description}
}

func newValidator() *Validator {
	return &Validator{
		unmarshalFunc: defaultUnmarshalFunc,
		assertFunc:    defaultAssertFunc,
		record:        &record{},
	}
}

// ResponseStatusCode validates response status code is expected or not.
func (v *Validator) ResponseStatusCode(t *testing.T, expected int) {
	v.assertFunc(t, expected, v.record.responseStatusCode, "response status code")
}

// RequestParams validated request params are expected or not.
func (v *Validator) RequestParams(t *testing.T, cases []TestCase) {
	for _, tc := range cases {
		data := Data{
			Name:        tc.Target,
			Value:       tc.Expected,
			Description: tc.Description,
		}
		v.requestParams = append(v.requestParams, data)
		pickAssertFunc(&tc, v)(t, tc.Expected, v.record.requestParams.Get(tc.Target), tc.Description)
	}
}

// RequestHeaders validates request headers are expected or not.
func (v *Validator) RequestHeaders(t *testing.T, cases []TestCase) {
	for _, tc := range cases {
		data := Data{
			Name:        tc.Target,
			Value:       tc.Expected,
			Description: tc.Description,
		}
		v.requestHeaders = append(v.requestHeaders, data)

		actual := v.record.requestHeaders.Get(tc.Target)
		if actual == "" {
			h, ok := v.record.requestHeaders[tc.Target]
			if !ok || len(h) == 0 {
				tFatalf(t, "request header %q is not found", tc.Target)
				return
			}
			actual = h[0]
		}

		pickAssertFunc(&tc, v)(t, tc.Expected, actual, tc.Description)
	}
}

// ResponseHeaders validates response headers are expected or not.
func (v *Validator) ResponseHeaders(t *testing.T, cases []TestCase) {
	for _, tc := range cases {
		data := Data{
			Name:        tc.Target,
			Value:       tc.Expected,
			Description: tc.Description,
		}
		v.responseHeaders = append(v.responseHeaders, data)

		actual := v.record.responseHeaders.Get(tc.Target)
		if actual == "" {
			h, ok := v.record.responseHeaders[tc.Target]
			if !ok || len(h) == 0 {
				tFatalf(t, "request header %q is not found", tc.Target)
				return
			}
			actual = h[0]
		}
		pickAssertFunc(&tc, v)(t, tc.Expected, actual, tc.Description)
	}
}

// RequestBody validates request body's fileds are expected or not. The request body
// is unmarshaled to the given struct. To extract a filed to validate, this uses dot-seprated
// expression in TestCase.Target. For example, if you want to access `Email` value in the
// following struct use `Setting.Name` in Target.
//
//   type User struct {
//       Setting Setting
//   }
//
//   type Setting struct {
//       Email string
//   }
//
func (v *Validator) RequestBody(t *testing.T, cases []TestCase, request interface{}) {
	// Unmarshal request body into the given struct
	if err := v.unmarshalFunc(v.record.requestBody, request); err != nil {
		tFatalf(t, "Failed to unmarshal request body: %s", err)
		return
	}
	v.validateFields(t, cases, request, &v.requestFields)
}

// ResponseBody validates response body's fields are expected or not. The response body
// is unmarshaled to the given struct. To extract a filed to validate, this uses dot-seprated
// expression in TestCase.Target. For example, if you want to access `Email` value in the
// following struct use `Setting.Name` in Target.
//
//   type User struct {
//       Setting Setting
//   }
//
//   type Setting struct {
//       Email string
//   }
//
func (v *Validator) ResponseBody(t *testing.T, cases []TestCase, response interface{}) {
	// Unmarshal request body into the given struct
	if err := v.unmarshalFunc(v.record.responseBody, response); err != nil {
		tFatalf(t, "Failed to unmarshal response body: %s", err)
	}
	v.validateFields(t, cases, response, &v.responseFields)
}

func (vl *Validator) validateFields(t *testing.T, cases []TestCase, v interface{}, fields *[]Data) {
	for _, tc := range cases {
		data := Data{
			Name:        tc.Target,
			Value:       tc.Expected,
			Description: tc.Description,
		}
		*fields = append(*fields, data)
		actual, _ := gpath.At(v, tc.Target)
		pickAssertFunc(&tc, vl)(t, tc.Expected, actual, tc.Description)
	}
}

func pickAssertFunc(tc *TestCase, v *Validator) assertFunc {
	if tc.AssertFunc != nil {
		return tc.AssertFunc
	}
	return v.assertFunc
}
<|endoftext|>


<commit_before>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package go_kafka_client

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"math"
	"reflect"
	"sort"
	"strconv"
	"strings"
)

const (
	/* Range partitioning works on a per-topic basis. For each topic, we lay out the available partitions in numeric order
	and the consumer threads in lexicographic order. We then divide the number of partitions by the total number of
	consumer streams (threads) to determine the number of partitions to assign to each consumer. If it does not evenly
	divide, then the first few consumers will have one extra partition. For example, suppose there are two consumers C1
	and C2 with two streams each, and there are five available partitions (p0, p1, p2, p3, p4). So each consumer thread
	will get at least one partition and the first consumer thread will get one extra partition. So the assignment will be:
	p0 -> C1-0, p1 -> C1-0, p2 -> C1-1, p3 -> C2-0, p4 -> C2-1 */
	RangeStrategy = "range"

	/* The round-robin partition assignor lays out all the available partitions and all the available consumer threads. It
	then proceeds to do a round-robin assignment from partition to consumer thread. If the subscriptions of all consumer
	instances are identical, then the partitions will be uniformly distributed. (i.e., the partition ownership counts
	will be within a delta of exactly one across all consumer threads.)

	(For simplicity of implementation) the assignor is allowed to assign a given topic-partition to any consumer instance
	and thread-id within that instance. Therefore, round-robin assignment is allowed only if:
	a) Every topic has the same number of streams within a consumer instance
	b) The set of subscribed topics is identical for every consumer instance within the group. */
	RoundRobinStrategy = "roundrobin"
)

type assignStrategy func(*assignmentContext) map[TopicAndPartition]ConsumerThreadId

func newPartitionAssignor(strategy string) assignStrategy {
	switch strategy {
	case RoundRobinStrategy:
		return roundRobinAssignor
	case RangeStrategy:
		return rangeAssignor
	default:
		panic(fmt.Sprintf("Invalid partition assignment strategy: %s", strategy))
	}
}

func roundRobinAssignor(context *assignmentContext) map[TopicAndPartition]ConsumerThreadId {
	ownershipDecision := make(map[TopicAndPartition]ConsumerThreadId)

	if len(context.ConsumersForTopic) > 0 {
		var headThreadIds []ConsumerThreadId
		for _, headThreadIds = range context.ConsumersForTopic {
			break
		}
		for _, threadIds := range context.ConsumersForTopic {
			if !reflect.DeepEqual(threadIds, headThreadIds) {
				panic("Round-robin assignor works only if all consumers in group subscribed to the same topics AND if the stream counts across topics are identical for a given consumer instance.")
			}
		}

		topicsAndPartitions := make([]*TopicAndPartition, 0)
		for topic, partitions := range context.PartitionsForTopic {
			for _, partition := range partitions {
				topicsAndPartitions = append(topicsAndPartitions, &TopicAndPartition{
					Topic:     topic,
					Partition: partition,
				})
			}
		}

		fmt.Printf("%v\n", topicsAndPartitions)

		sort.Sort(hashArray(topicsAndPartitions))
		threadIdsIterator := circularIterator(&headThreadIds)

		for _, topicPartition := range topicsAndPartitions {
			consumerThreadId := threadIdsIterator.Value.(ConsumerThreadId)
			if consumerThreadId.Consumer == context.ConsumerId {
				ownershipDecision[*topicPartition] = consumerThreadId
			}
			threadIdsIterator = threadIdsIterator.Next()
		}
	}

	return ownershipDecision
}

func rangeAssignor(context *assignmentContext) map[TopicAndPartition]ConsumerThreadId {
	ownershipDecision := make(map[TopicAndPartition]ConsumerThreadId)

	for topic, consumerThreadIds := range context.MyTopicThreadIds {
		consumersForTopic := context.ConsumersForTopic[topic]
		partitionsForTopic := context.PartitionsForTopic[topic]

		Debug(context.ConsumerId, partitionsForTopic)

		Tracef(context.ConsumerId, "partitionsForTopic: %d, consumersForTopic: %d", len(partitionsForTopic), len(consumersForTopic))

		nPartsPerConsumer := len(partitionsForTopic) / len(consumersForTopic)
		nConsumersWithExtraPart := len(partitionsForTopic) % len(consumersForTopic)

		Tracef(context.ConsumerId, "nPartsPerConsumer: %d, nConsumersWithExtraPart: %d", nPartsPerConsumer, nConsumersWithExtraPart)

		for _, consumerThreadId := range consumerThreadIds {
			myConsumerPosition := position(&consumersForTopic, consumerThreadId)
			Tracef(context.ConsumerId, "myConsumerPosition: %d", myConsumerPosition)
			if myConsumerPosition < 0 {
				panic(fmt.Sprintf("There is no %s in consumers for topic %s", consumerThreadId, topic))
			}
			startPart := nPartsPerConsumer*myConsumerPosition + int(math.Min(float64(myConsumerPosition), float64(nConsumersWithExtraPart)))
			nParts := nPartsPerConsumer
			if myConsumerPosition+1 <= nConsumersWithExtraPart {
				nParts = nPartsPerConsumer + 1
			}
			Tracef(context.ConsumerId, "startPart: %d, nParts: %d", startPart, nParts)

			if nParts <= 0 {
				Warnf(context.ConsumerId, "No broker partitions consumed by consumer thread %s for topic %s", consumerThreadId, topic)
			} else {
				for i := startPart; i < startPart+nParts; i++ {
					partition := partitionsForTopic[i]
					Infof(context.ConsumerId, "%s attempting to claim %s", consumerThreadId, &TopicAndPartition{Topic: topic, Partition: partition})
					ownershipDecision[TopicAndPartition{Topic: topic, Partition: partition}] = consumerThreadId
				}
			}
		}
	}

	return ownershipDecision
}

type assignmentContext struct {
	ConsumerId          string
	Group               string
	MyTopicThreadIds    map[string][]ConsumerThreadId
	MyTopicToNumStreams TopicsToNumStreams
	PartitionsForTopic  map[string][]int32
	ConsumersForTopic   map[string][]ConsumerThreadId
	Consumers           []string
	Brokers             []*BrokerInfo
	AllTopics           []string
}

func (context *assignmentContext) hash() string {
	hash := md5.New()
	for _, broker := range context.Brokers {
		io.WriteString(hash, strconv.Itoa(int(broker.Id)))
	}

	io.WriteString(hash, strings.Join(context.Consumers, ""))
	sort.Strings(context.AllTopics)
	for _, topic := range context.AllTopics {
		io.WriteString(hash, topic)
		if _, exists := context.PartitionsForTopic[topic]; !exists {
			continue
		}
		for _, partition := range context.PartitionsForTopic[topic] {
			io.WriteString(hash, strconv.Itoa(int(partition)))
		}
	}

	return hex.EncodeToString(hash.Sum(nil))
}

func newAssignmentContext(group string, consumerId string, excludeInternalTopics bool, coordinator ConsumerCoordinator) (*assignmentContext, error) {
	brokers, err := coordinator.GetAllBrokers()
	if err != nil {
		panic(fmt.Sprintf("Failed to create topicCount: %s", err))
	}
	allTopics, err := coordinator.GetAllTopics()
	if err != nil {
		panic(fmt.Sprintf("Failed to create topicCount: %s", err))
	}

	topicCount, err := NewTopicsToNumStreams(group, consumerId, coordinator, excludeInternalTopics)
	if err != nil {
		panic(fmt.Sprintf("Failed to create topicCount: %s", err))
	}
	myTopicThreadIds := topicCount.GetConsumerThreadIdsPerTopic()
	myTopics := make([]string, 0)
	for topic, _ := range myTopicThreadIds {
		myTopics = append(myTopics, topic)
	}
	partitionsForTopic, err := coordinator.GetPartitionsForTopics(myTopics)
	if err != nil {
		panic(fmt.Sprintf("Failed to create topicCount: %s", err))
	}
	consumersForTopic, err := coordinator.GetConsumersPerTopic(group, excludeInternalTopics)
	if err != nil {
		panic(fmt.Sprintf("Failed to create topicCount: %s", err))
	}
	consumers, err := coordinator.GetConsumersInGroup(group)
	if err != nil {
		panic(fmt.Sprintf("Failed to create topicCount: %s", err))
	}

	return &assignmentContext{
		ConsumerId:          consumerId,
		Group:               group,
		MyTopicThreadIds:    myTopicThreadIds,
		MyTopicToNumStreams: topicCount,
		PartitionsForTopic:  partitionsForTopic,
		ConsumersForTopic:   consumersForTopic,
		Consumers:           consumers,
		Brokers:             brokers,
		AllTopics:           allTopics,
	}, nil
}

func newStaticAssignmentContext(group string, consumerId string, consumersInGroup []string, allTopics []string, brokers []*BrokerInfo,
	topicCount TopicsToNumStreams, topicPartitionMap map[string][]int32) *assignmentContext {
	myTopicThreadIds := topicCount.GetConsumerThreadIdsPerTopic()
	consumersForTopic := make(map[string][]ConsumerThreadId)
	for topic := range topicPartitionMap {
		for _, consumer := range consumersInGroup {
			threadIdsPerTopic := makeConsumerThreadIdsPerTopic(consumer, topicCount.GetTopicsToNumStreamsMap())
			var threadIds []ConsumerThreadId
			for _, threadIds = range threadIdsPerTopic {
				break
			}
			consumersForTopic[topic] = append(consumersForTopic[topic], threadIds...)
		}
	}

	return &assignmentContext{
		ConsumerId:          consumerId,
		Group:               group,
		MyTopicThreadIds:    myTopicThreadIds,
		MyTopicToNumStreams: topicCount,
		PartitionsForTopic:  topicPartitionMap,
		ConsumersForTopic:   consumersForTopic,
		Consumers:           consumersInGroup,
		Brokers:             brokers,
		AllTopics:           allTopics,
	}
}
<commit_msg>sort brokers and consumers also before hashing<commit_after>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package go_kafka_client

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"math"
	"reflect"
	"sort"
	"strconv"
	"strings"
)

const (
	/* Range partitioning works on a per-topic basis. For each topic, we lay out the available partitions in numeric order
	and the consumer threads in lexicographic order. We then divide the number of partitions by the total number of
	consumer streams (threads) to determine the number of partitions to assign to each consumer. If it does not evenly
	divide, then the first few consumers will have one extra partition. For example, suppose there are two consumers C1
	and C2 with two streams each, and there are five available partitions (p0, p1, p2, p3, p4). So each consumer thread
	will get at least one partition and the first consumer thread will get one extra partition. So the assignment will be:
	p0 -> C1-0, p1 -> C1-0, p2 -> C1-1, p3 -> C2-0, p4 -> C2-1 */
	RangeStrategy = "range"

	/* The round-robin partition assignor lays out all the available partitions and all the available consumer threads. It
	then proceeds to do a round-robin assignment from partition to consumer thread. If the subscriptions of all consumer
	instances are identical, then the partitions will be uniformly distributed. (i.e., the partition ownership counts
	will be within a delta of exactly one across all consumer threads.)

	(For simplicity of implementation) the assignor is allowed to assign a given topic-partition to any consumer instance
	and thread-id within that instance. Therefore, round-robin assignment is allowed only if:
	a) Every topic has the same number of streams within a consumer instance
	b) The set of subscribed topics is identical for every consumer instance within the group. */
	RoundRobinStrategy = "roundrobin"
)

type assignStrategy func(*assignmentContext) map[TopicAndPartition]ConsumerThreadId

func newPartitionAssignor(strategy string) assignStrategy {
	switch strategy {
	case RoundRobinStrategy:
		return roundRobinAssignor
	case RangeStrategy:
		return rangeAssignor
	default:
		panic(fmt.Sprintf("Invalid partition assignment strategy: %s", strategy))
	}
}

func roundRobinAssignor(context *assignmentContext) map[TopicAndPartition]ConsumerThreadId {
	ownershipDecision := make(map[TopicAndPartition]ConsumerThreadId)

	if len(context.ConsumersForTopic) > 0 {
		var headThreadIds []ConsumerThreadId
		for _, headThreadIds = range context.ConsumersForTopic {
			break
		}
		for _, threadIds := range context.ConsumersForTopic {
			if !reflect.DeepEqual(threadIds, headThreadIds) {
				panic("Round-robin assignor works only if all consumers in group subscribed to the same topics AND if the stream counts across topics are identical for a given consumer instance.")
			}
		}

		topicsAndPartitions := make([]*TopicAndPartition, 0)
		for topic, partitions := range context.PartitionsForTopic {
			for _, partition := range partitions {
				topicsAndPartitions = append(topicsAndPartitions, &TopicAndPartition{
					Topic:     topic,
					Partition: partition,
				})
			}
		}

		fmt.Printf("%v\n", topicsAndPartitions)

		sort.Sort(hashArray(topicsAndPartitions))
		threadIdsIterator := circularIterator(&headThreadIds)

		for _, topicPartition := range topicsAndPartitions {
			consumerThreadId := threadIdsIterator.Value.(ConsumerThreadId)
			if consumerThreadId.Consumer == context.ConsumerId {
				ownershipDecision[*topicPartition] = consumerThreadId
			}
			threadIdsIterator = threadIdsIterator.Next()
		}
	}

	return ownershipDecision
}

func rangeAssignor(context *assignmentContext) map[TopicAndPartition]ConsumerThreadId {
	ownershipDecision := make(map[TopicAndPartition]ConsumerThreadId)

	for topic, consumerThreadIds := range context.MyTopicThreadIds {
		consumersForTopic := context.ConsumersForTopic[topic]
		partitionsForTopic := context.PartitionsForTopic[topic]

		Debug(context.ConsumerId, partitionsForTopic)

		Tracef(context.ConsumerId, "partitionsForTopic: %d, consumersForTopic: %d", len(partitionsForTopic), len(consumersForTopic))

		nPartsPerConsumer := len(partitionsForTopic) / len(consumersForTopic)
		nConsumersWithExtraPart := len(partitionsForTopic) % len(consumersForTopic)

		Tracef(context.ConsumerId, "nPartsPerConsumer: %d, nConsumersWithExtraPart: %d", nPartsPerConsumer, nConsumersWithExtraPart)

		for _, consumerThreadId := range consumerThreadIds {
			myConsumerPosition := position(&consumersForTopic, consumerThreadId)
			Tracef(context.ConsumerId, "myConsumerPosition: %d", myConsumerPosition)
			if myConsumerPosition < 0 {
				panic(fmt.Sprintf("There is no %s in consumers for topic %s", consumerThreadId, topic))
			}
			startPart := nPartsPerConsumer*myConsumerPosition + int(math.Min(float64(myConsumerPosition), float64(nConsumersWithExtraPart)))
			nParts := nPartsPerConsumer
			if myConsumerPosition+1 <= nConsumersWithExtraPart {
				nParts = nPartsPerConsumer + 1
			}
			Tracef(context.ConsumerId, "startPart: %d, nParts: %d", startPart, nParts)

			if nParts <= 0 {
				Warnf(context.ConsumerId, "No broker partitions consumed by consumer thread %s for topic %s", consumerThreadId, topic)
			} else {
				for i := startPart; i < startPart+nParts; i++ {
					partition := partitionsForTopic[i]
					Infof(context.ConsumerId, "%s attempting to claim %s", consumerThreadId, &TopicAndPartition{Topic: topic, Partition: partition})
					ownershipDecision[TopicAndPartition{Topic: topic, Partition: partition}] = consumerThreadId
				}
			}
		}
	}

	return ownershipDecision
}

type assignmentContext struct {
	ConsumerId          string
	Group               string
	MyTopicThreadIds    map[string][]ConsumerThreadId
	MyTopicToNumStreams TopicsToNumStreams
	PartitionsForTopic  map[string][]int32
	ConsumersForTopic   map[string][]ConsumerThreadId
	Consumers           []string
	Brokers             []*BrokerInfo
	AllTopics           []string
}

func (context *assignmentContext) hash() string {
	hash := md5.New()
	sort.Sort(byId(context.Brokers))
	for _, broker := range context.Brokers {
		io.WriteString(hash, strconv.Itoa(int(broker.Id)))
	}

	sort.Strings(context.Consumers)
	io.WriteString(hash, strings.Join(context.Consumers, ""))
	sort.Strings(context.AllTopics)
	for _, topic := range context.AllTopics {
		io.WriteString(hash, topic)
		if _, exists := context.PartitionsForTopic[topic]; !exists {
			continue
		}
		for _, partition := range context.PartitionsForTopic[topic] {
			io.WriteString(hash, strconv.Itoa(int(partition)))
		}
	}

	return hex.EncodeToString(hash.Sum(nil))
}

func newAssignmentContext(group string, consumerId string, excludeInternalTopics bool, coordinator ConsumerCoordinator) (*assignmentContext, error) {
	brokers, err := coordinator.GetAllBrokers()
	if err != nil {
		panic(fmt.Sprintf("Failed to create topicCount: %s", err))
	}
	allTopics, err := coordinator.GetAllTopics()
	if err != nil {
		panic(fmt.Sprintf("Failed to create topicCount: %s", err))
	}

	topicCount, err := NewTopicsToNumStreams(group, consumerId, coordinator, excludeInternalTopics)
	if err != nil {
		panic(fmt.Sprintf("Failed to create topicCount: %s", err))
	}
	myTopicThreadIds := topicCount.GetConsumerThreadIdsPerTopic()
	myTopics := make([]string, 0)
	for topic, _ := range myTopicThreadIds {
		myTopics = append(myTopics, topic)
	}
	partitionsForTopic, err := coordinator.GetPartitionsForTopics(myTopics)
	if err != nil {
		panic(fmt.Sprintf("Failed to create topicCount: %s", err))
	}
	consumersForTopic, err := coordinator.GetConsumersPerTopic(group, excludeInternalTopics)
	if err != nil {
		panic(fmt.Sprintf("Failed to create topicCount: %s", err))
	}
	consumers, err := coordinator.GetConsumersInGroup(group)
	if err != nil {
		panic(fmt.Sprintf("Failed to create topicCount: %s", err))
	}

	return &assignmentContext{
		ConsumerId:          consumerId,
		Group:               group,
		MyTopicThreadIds:    myTopicThreadIds,
		MyTopicToNumStreams: topicCount,
		PartitionsForTopic:  partitionsForTopic,
		ConsumersForTopic:   consumersForTopic,
		Consumers:           consumers,
		Brokers:             brokers,
		AllTopics:           allTopics,
	}, nil
}

func newStaticAssignmentContext(group string, consumerId string, consumersInGroup []string, allTopics []string, brokers []*BrokerInfo,
	topicCount TopicsToNumStreams, topicPartitionMap map[string][]int32) *assignmentContext {
	myTopicThreadIds := topicCount.GetConsumerThreadIdsPerTopic()
	consumersForTopic := make(map[string][]ConsumerThreadId)
	for topic := range topicPartitionMap {
		for _, consumer := range consumersInGroup {
			threadIdsPerTopic := makeConsumerThreadIdsPerTopic(consumer, topicCount.GetTopicsToNumStreamsMap())
			var threadIds []ConsumerThreadId
			for _, threadIds = range threadIdsPerTopic {
				break
			}
			consumersForTopic[topic] = append(consumersForTopic[topic], threadIds...)
		}
	}

	return &assignmentContext{
		ConsumerId:          consumerId,
		Group:               group,
		MyTopicThreadIds:    myTopicThreadIds,
		MyTopicToNumStreams: topicCount,
		PartitionsForTopic:  topicPartitionMap,
		ConsumersForTopic:   consumersForTopic,
		Consumers:           consumersInGroup,
		Brokers:             brokers,
		AllTopics:           allTopics,
	}
}
<|endoftext|>