<commit_before>package widget

import (
	"fmt"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/qor/admin"
	"github.com/qor/media/oss"
	"github.com/qor/qor"
	"github.com/qor/qor/resource"
	"github.com/qor/qor/utils"
	"github.com/qor/serializable_meta"
)

// QorWidgetSettingInterface qor widget setting interface
type QorWidgetSettingInterface interface {
	GetPreviewIcon() string
	GetWidgetName() string
	SetWidgetName(string)
	GetGroupName() string
	SetGroupName(string)
	GetScope() string
	SetScope(string)
	GetTemplate() string
	SetTemplate(string)
	serializable_meta.SerializableMetaInterface
}

// QorWidgetSetting default qor widget setting struct
type QorWidgetSetting struct {
	Name        string `gorm:"primary_key"`
	Scope       string `gorm:"primary_key;size:128;default:'default'"`
	Description string
	Shared      bool
	WidgetType  string
	GroupName   string
	Template    string
	PreviewIcon oss.OSS
	serializable_meta.SerializableMeta
	CreatedAt time.Time
	UpdatedAt time.Time
}

// ResourceName get widget setting's resource name
func (widgetSetting *QorWidgetSetting) ResourceName() string {
	return "Widget Content"
}

// GetSerializableArgumentKind get serializable kind
func (widgetSetting *QorWidgetSetting) GetSerializableArgumentKind() string {
	if widgetSetting.WidgetType != "" {
		return widgetSetting.WidgetType
	}
	return widgetSetting.Kind
}

// SetSerializableArgumentKind set serializable kind
func (widgetSetting *QorWidgetSetting) SetSerializableArgumentKind(name string) {
	widgetSetting.WidgetType = name
	widgetSetting.Kind = name
}

// GetPreviewIcon get preview icon
func (widgetSetting QorWidgetSetting) GetPreviewIcon() string {
	return widgetSetting.PreviewIcon.URL()
}

// GetWidgetName get widget setting's group name
func (widgetSetting QorWidgetSetting) GetWidgetName() string {
	return widgetSetting.Name
}

// SetWidgetName set widget setting's group name
func (widgetSetting *QorWidgetSetting) SetWidgetName(name string) {
	widgetSetting.Name = name
}

// GetGroupName get widget setting's group name
func (widgetSetting QorWidgetSetting) GetGroupName() string {
	return widgetSetting.GroupName
}

// SetGroupName set widget setting's group name
func (widgetSetting *QorWidgetSetting) SetGroupName(groupName string) {
	widgetSetting.GroupName = groupName
}

// GetScope get widget's scope
func (widgetSetting QorWidgetSetting) GetScope() string {
	return widgetSetting.Scope
}

// SetScope set widget setting's scope
func (widgetSetting *QorWidgetSetting) SetScope(scope string) {
	widgetSetting.Scope = scope
}

// GetTemplate get used widget template
func (widgetSetting QorWidgetSetting) GetTemplate() string {
	if widget := GetWidget(widgetSetting.GetSerializableArgumentKind()); widget != nil {
		for _, value := range widget.Templates {
			if value == widgetSetting.Template {
				return value
			}
		}

		// return first value of defined widget templates
		for _, value := range widget.Templates {
			return value
		}
	}
	return ""
}

// SetTemplate set used widget's template
func (widgetSetting *QorWidgetSetting) SetTemplate(template string) {
	widgetSetting.Template = template
}

// GetSerializableArgumentResource get setting's argument's resource
func (widgetSetting *QorWidgetSetting) GetSerializableArgumentResource() *admin.Resource {
	widget := GetWidget(widgetSetting.GetSerializableArgumentKind())
	if widget != nil {
		return widget.Setting
	}
	return nil
}

// ConfigureQorResource a method used to config Widget for qor admin
func (widgetSetting *QorWidgetSetting) ConfigureQorResource(res resource.Resourcer) {
	if res, ok := res.(*admin.Resource); ok {
		if res.GetMeta("Name") == nil {
			res.Meta(&admin.Meta{Name: "Name"})
		}

		if res.GetMeta("DisplayName") == nil {
			res.Meta(&admin.Meta{Name: "DisplayName", Label: "Name", Type: "readonly", FieldName: "Name"})
		}

		res.Meta(&admin.Meta{
			Name: "Scope",
			Type: "hidden",
			Valuer: func(result interface{}, context *qor.Context) interface{} {
				if scope := context.Request.URL.Query().Get("widget_scope"); scope != "" {
					return scope
				}

				if setting, ok := result.(QorWidgetSettingInterface); ok {
					if scope := setting.GetScope(); scope != "" {
						return scope
					}
				}

				return "default"
			},
			Setter: func(result interface{}, metaValue *resource.MetaValue, context *qor.Context) {
				if setting, ok := result.(QorWidgetSettingInterface); ok {
					setting.SetScope(utils.ToString(metaValue.Value))
				}
			},
		})

		res.Meta(&admin.Meta{
			Name: "Widgets",
			Type: "select_one",
			Valuer: func(result interface{}, context *qor.Context) interface{} {
				if typ := context.Request.URL.Query().Get("widget_type"); typ != "" {
					return typ
				}

				if setting, ok := result.(QorWidgetSettingInterface); ok {
					widget := GetWidget(setting.GetSerializableArgumentKind())
					if widget == nil {
						return ""
					}
					return widget.Name
				}

				return ""
			},
			Collection: func(result interface{}, context *qor.Context) (results [][]string) {
				if setting, ok := result.(QorWidgetSettingInterface); ok {
					if setting.GetWidgetName() == "" {
						for _, widget := range registeredWidgets {
							results = append(results, []string{widget.Name, widget.Name})
						}
					} else {
						groupName := setting.GetGroupName()
						for _, group := range registeredWidgetsGroup {
							if group.Name == groupName {
								for _, widget := range group.Widgets {
									results = append(results, []string{widget, widget})
								}
							}
						}
					}

					if len(results) == 0 {
						results = append(results, []string{setting.GetSerializableArgumentKind(), setting.GetSerializableArgumentKind()})
					}
				}
				return
			},
			Setter: func(result interface{}, metaValue *resource.MetaValue, context *qor.Context) {
				if setting, ok := result.(QorWidgetSettingInterface); ok {
					setting.SetSerializableArgumentKind(utils.ToString(metaValue.Value))
				}
			},
		})

		res.Meta(&admin.Meta{
			Name: "Template",
			Type: "select_one",
			Valuer: func(result interface{}, context *qor.Context) interface{} {
				if setting, ok := result.(QorWidgetSettingInterface); ok {
					return setting.GetTemplate()
				}
				return ""
			},
			Collection: func(result interface{}, context *qor.Context) (results [][]string) {
				if setting, ok := result.(QorWidgetSettingInterface); ok {
					if widget := GetWidget(setting.GetSerializableArgumentKind()); widget != nil {
						for _, value := range widget.Templates {
							results = append(results, []string{value, value})
						}
					}
				}
				return
			},
			Setter: func(result interface{}, metaValue *resource.MetaValue, context *qor.Context) {
				if setting, ok := result.(QorWidgetSettingInterface); ok {
					setting.SetTemplate(utils.ToString(metaValue.Value))
				}
			},
		})

		res.Meta(&admin.Meta{
			Name:  "Shared",
			Label: "This widget is shared",
		})

		res.Scope(&admin.Scope{
			Name:  "Shared",
			Label: "Shared Widgets",
			Handle: func(db *gorm.DB, _ *qor.Context) *gorm.DB {
				return db.Where("shared = ?", true)
			},
		})

		res.Action(&admin.Action{
			Name: "Preview",
			URL: func(record interface{}, context *admin.Context) string {
				return fmt.Sprintf("%v/%v/%v/!preview", context.Admin.GetRouter().Prefix, res.ToParam(), record.(QorWidgetSettingInterface).GetWidgetName())
			},
			Modes: []string{"edit", "menu_item"},
		})

		res.UseTheme("widget")

		res.IndexAttrs("Name", "Description", "CreatedAt", "UpdatedAt")
		res.ShowAttrs("Name", "Scope", "WidgetType", "Template", "Description", "Value", "CreatedAt", "UpdatedAt")
		res.EditAttrs(
			"DisplayName", "Description", "Scope", "Widgets", "Template",
			&admin.Section{
				Title: "Settings",
				Rows:  [][]string{{"Kind"}, {"SerializableMeta"}},
			},
			"Shared",
		)
		res.NewAttrs("Name", "Description", "Scope", "Widgets", "Template")
	}
}
<commit_msg>Add Shared to new attrs<commit_after>package widget

import (
	"fmt"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/qor/admin"
	"github.com/qor/media/oss"
	"github.com/qor/qor"
	"github.com/qor/qor/resource"
	"github.com/qor/qor/utils"
	"github.com/qor/serializable_meta"
)

// QorWidgetSettingInterface qor widget setting interface
type QorWidgetSettingInterface interface {
	GetPreviewIcon() string
	GetWidgetName() string
	SetWidgetName(string)
	GetGroupName() string
	SetGroupName(string)
	GetScope() string
	SetScope(string)
	GetTemplate() string
	SetTemplate(string)
	serializable_meta.SerializableMetaInterface
}

// QorWidgetSetting default qor widget setting struct
type QorWidgetSetting struct {
	Name        string `gorm:"primary_key"`
	Scope       string `gorm:"primary_key;size:128;default:'default'"`
	Description string
	Shared      bool
	WidgetType  string
	GroupName   string
	Template    string
	PreviewIcon oss.OSS
	serializable_meta.SerializableMeta
	CreatedAt time.Time
	UpdatedAt time.Time
}

// ResourceName get widget setting's resource name
func (widgetSetting *QorWidgetSetting) ResourceName() string {
	return "Widget Content"
}

// GetSerializableArgumentKind get serializable kind
func (widgetSetting *QorWidgetSetting) GetSerializableArgumentKind() string {
	if widgetSetting.WidgetType != "" {
		return widgetSetting.WidgetType
	}
	return widgetSetting.Kind
}

// SetSerializableArgumentKind set serializable kind
func (widgetSetting *QorWidgetSetting) SetSerializableArgumentKind(name string) {
	widgetSetting.WidgetType = name
	widgetSetting.Kind = name
}

// GetPreviewIcon get preview icon
func (widgetSetting QorWidgetSetting) GetPreviewIcon() string {
	return widgetSetting.PreviewIcon.URL()
}

// GetWidgetName get widget setting's group name
func (widgetSetting QorWidgetSetting) GetWidgetName() string {
	return widgetSetting.Name
}

// SetWidgetName set widget setting's group name
func (widgetSetting *QorWidgetSetting) SetWidgetName(name string) {
	widgetSetting.Name = name
}

// GetGroupName get widget setting's group name
func (widgetSetting QorWidgetSetting) GetGroupName() string {
	return widgetSetting.GroupName
}

// SetGroupName set widget setting's group name
func (widgetSetting *QorWidgetSetting) SetGroupName(groupName string) {
	widgetSetting.GroupName = groupName
}

// GetScope get widget's scope
func (widgetSetting QorWidgetSetting) GetScope() string {
	return widgetSetting.Scope
}

// SetScope set widget setting's scope
func (widgetSetting *QorWidgetSetting) SetScope(scope string) {
	widgetSetting.Scope = scope
}

// GetTemplate get used widget template
func (widgetSetting QorWidgetSetting) GetTemplate() string {
	if widget := GetWidget(widgetSetting.GetSerializableArgumentKind()); widget != nil {
		for _, value := range widget.Templates {
			if value == widgetSetting.Template {
				return value
			}
		}

		// return first value of defined widget templates
		for _, value := range widget.Templates {
			return value
		}
	}
	return ""
}

// SetTemplate set used widget's template
func (widgetSetting *QorWidgetSetting) SetTemplate(template string) {
	widgetSetting.Template = template
}

// GetSerializableArgumentResource get setting's argument's resource
func (widgetSetting *QorWidgetSetting) GetSerializableArgumentResource() *admin.Resource {
	widget := GetWidget(widgetSetting.GetSerializableArgumentKind())
	if widget != nil {
		return widget.Setting
	}
	return nil
}

// ConfigureQorResource a method used to config Widget for qor admin
func (widgetSetting *QorWidgetSetting) ConfigureQorResource(res resource.Resourcer) {
	if res, ok := res.(*admin.Resource); ok {
		if res.GetMeta("Name") == nil {
			res.Meta(&admin.Meta{Name: "Name"})
		}

		if res.GetMeta("DisplayName") == nil {
			res.Meta(&admin.Meta{Name: "DisplayName", Label: "Name", Type: "readonly", FieldName: "Name"})
		}

		if res.GetMeta("Scope") == nil {
			res.Meta(&admin.Meta{
				Name: "Scope",
				Type: "hidden",
				Valuer: func(result interface{}, context *qor.Context) interface{} {
					if scope := context.Request.URL.Query().Get("widget_scope"); scope != "" {
						return scope
					}

					if setting, ok := result.(QorWidgetSettingInterface); ok {
						if scope := setting.GetScope(); scope != "" {
							return scope
						}
					}

					return "default"
				},
				Setter: func(result interface{}, metaValue *resource.MetaValue, context *qor.Context) {
					if setting, ok := result.(QorWidgetSettingInterface); ok {
						setting.SetScope(utils.ToString(metaValue.Value))
					}
				},
			})
		}

		if res.GetMeta("Widgets") == nil {
			res.Meta(&admin.Meta{
				Name: "Widgets",
				Type: "select_one",
				Valuer: func(result interface{}, context *qor.Context) interface{} {
					if typ := context.Request.URL.Query().Get("widget_type"); typ != "" {
						return typ
					}

					if setting, ok := result.(QorWidgetSettingInterface); ok {
						widget := GetWidget(setting.GetSerializableArgumentKind())
						if widget == nil {
							return ""
						}
						return widget.Name
					}

					return ""
				},
				Collection: func(result interface{}, context *qor.Context) (results [][]string) {
					if setting, ok := result.(QorWidgetSettingInterface); ok {
						if setting.GetWidgetName() == "" {
							for _, widget := range registeredWidgets {
								results = append(results, []string{widget.Name, widget.Name})
							}
						} else {
							groupName := setting.GetGroupName()
							for _, group := range registeredWidgetsGroup {
								if group.Name == groupName {
									for _, widget := range group.Widgets {
										results = append(results, []string{widget, widget})
									}
								}
							}
						}

						if len(results) == 0 {
							results = append(results, []string{setting.GetSerializableArgumentKind(), setting.GetSerializableArgumentKind()})
						}
					}
					return
				},
				Setter: func(result interface{}, metaValue *resource.MetaValue, context *qor.Context) {
					if setting, ok := result.(QorWidgetSettingInterface); ok {
						setting.SetSerializableArgumentKind(utils.ToString(metaValue.Value))
					}
				},
			})
		}

		if res.GetMeta("Template") == nil {
			res.Meta(&admin.Meta{
				Name: "Template",
				Type: "select_one",
				Valuer: func(result interface{}, context *qor.Context) interface{} {
					if setting, ok := result.(QorWidgetSettingInterface); ok {
						return setting.GetTemplate()
					}
					return ""
				},
				Collection: func(result interface{}, context *qor.Context) (results [][]string) {
					if setting, ok := result.(QorWidgetSettingInterface); ok {
						if widget := GetWidget(setting.GetSerializableArgumentKind()); widget != nil {
							for _, value := range widget.Templates {
								results = append(results, []string{value, value})
							}
						}
					}
					return
				},
				Setter: func(result interface{}, metaValue *resource.MetaValue, context *qor.Context) {
					if setting, ok := result.(QorWidgetSettingInterface); ok {
						setting.SetTemplate(utils.ToString(metaValue.Value))
					}
				},
			})
		}

		if res.GetMeta("Shared") == nil {
			res.Meta(&admin.Meta{
				Name:  "Shared",
				Label: "This widget is shared",
			})
		}

		res.Scope(&admin.Scope{
			Name:  "Shared",
			Label: "Shared Widgets",
			Handle: func(db *gorm.DB, _ *qor.Context) *gorm.DB {
				return db.Where("shared = ?", true)
			},
		})

		res.Action(&admin.Action{
			Name: "Preview",
			URL: func(record interface{}, context *admin.Context) string {
				return fmt.Sprintf("%v/%v/%v/!preview", context.Admin.GetRouter().Prefix, res.ToParam(), record.(QorWidgetSettingInterface).GetWidgetName())
			},
			Modes: []string{"edit", "menu_item"},
		})

		res.UseTheme("widget")

		res.IndexAttrs("Name", "Description", "CreatedAt", "UpdatedAt")
		res.ShowAttrs("Name", "Scope", "WidgetType", "Template", "Description", "Value", "CreatedAt", "UpdatedAt")
		res.EditAttrs(
			"DisplayName", "Description", "Scope", "Widgets", "Template",
			&admin.Section{
				Title: "Settings",
				Rows:  [][]string{{"Kind"}, {"SerializableMeta"}},
			},
			"Shared",
		)
		res.NewAttrs("Name", "Description", "Scope", "Widgets", "Template",
			&admin.Section{
				Title: "Settings",
				Rows:  [][]string{{"Kind"}, {"SerializableMeta"}},
			},
			"Shared",
		)
	}
}
<|endoftext|>


<commit_before>package model

import (
	"encoding/json"
	e "github.com/lastbackend/lastbackend/libs/errors"
	"github.com/lastbackend/lastbackend/pkg/service"
	"github.com/lastbackend/lastbackend/pkg/util/table"
	"time"
	"fmt"
)

type ServiceList []Service

type Service struct {
	// Service uuid, incremented automatically
	ID string `json:"id" gorethink:"id,omitempty"`
	// Service user
	User string `json:"user" gorethink:"user,omitempty"`
	// Service project
	Project string `json:"project" gorethink:"project,omitempty"`
	// Service image
	Image string `json:"image" gorethink:"image,omitempty"`
	// Service name
	Name string `json:"name" gorethink:"name,omitempty"`
	// Service spec
	Spec *service.Service `json:"spec,omitempty" gorethink:"-"`
	// Service created time
	Created time.Time `json:"created" gorethink:"created,omitempty"`
	// Service updated time
	Updated time.Time `json:"updated" gorethink:"updated,omitempty"`
}

func (s *Service) ToJson() ([]byte, *e.Err) {
	buf, err := json.Marshal(s)
	if err != nil {
		return nil, e.New("service").Unknown(err)
	}

	return buf, nil
}

func (s *Service) DrawTable(projectName string) {
	table.PrintHorizontal(map[string]interface{}{
		"ID":      s.ID,
		"NAME":    s.Name,
		"PROJECT": projectName,
		"PODS":    s.Spec.PodList.ListMeta.Total,
	})

	t := table.New([]string{" ", "NAME", "STATUS", "RESTARTS", "CONTAINERS"})
	t.VisibleHeader = true

	for _, pod := range s.Spec.PodList.Pods {
		t.AddRow(map[string]interface{}{
			" ":          "",
			"NAME":       pod.ObjectMeta.Name,
			"STATUS":     pod.PodStatus.PodPhase,
			"RESTARTS":   pod.RestartCount,
			"CONTAINERS": pod.Containers.ListMeta.Total,
		})
	}
	t.AddRow(map[string]interface{}{})

	t.Print()
}

func (s *ServiceList) ToJson() ([]byte, *e.Err) {

	if s == nil {
		return []byte("[]"), nil
	}

	buf, err := json.Marshal(s)
	if err != nil {
		return nil, e.New("service").Unknown(err)
	}

	return buf, nil
}

func (s *ServiceList) DrawTable(projectName string) {
	fmt.Print(" Project ", projectName + "\n\n")

	for _, s := range *s {
		//tservice :=  table.New([]string{"ID", "NAME", "PODS"})
		//tservice.VisibleHeader = true
		//
		//tservice.AddRow(map[string]interface{}{
		//	"ID": s.ID,
		//	"NAME": s.Name,
		//	"PODS": s.Spec.PodList.ListMeta.Total,
		//})
		//tservice.Print()

		table.PrintHorizontal(map[string]interface{}{
			"ID":      s.ID,
			"NAME":    s.Name,
			"PODS":    s.Spec.PodList.ListMeta.Total,
		})

		for _, pod := range s.Spec.PodList.Pods {
			tpods := table.New([]string{" ", "NAME", "STATUS", "RESTARTS", "CONTAINERS"})
			tpods.VisibleHeader = true

			tpods.AddRow(map[string]interface{}{
				" ":          "",
				"NAME":       pod.ObjectMeta.Name,
				"STATUS":     pod.PodStatus.PodPhase,
				"RESTARTS":   pod.RestartCount,
				"CONTAINERS": pod.Containers.ListMeta.Total,
			})
			tpods.Print()
		}

		fmt.Print("\n\n")
	}
}
<commit_msg>Delete comments service cmd view<commit_after>package model

import (
	"encoding/json"
	e "github.com/lastbackend/lastbackend/libs/errors"
	"github.com/lastbackend/lastbackend/pkg/service"
	"github.com/lastbackend/lastbackend/pkg/util/table"
	"time"
	"fmt"
)

type ServiceList []Service

type Service struct {
	// Service uuid, incremented automatically
	ID string `json:"id" gorethink:"id,omitempty"`
	// Service user
	User string `json:"user" gorethink:"user,omitempty"`
	// Service project
	Project string `json:"project" gorethink:"project,omitempty"`
	// Service image
	Image string `json:"image" gorethink:"image,omitempty"`
	// Service name
	Name string `json:"name" gorethink:"name,omitempty"`
	// Service spec
	Spec *service.Service `json:"spec,omitempty" gorethink:"-"`
	// Service created time
	Created time.Time `json:"created" gorethink:"created,omitempty"`
	// Service updated time
	Updated time.Time `json:"updated" gorethink:"updated,omitempty"`
}

func (s *Service) ToJson() ([]byte, *e.Err) {
	buf, err := json.Marshal(s)
	if err != nil {
		return nil, e.New("service").Unknown(err)
	}

	return buf, nil
}

func (s *Service) DrawTable(projectName string) {
	table.PrintHorizontal(map[string]interface{}{
		"ID":      s.ID,
		"NAME":    s.Name,
		"PROJECT": projectName,
		"PODS":    s.Spec.PodList.ListMeta.Total,
	})

	t := table.New([]string{" ", "NAME", "STATUS", "RESTARTS", "CONTAINERS"})
	t.VisibleHeader = true

	for _, pod := range s.Spec.PodList.Pods {
		t.AddRow(map[string]interface{}{
			" ":          "",
			"NAME":       pod.ObjectMeta.Name,
			"STATUS":     pod.PodStatus.PodPhase,
			"RESTARTS":   pod.RestartCount,
			"CONTAINERS": pod.Containers.ListMeta.Total,
		})
	}
	t.AddRow(map[string]interface{}{})

	t.Print()
}

func (s *ServiceList) ToJson() ([]byte, *e.Err) {

	if s == nil {
		return []byte("[]"), nil
	}

	buf, err := json.Marshal(s)
	if err != nil {
		return nil, e.New("service").Unknown(err)
	}

	return buf, nil
}

func (s *ServiceList) DrawTable(projectName string) {
	fmt.Print(" Project ", projectName + "\n\n")

	for _, s := range *s {
		table.PrintHorizontal(map[string]interface{}{
			"ID":      s.ID,
			"NAME":    s.Name,
			"PODS":    s.Spec.PodList.ListMeta.Total,
		})

		for _, pod := range s.Spec.PodList.Pods {
			tpods := table.New([]string{" ", "NAME", "STATUS", "RESTARTS", "CONTAINERS"})
			tpods.VisibleHeader = true

			tpods.AddRow(map[string]interface{}{
				" ":          "",
				"NAME":       pod.ObjectMeta.Name,
				"STATUS":     pod.PodStatus.PodPhase,
				"RESTARTS":   pod.RestartCount,
				"CONTAINERS": pod.Containers.ListMeta.Total,
			})
			tpods.Print()
		}

		fmt.Print("\n\n")
	}
}
<|endoftext|>


<commit_before>package router

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"log"
	"sync"
	"time"
)

const GossipInterval = 30 * time.Second

type GossipData interface {
	Encode() []byte
	Merge(GossipData)
}

type Gossip interface {
	// specific message from one peer to another
	// intermediate peers relay it using unicast topology.
	GossipUnicast(dstPeerName PeerName, msg []byte) error
	// send gossip to every peer, relayed using broadcast topology.
	GossipBroadcast(update GossipData) error
}

type Gossiper interface {
	OnGossipUnicast(sender PeerName, msg []byte) error
	// merge received data into state and return a representation of
	// the received data, for further propagation
	OnGossipBroadcast(update []byte) (GossipData, error)
	// return state of everything we know; gets called periodically
	Gossip() GossipData
	// merge received data into state and return "everything new I've
	// just learnt", or nil if nothing in the received data was new
	OnGossip(update []byte) (GossipData, error)
}

// Accumulates GossipData that needs to be sent to one destination,
// and sends it when possible.
type GossipSender struct {
	send func(GossipData)
	cell chan GossipData
	// for testing
	sent    bool
	flushch chan chan bool
}

func NewGossipSender(send func(GossipData)) *GossipSender {
	return &GossipSender{send: send}
}

func (sender *GossipSender) Start() {
	sender.cell = make(chan GossipData, 1)
	sender.flushch = make(chan chan bool)
	go sender.run()
}

func (sender *GossipSender) run() {
	for {
		select {
		case pending := <-sender.cell:
			if pending == nil { // receive zero value when chan is closed
				return
			}
			sender.send(pending)
			sender.sent = true
		case ch := <-sender.flushch:
			// send anything pending, then reply back whether we sent
			// anything since previous flush
			select {
			case pending := <-sender.cell:
				sender.send(pending)
				sender.sent = true
			default:
			}
			ch <- sender.sent
			sender.sent = false
		}
	}
}

func (sender *GossipSender) Send(data GossipData) {
	// NB: this must not be invoked concurrently
	select {
	case pending := <-sender.cell:
		pending.Merge(data)
		sender.cell <- pending
	default:
		sender.cell <- data
	}
}

func (sender *GossipSender) Stop() {
	close(sender.cell)
}

type connectionSenders map[Connection]*GossipSender
type peerSenders map[PeerName]*GossipSender

type GossipChannel struct {
	sync.Mutex
	ourself      *LocalPeer
	routes       *Routes
	name         string
	gossiper     Gossiper
	senders      connectionSenders
	broadcasters peerSenders
}

type GossipChannels map[string]*GossipChannel

func (router *Router) NewGossip(channelName string, g Gossiper) Gossip {
	channel := &GossipChannel{
		ourself:      router.Ourself,
		routes:       router.Routes,
		name:         channelName,
		gossiper:     g,
		senders:      make(connectionSenders),
		broadcasters: make(peerSenders)}
	router.GossipChannels[channelName] = channel
	return channel
}

func (router *Router) SendAllGossip() {
	for _, channel := range router.GossipChannels {
		if gossip := channel.gossiper.Gossip(); gossip != nil {
			channel.Send(router.Ourself.Name, gossip)
		}
	}
}

func (router *Router) SendAllGossipDown(conn Connection) {
	for _, channel := range router.GossipChannels {
		if gossip := channel.gossiper.Gossip(); gossip != nil {
			channel.SendDown(conn, channel.gossiper.Gossip())
		}
	}
}

func (router *Router) handleGossip(tag ProtocolTag, payload []byte) error {
	decoder := gob.NewDecoder(bytes.NewReader(payload))
	var channelName string
	if err := decoder.Decode(&channelName); err != nil {
		return err
	}
	channel, found := router.GossipChannels[channelName]
	if !found {
		return fmt.Errorf("[gossip] received unknown channel with name %s", channelName)
	}
	var srcName PeerName
	if err := decoder.Decode(&srcName); err != nil {
		return err
	}
	switch tag {
	case ProtocolGossipUnicast:
		return channel.deliverUnicast(srcName, payload, decoder)
	case ProtocolGossipBroadcast:
		return channel.deliverBroadcast(srcName, payload, decoder)
	case ProtocolGossip:
		return channel.deliver(srcName, payload, decoder)
	}
	return nil
}

func (c *GossipChannel) deliverUnicast(srcName PeerName, origPayload []byte, dec *gob.Decoder) error {
	var destName PeerName
	if err := dec.Decode(&destName); err != nil {
		return err
	}
	if c.ourself.Name != destName {
		return c.relayUnicast(destName, origPayload)
	}
	var payload []byte
	if err := dec.Decode(&payload); err != nil {
		return err
	}
	return c.gossiper.OnGossipUnicast(srcName, payload)
}

func (c *GossipChannel) deliverBroadcast(srcName PeerName, _ []byte, dec *gob.Decoder) error {
	var payload []byte
	if err := dec.Decode(&payload); err != nil {
		return err
	}
	data, err := c.gossiper.OnGossipBroadcast(payload)
	if err != nil || data == nil {
		return err
	}
	return c.relayBroadcast(srcName, data)
}

func (c *GossipChannel) deliver(srcName PeerName, _ []byte, dec *gob.Decoder) error {
	var payload []byte
	if err := dec.Decode(&payload); err != nil {
		return err
	}
	if data, err := c.gossiper.OnGossip(payload); err != nil {
		return err
	} else if data != nil {
		c.Send(srcName, data)
	}
	return nil
}

func (c *GossipChannel) Send(srcName PeerName, data GossipData) {
	// do this outside the lock below so we avoid lock nesting
	c.routes.EnsureRecalculated()
	selectedConnections := make(ConnectionSet)
	for name := range c.routes.RandomNeighbours(srcName) {
		if conn, found := c.ourself.ConnectionTo(name); found {
			selectedConnections[conn] = void
		}
	}
	if len(selectedConnections) == 0 {
		return
	}
	connections := c.ourself.Connections()
	c.Lock()
	defer c.Unlock()
	// GC - randomly (courtesy of go's map iterator) pick some
	// existing entries and stop&remove them if the associated
	// connection is no longer active.  We stop as soon as we
	// encounter a valid entry; the idea being that when there is
	// little or no garbage then this executes close to O(1)[1],
	// whereas when there is lots of garbage we remove it quickly.
	//
	// [1] TODO Unfortunately, due to the desire to avoid nested
	// locks, instead of simply invoking Peer.ConnectionTo(name)
	// below, we have that Peer.Connections() invocation above. That
	// is O(n_our_connections) at best.
	for conn, sender := range c.senders {
		if _, found := connections[conn]; !found {
			delete(c.senders, conn)
			sender.Stop()
		} else {
			break
		}
	}
	for conn := range selectedConnections {
		c.sendDown(conn, data)
	}
}

func (c *GossipChannel) SendDown(conn Connection, data GossipData) {
	c.Lock()
	c.sendDown(conn, data)
	c.Unlock()
}

func (c *GossipChannel) sendDown(conn Connection, data GossipData) {
	sender, found := c.senders[conn]
	if !found {
		sender = NewGossipSender(func(pending GossipData) {
			protocolMsg := ProtocolMsg{ProtocolGossip, GobEncode(c.name, c.ourself.Name, pending.Encode())}
			conn.(ProtocolSender).SendProtocolMsg(protocolMsg)
		})
		c.senders[conn] = sender
		sender.Start()
	}
	sender.Send(data)
}

func (c *GossipChannel) GossipUnicast(dstPeerName PeerName, msg []byte) error {
	return c.relayUnicast(dstPeerName, GobEncode(c.name, c.ourself.Name, dstPeerName, msg))
}

func (c *GossipChannel) GossipBroadcast(update GossipData) error {
	return c.relayBroadcast(c.ourself.Name, update)
}

func (c *GossipChannel) relayUnicast(dstPeerName PeerName, buf []byte) error {
	if relayPeerName, found := c.routes.UnicastAll(dstPeerName); !found {
		c.log("unknown relay destination:", dstPeerName)
	} else if conn, found := c.ourself.ConnectionTo(relayPeerName); !found {
		c.log("unable to find connection to relay peer", relayPeerName)
	} else {
		conn.(ProtocolSender).SendProtocolMsg(ProtocolMsg{ProtocolGossipUnicast, buf})
	}
	return nil
}

func (c *GossipChannel) relayBroadcast(srcName PeerName, update GossipData) error {
	names := c.routes.PeerNames() // do this outside the lock so they don't nest
	c.Lock()
	defer c.Unlock()
	// GC - randomly (courtesy of go's map iterator) pick some
	// existing broadcasters and stop&remove them if their source peer
	// is unknown. We stop as soon as we encounter a valid entry; the
	// idea being that when there is little or no garbage then this
	// executes close to O(1)[1], whereas when there is lots of
	// garbage we remove it quickly.
	//
	// [1] TODO Unfortunately, due to the desire to avoid nested
	// locks, instead of simply invoking Peers.Fetch(name) below, we
	// have that Peers.Names() invocation above. That is O(n_peers) at
	// best.
	for name, broadcaster := range c.broadcasters {
		if _, found := names[name]; !found {
			delete(c.broadcasters, name)
			broadcaster.Stop()
		} else {
			break
		}
	}
	broadcaster, found := c.broadcasters[srcName]
	if !found {
		broadcaster = NewGossipSender(func(pending GossipData) { c.sendBroadcast(srcName, pending) })
		c.broadcasters[srcName] = broadcaster
		broadcaster.Start()
	}
	broadcaster.Send(update)
	return nil
}

func (c *GossipChannel) sendBroadcast(srcName PeerName, update GossipData) {
	c.routes.EnsureRecalculated()
	nextHops := c.routes.BroadcastAll(srcName)
	if len(nextHops) == 0 {
		return
	}
	protocolMsg := ProtocolMsg{ProtocolGossipBroadcast, GobEncode(c.name, srcName, update.Encode())}
	// FIXME a single blocked connection can stall us
	for _, conn := range c.ourself.ConnectionsTo(nextHops) {
		conn.(ProtocolSender).SendProtocolMsg(protocolMsg)
	}
}

func (c *GossipChannel) log(args ...interface{}) {
	log.Println(append(append([]interface{}{}, "[gossip "+c.name+"]:"), args...)...)
}

// for testing

func (router *Router) sendPendingGossip() bool {
	sentSomething := false
	for _, channel := range router.GossipChannels {
		channel.Lock()
		for _, sender := range channel.senders {
			sentSomething = sender.flush() || sentSomething
		}
		for _, sender := range channel.broadcasters {
			sentSomething = sender.flush() || sentSomething
		}
		channel.Unlock()
	}
	return sentSomething
}

func (sender *GossipSender) flush() bool {
	ch := make(chan bool)
	sender.flushch <- ch
	return <-ch
}
<commit_msg>refactor: introduce NewGossipChannel<commit_after>package router

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"log"
	"sync"
	"time"
)

const GossipInterval = 30 * time.Second

type GossipData interface {
	Encode() []byte
	Merge(GossipData)
}

type Gossip interface {
	// specific message from one peer to another
	// intermediate peers relay it using unicast topology.
	GossipUnicast(dstPeerName PeerName, msg []byte) error
	// send gossip to every peer, relayed using broadcast topology.
	GossipBroadcast(update GossipData) error
}

type Gossiper interface {
	OnGossipUnicast(sender PeerName, msg []byte) error
	// merge received data into state and return a representation of
	// the received data, for further propagation
	OnGossipBroadcast(update []byte) (GossipData, error)
	// return state of everything we know; gets called periodically
	Gossip() GossipData
	// merge received data into state and return "everything new I've
	// just learnt", or nil if nothing in the received data was new
	OnGossip(update []byte) (GossipData, error)
}

// Accumulates GossipData that needs to be sent to one destination,
// and sends it when possible.
type GossipSender struct {
	send func(GossipData)
	cell chan GossipData
	// for testing
	sent    bool
	flushch chan chan bool
}

func NewGossipSender(send func(GossipData)) *GossipSender {
	return &GossipSender{send: send}
}

func (sender *GossipSender) Start() {
	sender.cell = make(chan GossipData, 1)
	sender.flushch = make(chan chan bool)
	go sender.run()
}

func (sender *GossipSender) run() {
	for {
		select {
		case pending := <-sender.cell:
			if pending == nil { // receive zero value when chan is closed
				return
			}
			sender.send(pending)
			sender.sent = true
		case ch := <-sender.flushch:
			// send anything pending, then reply back whether we sent
			// anything since previous flush
			select {
			case pending := <-sender.cell:
				sender.send(pending)
				sender.sent = true
			default:
			}
			ch <- sender.sent
			sender.sent = false
		}
	}
}

func (sender *GossipSender) Send(data GossipData) {
	// NB: this must not be invoked concurrently
	select {
	case pending := <-sender.cell:
		pending.Merge(data)
		sender.cell <- pending
	default:
		sender.cell <- data
	}
}

func (sender *GossipSender) Stop() {
	close(sender.cell)
}

type connectionSenders map[Connection]*GossipSender
type peerSenders map[PeerName]*GossipSender

type GossipChannel struct {
	sync.Mutex
	name         string
	ourself      *LocalPeer
	routes       *Routes
	gossiper     Gossiper
	senders      connectionSenders
	broadcasters peerSenders
}

type GossipChannels map[string]*GossipChannel

func NewGossipChannel(channelName string, ourself *LocalPeer, routes *Routes, g Gossiper) *GossipChannel {
	return &GossipChannel{
		name:         channelName,
		ourself:      ourself,
		routes:       routes,
		gossiper:     g,
		senders:      make(connectionSenders),
		broadcasters: make(peerSenders)}
}

func (router *Router) NewGossip(channelName string, g Gossiper) Gossip {
	channel := NewGossipChannel(channelName, router.Ourself, router.Routes, g)
	router.GossipChannels[channelName] = channel
	return channel
}

func (router *Router) SendAllGossip() {
	for _, channel := range router.GossipChannels {
		if gossip := channel.gossiper.Gossip(); gossip != nil {
			channel.Send(router.Ourself.Name, gossip)
		}
	}
}

func (router *Router) SendAllGossipDown(conn Connection) {
	for _, channel := range router.GossipChannels {
		if gossip := channel.gossiper.Gossip(); gossip != nil {
			channel.SendDown(conn, channel.gossiper.Gossip())
		}
	}
}

func (router *Router) handleGossip(tag ProtocolTag, payload []byte) error {
	decoder := gob.NewDecoder(bytes.NewReader(payload))
	var channelName string
	if err := decoder.Decode(&channelName); err != nil {
		return err
	}
	channel, found := router.GossipChannels[channelName]
	if !found {
		return fmt.Errorf("[gossip] received unknown channel with name %s", channelName)
	}
	var srcName PeerName
	if err := decoder.Decode(&srcName); err != nil {
		return err
	}
	switch tag {
	case ProtocolGossipUnicast:
		return channel.deliverUnicast(srcName, payload, decoder)
	case ProtocolGossipBroadcast:
		return channel.deliverBroadcast(srcName, payload, decoder)
	case ProtocolGossip:
		return channel.deliver(srcName, payload, decoder)
	}
	return nil
}

func (c *GossipChannel) deliverUnicast(srcName PeerName, origPayload []byte, dec *gob.Decoder) error {
	var destName PeerName
	if err := dec.Decode(&destName); err != nil {
		return err
	}
	if c.ourself.Name != destName {
		return c.relayUnicast(destName, origPayload)
	}
	var payload []byte
	if err := dec.Decode(&payload); err != nil {
		return err
	}
	return c.gossiper.OnGossipUnicast(srcName, payload)
}

func (c *GossipChannel) deliverBroadcast(srcName PeerName, _ []byte, dec *gob.Decoder) error {
	var payload []byte
	if err := dec.Decode(&payload); err != nil {
		return err
	}
	data, err := c.gossiper.OnGossipBroadcast(payload)
	if err != nil || data == nil {
		return err
	}
	return c.relayBroadcast(srcName, data)
}

func (c *GossipChannel) deliver(srcName PeerName, _ []byte, dec *gob.Decoder) error {
	var payload []byte
	if err := dec.Decode(&payload); err != nil {
		return err
	}
	if data, err := c.gossiper.OnGossip(payload); err != nil {
		return err
	} else if data != nil {
		c.Send(srcName, data)
	}
	return nil
}

func (c *GossipChannel) Send(srcName PeerName, data GossipData) {
	// do this outside the lock below so we avoid lock nesting
	c.routes.EnsureRecalculated()
	selectedConnections := make(ConnectionSet)
	for name := range c.routes.RandomNeighbours(srcName) {
		if conn, found := c.ourself.ConnectionTo(name); found {
			selectedConnections[conn] = void
		}
	}
	if len(selectedConnections) == 0 {
		return
	}
	connections := c.ourself.Connections()
	c.Lock()
	defer c.Unlock()
	// GC - randomly (courtesy of go's map iterator) pick some
	// existing entries and stop&remove them if the associated
	// connection is no longer active.  We stop as soon as we
	// encounter a valid entry; the idea being that when there is
	// little or no garbage then this executes close to O(1)[1],
	// whereas when there is lots of garbage we remove it quickly.
	//
	// [1] TODO Unfortunately, due to the desire to avoid nested
	// locks, instead of simply invoking Peer.ConnectionTo(name)
	// below, we have that Peer.Connections() invocation above. That
	// is O(n_our_connections) at best.
	for conn, sender := range c.senders {
		if _, found := connections[conn]; !found {
			delete(c.senders, conn)
			sender.Stop()
		} else {
			break
		}
	}
	for conn := range selectedConnections {
		c.sendDown(conn, data)
	}
}

func (c *GossipChannel) SendDown(conn Connection, data GossipData) {
	c.Lock()
	c.sendDown(conn, data)
	c.Unlock()
}

func (c *GossipChannel) sendDown(conn Connection, data GossipData) {
	sender, found := c.senders[conn]
	if !found {
		sender = NewGossipSender(func(pending GossipData) {
			protocolMsg := ProtocolMsg{ProtocolGossip, GobEncode(c.name, c.ourself.Name, pending.Encode())}
			conn.(ProtocolSender).SendProtocolMsg(protocolMsg)
		})
		c.senders[conn] = sender
		sender.Start()
	}
	sender.Send(data)
}

func (c *GossipChannel) GossipUnicast(dstPeerName PeerName, msg []byte) error {
	return c.relayUnicast(dstPeerName, GobEncode(c.name, c.ourself.Name, dstPeerName, msg))
}

func (c *GossipChannel) GossipBroadcast(update GossipData) error {
	return c.relayBroadcast(c.ourself.Name, update)
}

func (c *GossipChannel) relayUnicast(dstPeerName PeerName, buf []byte) error {
	if relayPeerName, found := c.routes.UnicastAll(dstPeerName); !found {
		c.log("unknown relay destination:", dstPeerName)
	} else if conn, found := c.ourself.ConnectionTo(relayPeerName); !found {
		c.log("unable to find connection to relay peer", relayPeerName)
	} else {
		conn.(ProtocolSender).SendProtocolMsg(ProtocolMsg{ProtocolGossipUnicast, buf})
	}
	return nil
}

func (c *GossipChannel) relayBroadcast(srcName PeerName, update GossipData) error {
	names := c.routes.PeerNames() // do this outside the lock so they don't nest
	c.Lock()
	defer c.Unlock()
	// GC - randomly (courtesy of go's map iterator) pick some
	// existing broadcasters and stop&remove them if their source peer
	// is unknown. We stop as soon as we encounter a valid entry; the
	// idea being that when there is little or no garbage then this
	// executes close to O(1)[1], whereas when there is lots of
	// garbage we remove it quickly.
	//
	// [1] TODO Unfortunately, due to the desire to avoid nested
	// locks, instead of simply invoking Peers.Fetch(name) below, we
	// have that Peers.Names() invocation above. That is O(n_peers) at
	// best.
	for name, broadcaster := range c.broadcasters {
		if _, found := names[name]; !found {
			delete(c.broadcasters, name)
			broadcaster.Stop()
		} else {
			break
		}
	}
	broadcaster, found := c.broadcasters[srcName]
	if !found {
		broadcaster = NewGossipSender(func(pending GossipData) { c.sendBroadcast(srcName, pending) })
		c.broadcasters[srcName] = broadcaster
		broadcaster.Start()
	}
	broadcaster.Send(update)
	return nil
}

func (c *GossipChannel) sendBroadcast(srcName PeerName, update GossipData) {
	c.routes.EnsureRecalculated()
	nextHops := c.routes.BroadcastAll(srcName)
	if len(nextHops) == 0 {
		return
	}
	protocolMsg := ProtocolMsg{ProtocolGossipBroadcast, GobEncode(c.name, srcName, update.Encode())}
	// FIXME a single blocked connection can stall us
	for _, conn := range c.ourself.ConnectionsTo(nextHops) {
		conn.(ProtocolSender).SendProtocolMsg(protocolMsg)
	}
}

func (c *GossipChannel) log(args ...interface{}) {
	log.Println(append(append([]interface{}{}, "[gossip "+c.name+"]:"), args...)...)
}

// for testing

func (router *Router) sendPendingGossip() bool {
	sentSomething := false
	for _, channel := range router.GossipChannels {
		channel.Lock()
		for _, sender := range channel.senders {
			sentSomething = sender.flush() || sentSomething
		}
		for _, sender := range channel.broadcasters {
			sentSomething = sender.flush() || sentSomething
		}
		channel.Unlock()
	}
	return sentSomething
}

func (sender *GossipSender) flush() bool {
	ch := make(chan bool)
	sender.flushch <- ch
	return <-ch
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "io"
import "os"
import "time"

/*
[B10]: ARIB-STD B10
[ISO]: ISO/IEC 13818-1
*/

const TS_PACKET_SIZE = 188

type AnalyzerState struct {
	pmtPids          map[int]bool
	pcrPid           int
	captionPid       int
	currentTimestamp SystemClock
	clockOffset      int64
}

type SystemClock int64

func main() {
	if len(os.Args) == 1 {
		fmt.Fprintf(os.Stderr, "usage: %s MPEG2-TS-FILE\n", os.Args[0])
		os.Exit(1)
	}
	fin, err := os.Open(os.Args[1])
	if err != nil {
		panic(err)
	}
	defer func() {
		if err := fin.Close(); err != nil {
			panic(err)
		}
	}()

	buf := make([]byte, TS_PACKET_SIZE)
	state := new(AnalyzerState)
	state.pcrPid = -1
	state.captionPid = -1

	for {
		n, err := fin.Read(buf)
		if err != nil && err != io.EOF {
			panic(err)
		}
		if n == 0 {
			break
		}

		analyzePacket(buf, state)
	}
}

func assertSyncByte(packet []byte) {
	if packet[0] != 0x47 {
		panic("sync_byte failed")
	}
}

func analyzePacket(packet []byte, state *AnalyzerState) {
	assertSyncByte(packet)

	payload_unit_start_indicator := (packet[1] & 0x40) != 0
	pid := int(packet[1]&0x1f)<<8 | int(packet[2])
	hasAdaptation := (packet[3] & 0x20) != 0
	hasPayload := (packet[3] & 0x10) != 0
	p := packet[4:]

	if hasAdaptation {
		// [ISO] 2.4.3.4
		// Table 2-6
		adaptation_field_length := p[0]
		p = p[1:]
		pcr_flag := (p[0] & 0x10) != 0
		if pcr_flag && pid == state.pcrPid {
			state.currentTimestamp = extractPcr(p)
		}
		p = p[adaptation_field_length:]
	}

	if hasPayload {
		if pid == 0 {
			if len(state.pmtPids) == 0 {
				state.pmtPids = extractPmtPids(p[1:])
				fmt.Fprintf(os.Stderr, "Found %d pids: %v\n", len(state.pmtPids), state.pmtPids)
			}
		} else if state.pmtPids != nil && state.pmtPids[pid] {
			if state.captionPid == -1 && payload_unit_start_indicator {
				// PMT section
				pcrPid := extractPcrPid(p[1:])
				captionPid := extractCaptionPid(p[1:])
				if captionPid != -1 {
					fmt.Fprintf(os.Stderr, "caption pid = %d, PCR_PID = %d\n", captionPid, pcrPid)
					state.pcrPid = pcrPid
					state.captionPid = captionPid
				}
			}
		} else if pid == 0x0014 {
			// Time Offset Table
			// [B10] 5.2.9
			t := extractJstTime(p[1:])
			if t != 0 {
				state.clockOffset = t*100 - state.currentTimestamp.centitime()
			}
		}
	}
}

func extractPmtPids(payload []byte) map[int]bool {
	// [ISO] 2.4.4.3
	// Table 2-25
	table_id := payload[0]
	pids := make(map[int]bool)
	if table_id != 0x00 {
		return pids
	}
	section_length := int(payload[1]&0x0F)<<8 | int(payload[2])
	index := 8
	for index < 3+section_length-4 {
		program_number := int(payload[index+0])<<8 | int(payload[index+1])
		if program_number != 0 {
			program_map_PID := int(payload[index+2]&0x1F)<<8 | int(payload[index+3])
			pids[program_map_PID] = true
		}
		index += 4
	}
	return pids
}

func extractPcrPid(payload []byte) int {
	return (int(payload[8]&0x1f) << 8) | int(payload[9])
}

func extractCaptionPid(payload []byte) int {
	// [ISO] 2.4.4.8 Program Map Table
	// Table 2-28
	table_id := payload[0]
	if table_id != 0x02 {
		return -1
	}
	section_length := int(payload[1]&0x0F)<<8 | int(payload[2])
	if section_length >= len(payload) {
		return -1
	}

	program_info_length := int(payload[10]&0x0F)<<8 | int(payload[11])
	index := 12 + program_info_length

	for index < 3+section_length-4 {
		stream_type := payload[index+0]
		ES_info_length := int(payload[index+3]&0xF)<<8 | int(payload[index+4])
		if stream_type == 0x06 {
			elementary_PID := int(payload[index+1]&0x1F)<<8 | int(payload[index+2])
			subIndex := index + 5
			for subIndex < index+ES_info_length {
				// [ISO] 2.6 Program and program element descriptors
				descriptor_tag := payload[subIndex+0]
				descriptor_length := int(payload[subIndex+1])
				if descriptor_tag == 0x52 {
					// [B10] 6.2.16 Stream identifier descriptor
					// 表 6-28
					component_tag := payload[subIndex+2]
					if component_tag == 0x87 {
						return elementary_PID
					}
				}
				subIndex += 2 + descriptor_length
			}
		}
		index += 5 + ES_info_length
	}
	return -1
}

func extractPcr(payload []byte) SystemClock {
	pcr_base := (int64(payload[1]) << 25) |
		(int64(payload[2]) << 17) |
		(int64(payload[3]) << 9) |
		(int64(payload[4]) << 1) |
		(int64(payload[5]&0x80) >> 7)
	pcr_ext := (int64(payload[5] & 0x01)) | int64(payload[6])
	// [ISO] 2.4.2.2
	return SystemClock(pcr_base*300 + pcr_ext)
}

func extractJstTime(payload []byte) int64 {
	if payload[0] != 0x73 {
		return 0
	}

	// [B10] Appendix C
	MJD := (int(payload[3]) << 8) | int(payload[4])
	y := int((float64(MJD) - 15078.2) / 365.25)
	m := int((float64(MJD) - 14956.1 - float64(int(float64(y)*365.25))) / 30.6001)
	k := 0
	if m == 14 || m == 15 {
		k = 1
	}
	year := y + k + 1900
	month := m - 2 - k*12
	day := MJD - 14956 - int(float64(y)*365.25) - int(float64(m)*30.6001)
	hour := decodeBcd(payload[5])
	minute := decodeBcd(payload[6])
	second := decodeBcd(payload[7])

	str := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d+09:00", year, month, day, hour, minute, second)
	t, err := time.Parse(time.RFC3339, str)
	if err != nil {
		panic(err)
	}
	return t.Unix()
}

func decodeBcd(n byte) int {
	return (int(n)>>4)*10 + int(n&0x0f)
}

const K int64 = 27000000

func (clock SystemClock) centitime() int64 {
	return int64(clock) / (K / 100)
}
<commit_msg>Print subtitle<commit_after>package main

import "fmt"
import "io"
import "os"
import "time"

/*
[B10]: ARIB-STD B10
[ISO]: ISO/IEC 13818-1
*/

const TS_PACKET_SIZE = 188

type AnalyzerState struct {
	pmtPids           map[int]bool
	pcrPid            int
	captionPid        int
	currentTimestamp  SystemClock
	clockOffset       int64
	previousSubtitle  string
	previousIsBlank   bool
	previousTimestamp SystemClock
	preludePrinted    bool
}

type SystemClock int64

func main() {
	if len(os.Args) == 1 {
		fmt.Fprintf(os.Stderr, "usage: %s MPEG2-TS-FILE\n", os.Args[0])
		os.Exit(1)
	}
	fin, err := os.Open(os.Args[1])
	if err != nil {
		panic(err)
	}
	defer func() {
		if err := fin.Close(); err != nil {
			panic(err)
		}
	}()

	buf := make([]byte, TS_PACKET_SIZE)
	state := new(AnalyzerState)
	state.pcrPid = -1
	state.captionPid = -1

	for {
		n, err := fin.Read(buf)
		if err != nil && err != io.EOF {
			panic(err)
		}
		if n == 0 {
			break
		}

		analyzePacket(buf, state)
	}
}

func assertSyncByte(packet []byte) {
	if packet[0] != 0x47 {
		panic("sync_byte failed")
	}
}

func analyzePacket(packet []byte, state *AnalyzerState) {
	assertSyncByte(packet)

	payload_unit_start_indicator := (packet[1] & 0x40) != 0
	pid := int(packet[1]&0x1f)<<8 | int(packet[2])
	hasAdaptation := (packet[3] & 0x20) != 0
	hasPayload := (packet[3] & 0x10) != 0
	p := packet[4:]

	if hasAdaptation {
		// [ISO] 2.4.3.4
		// Table 2-6
		adaptation_field_length := p[0]
		p = p[1:]
		pcr_flag := (p[0] & 0x10) != 0
		if pcr_flag && pid == state.pcrPid {
			state.currentTimestamp = extractPcr(p)
		}
		p = p[adaptation_field_length:]
	}

	if hasPayload {
		if pid == 0 {
			if len(state.pmtPids) == 0 {
				state.pmtPids = extractPmtPids(p[1:])
				fmt.Fprintf(os.Stderr, "Found %d pids: %v\n", len(state.pmtPids), state.pmtPids)
			}
		} else if state.pmtPids != nil && state.pmtPids[pid] {
			if state.captionPid == -1 && payload_unit_start_indicator {
				// PMT section
				pcrPid := extractPcrPid(p[1:])
				captionPid := extractCaptionPid(p[1:])
				if captionPid != -1 {
					fmt.Fprintf(os.Stderr, "caption pid = %d, PCR_PID = %d\n", captionPid, pcrPid)
					state.pcrPid = pcrPid
					state.captionPid = captionPid
				}
			}
		} else if pid == 0x0014 {
			// Time Offset Table
			// [B10] 5.2.9
			t := extractJstTime(p[1:])
			if t != 0 {
				state.clockOffset = t*100 - state.currentTimestamp.centitime()
			}
		} else if pid == state.captionPid {
			if payload_unit_start_indicator {
				dumpCaption(p, state)
			}
		}
	}
}

func extractPmtPids(payload []byte) map[int]bool {
	// [ISO] 2.4.4.3
	// Table 2-25
	table_id := payload[0]
	pids := make(map[int]bool)
	if table_id != 0x00 {
		return pids
	}
	section_length := int(payload[1]&0x0F)<<8 | int(payload[2])
	index := 8
	for index < 3+section_length-4 {
		program_number := int(payload[index+0])<<8 | int(payload[index+1])
		if program_number != 0 {
			program_map_PID := int(payload[index+2]&0x1F)<<8 | int(payload[index+3])
			pids[program_map_PID] = true
		}
		index += 4
	}
	return pids
}

func extractPcrPid(payload []byte) int {
	return (int(payload[8]&0x1f) << 8) | int(payload[9])
}

func extractCaptionPid(payload []byte) int {
	// [ISO] 2.4.4.8 Program Map Table
	// Table 2-28
	table_id := payload[0]
	if table_id != 0x02 {
		return -1
	}
	section_length := int(payload[1]&0x0F)<<8 | int(payload[2])
	if section_length >= len(payload) {
		return -1
	}

	program_info_length := int(payload[10]&0x0F)<<8 | int(payload[11])
	index := 12 + program_info_length

	for index < 3+section_length-4 {
		stream_type := payload[index+0]
		ES_info_length := int(payload[index+3]&0xF)<<8 | int(payload[index+4])
		if stream_type == 0x06 {
			elementary_PID := int(payload[index+1]&0x1F)<<8 | int(payload[index+2])
			subIndex := index + 5
			for subIndex < index+ES_info_length {
				// [ISO] 2.6 Program and program element descriptors
				descriptor_tag := payload[subIndex+0]
				descriptor_length := int(payload[subIndex+1])
				if descriptor_tag == 0x52 {
					// [B10] 6.2.16 Stream identifier descriptor
					// 表 6-28
					component_tag := payload[subIndex+2]
					if component_tag == 0x87 {
						return elementary_PID
					}
				}
				subIndex += 2 + descriptor_length
			}
		}
		index += 5 + ES_info_length
	}
	return -1
}

func extractPcr(payload []byte) SystemClock {
	pcr_base := (int64(payload[1]) << 25) |
		(int64(payload[2]) << 17) |
		(int64(payload[3]) << 9) |
		(int64(payload[4]) << 1) |
		(int64(payload[5]&0x80) >> 7)
	pcr_ext := (int64(payload[5] & 0x01)) | int64(payload[6])
	// [ISO] 2.4.2.2
	return SystemClock(pcr_base*300 + pcr_ext)
}

func extractJstTime(payload []byte) int64 {
	if payload[0] != 0x73 {
		return 0
	}

	// [B10] Appendix C
	MJD := (int(payload[3]) << 8) | int(payload[4])
	y := int((float64(MJD) - 15078.2) / 365.25)
	m := int((float64(MJD) - 14956.1 - float64(int(float64(y)*365.25))) / 30.6001)
	k := 0
	if m == 14 || m == 15 {
		k = 1
	}
	year := y + k + 1900
	month := m - 2 - k*12
	day := MJD - 14956 - int(float64(y)*365.25) - int(float64(m)*30.6001)
	hour := decodeBcd(payload[5])
	minute := decodeBcd(payload[6])
	second := decodeBcd(payload[7])

	str := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d+09:00", year, month, day, hour, minute, second)
	t, err := time.Parse(time.RFC3339, str)
	if err != nil {
		panic(err)
	}
	return t.Unix()
}

func decodeBcd(n byte) int {
	return (int(n)>>4)*10 + int(n&0x0f)
}

func dumpCaption(payload []byte, state *AnalyzerState) {
	PES_header_data_length := payload[8]
	PES_data_packet_header_length := payload[11+PES_header_data_length] & 0x0F
	p := payload[12+PES_header_data_length+PES_data_packet_header_length:]

	// [B24] Table 9-1 (p184)
	data_group_id := (p[0] & 0xFC) >> 2
	if data_group_id == 0x00 || data_group_id == 0x20 {
		// [B24] Table 9-3 (p186)
		// caption_management_data
		num_languages := p[6]
		p = p[7+num_languages*5:]
	} else {
		// caption_data
		p = p[6:]
	}
	// [B24] Table 9-3 (p186)
	data_unit_loop_length := (int(p[0]) << 16) | (int(p[1]) << 8) | int(p[2])
	index := 0
	for index < data_unit_loop_length {
		q := p[index:]
		data_unit_parameter := q[4]
		data_unit_size := (int(q[5]) << 16) | (int(q[6]) << 8) | int(q[7])
		if data_unit_parameter == 0x20 {
			if len(state.previousSubtitle) != 0 && !(isBlank(state.previousSubtitle) && state.previousIsBlank) {
				prevTimeCenti := state.previousTimestamp.centitime() + state.clockOffset
				curTimeCenti := state.currentTimestamp.centitime() + state.clockOffset
				prevTime := prevTimeCenti / 100
				curTime := curTimeCenti / 100
				prevCenti := prevTimeCenti % 100
				curCenti := curTimeCenti % 100
				prev := time.Unix(prevTime, 0)
				cur := time.Unix(curTime, 0)
				if !state.preludePrinted {
					printPrelude()
					state.preludePrinted = true
				}
				fmt.Printf("Dialogue: 0,%02d:%02d:%02d.%02d,%02d:%02d:%02d.%02d,Default,,,,,,%s\n",
					prev.Hour(), prev.Minute(), prev.Second(), prevCenti,
					cur.Hour(), cur.Minute(), cur.Second(), curCenti,
					state.previousSubtitle)
			}
			state.previousIsBlank = isBlank(state.previousSubtitle)
			state.previousSubtitle = decodeCprofile(q[8:], data_unit_size)
			state.previousTimestamp = state.currentTimestamp
		}
		index += 5 + data_unit_size
	}
}

func isBlank(str string) bool {
	for _, c := range str {
		if c != ' ' {
			return false
		}
	}
	return true
}

func printPrelude() {
	fmt.Println("[Script Info]")
	fmt.Println("ScriptType: v4.00+")
	fmt.Println("Collisions: Normal")
	fmt.Println("ScaledBorderAndShadow: yes")
	fmt.Println("Timer: 100.0000")
	fmt.Println("\n[Events]")
}

func decodeCprofile(str []byte, length int) string {
	return "dummy"
}

const K int64 = 27000000

func (clock SystemClock) centitime() int64 {
	return int64(clock) / (K / 100)
}
<|endoftext|>


<commit_before>package routes

import (
	"fmt"
	"log"
	"net/http"
	"net/url"
)

type router struct {
	root *url.URL

	getHandlers  map[string]GetHandlerFunc
	postHandlers map[string]PostHandlerFunc
}

func NewRouter(rootPath string) (*router, error) {
	r := &router{}

	err := r.SetRootPath(rootPath)
	if err != nil {
		return r, err
	}

	return r, nil
}

func (r *router) SetRootPath(path string) error {
	newRoot, err := url.Parse(path)
	if err != nil {
		return fmt.Errorf("invalid path format %s: %v", path, err)
	}

	r.root = newRoot

	return nil
}

func (r *router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	path := req.URL.Path
	method := req.Method

	switch method {

	case http.MethodGet:
		if route, ok := r.getHandlers[path]; ok {
			route(&w, valuesToGetParams(req.URL.Query()), nil)
		}
		http.NotFound(w, req)
	case http.MethodPost:
		if route, ok := r.postHandlers[path]; ok {
			var body []byte
			_, err := req.Body.Read(body)
			if err != nil {
				w.WriteHeader(http.StatusInternalServerError)
				log.Panicf("can not read request body: %v", err)
			}

			route(&w, PostBody(body), nil)
		}
		http.NotFound(w, req)

	default:
		w.WriteHeader(http.StatusMethodNotAllowed)
		fmt.Fprintf(w, "method not allowed: %s", method)
	}
}

type GetHandlerFunc func(*http.ResponseWriter, GetParams, PathParams)
type PostHandlerFunc func(*http.ResponseWriter, PostBody, PathParams)

type PathParams map[string][]byte
type GetParams map[string]string
type PostBody []byte // Byte array with request body

func valuesToGetParams(values url.Values) GetParams {
	var params map[string]string
	for key := range values {
		params[key] = values.Get(key)
	}
	return params
}
<commit_msg>Add scratch for router structure.<commit_after>package routes

import (
	"fmt"
	"net/http"
	"net/url"
)

func NewRouter(rootPath string) (*router, error) {
	r := &router{}

	err := r.SetRootPath(rootPath)
	if err != nil {
		return r, err
	}

	return r, nil
}

type Pattern string

type router struct {
	root *url.URL

	getHandlers  map[Pattern]GetHandlerFunc
	postHandlers map[Pattern]PostHandlerFunc
}

// Set router root path, other paths will be relative to it
func (r *router) SetRootPath(path string) error {
	newRoot, err := url.Parse(path)
	if err != nil {
		return fmt.Errorf("invalid path format %s: %v", path, err)
	}

	r.root = newRoot

	return nil
}

//Helper types for different http method handlers
type GetHandlerFunc func(*http.ResponseWriter, GetParams, PathParams)
type PostHandlerFunc func(*http.ResponseWriter, PostBody, PathParams)

// Example: url "/api/v1/users/1" and pattern "/api/v1/users/:id"
// path params = {"id": "1"}
type PathParams map[string][]byte

// Get params stands for "query params"
type GetParams map[string]string

// Converts url.Url.Query() from "Values" (map[string][]string)
// to "GetParams" (map[string]string)
func valuesToGetParams(values url.Values) GetParams {
	var params map[string]string
	for key := range values {
		params[key] = values.Get(key)
	}
	return params
}

// Type for http post body
type PostBody []byte // Byte array with request body

// Add new get handler
func (r *router) Get(pattern string, handler GetHandlerFunc) error {
	fullPattern, err := r.root.Parse(pattern)
	if err != nil {
		return err
	}

	r.getHandlers[Pattern(fullPattern.Path)] = handler

	return nil
}

func (r *router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	method := req.Method

	switch method {

	case http.MethodGet:
		r.handleGet(w, req)
	case http.MethodPost:
		r.handlePost(w, req)
	default:
		w.WriteHeader(http.StatusMethodNotAllowed)
		fmt.Fprintf(w, "method not allowed: %s", method)
	}
}

func (r *router) handlePost(w http.ResponseWriter, req *http.Request) {

}

func (r *router) handleGet(w http.ResponseWriter, req *http.Request) {

}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"koding/artifact"
	"koding/common"
	"koding/db/mongodb/modelhelper"
	"koding/tools/config"
	"net"
	"net/http"
	"runtime"
	"time"

	"github.com/PuerkitoBio/throttled"
	"github.com/PuerkitoBio/throttled/store"
	"github.com/koding/logging"
	"github.com/koding/metrics"
	"github.com/koding/redis"
)

var (
	WorkerName = "ingestor"
	flagConfig = flag.String("c", "dev", "Configuration profile from file")
)

func initializeConf() *config.Config {
	runtime.GOMAXPROCS(runtime.NumCPU())

	flag.Parse()
	if *flagConfig == "" {
		panic("Please define config file with -c")
	}

	return config.MustConfig(*flagConfig)
}

func main() {
	log := common.CreateLogger(WorkerName, false)

	conf := initializeConf()
	modelhelper.Initialize(conf.Mongo)

	defer modelhelper.Close()

	redisConn, err := redis.NewRedisSession(&redis.RedisConf{Server: conf.Redis})
	if err != nil {
		log.Fatal(err.Error())
	}

	defer redisConn.Close()

	dogclient, err := metrics.NewDogStatsD(WorkerName)
	if err != nil {
		log.Fatal(err.Error())
	}

	stathandler := &GatherStat{log: log, dog: dogclient}
	errhandler := &GatherError{log: log, dog: dogclient}

	mux := http.NewServeMux()

	th := throttled.RateLimit(
		throttled.Q{Requests: 10, Window: time.Hour},
		&throttled.VaryBy{Path: true},
		store.NewRedisStore(redisConn.Pool(), WorkerName, 0),
	)

	tStathandler := th.Throttle(stathandler)
	mux.Handle("/ingest", tStathandler)

	tErrHandler := th.Throttle(errhandler)
	mux.Handle("/errors", tErrHandler)

	mux.HandleFunc("/version", artifact.VersionHandler())
	mux.HandleFunc("/healthCheck", artifact.HealthCheckHandler(WorkerName))

	port := fmt.Sprintf("%v", conf.GatherIngestor.Port)

	log.Info("Listening on server: %s", port)

	listener, err := net.Listen("tcp", ":"+port)
	if err != nil {
		log.Fatal(err.Error())
	}

	defer listener.Close()

	if err = http.Serve(listener, mux); err != nil {
		log.Fatal(err.Error())
	}
}

func write500Err(log logging.Logger, err error, w http.ResponseWriter) {
	writeErr(http.StatusInternalServerError, log, err, w)
}

func write404Err(log logging.Logger, err error, w http.ResponseWriter) {
	writeErr(http.StatusBadRequest, log, err, w)
}

func writeErr(code int, log logging.Logger, err error, w http.ResponseWriter) {
	log.Error(err.Error())

	w.WriteHeader(code)
	w.Write([]byte(err.Error()))
}
<commit_msg>gatheringestor: throttle by remove address, but not path<commit_after>package main

import (
	"flag"
	"fmt"
	"koding/artifact"
	"koding/common"
	"koding/db/mongodb/modelhelper"
	"koding/tools/config"
	"net"
	"net/http"
	"runtime"

	"github.com/PuerkitoBio/throttled"
	"github.com/PuerkitoBio/throttled/store"
	"github.com/koding/logging"
	"github.com/koding/metrics"
	"github.com/koding/redis"
)

var (
	WorkerName = "ingestor"
	flagConfig = flag.String("c", "dev", "Configuration profile from file")
)

func initializeConf() *config.Config {
	runtime.GOMAXPROCS(runtime.NumCPU())

	flag.Parse()
	if *flagConfig == "" {
		panic("Please define config file with -c")
	}

	return config.MustConfig(*flagConfig)
}

func main() {
	log := common.CreateLogger(WorkerName, false)

	conf := initializeConf()
	modelhelper.Initialize(conf.Mongo)

	defer modelhelper.Close()

	redisConn, err := redis.NewRedisSession(&redis.RedisConf{Server: conf.Redis})
	if err != nil {
		log.Fatal(err.Error())
	}

	defer redisConn.Close()

	dogclient, err := metrics.NewDogStatsD(WorkerName)
	if err != nil {
		log.Fatal(err.Error())
	}

	stathandler := &GatherStat{log: log, dog: dogclient}
	errhandler := &GatherError{log: log, dog: dogclient}

	mux := http.NewServeMux()

	th := throttled.RateLimit(
		throttled.PerHour(10),
		&throttled.VaryBy{RemoteAddr: true, Path: false},
		store.NewRedisStore(redisConn.Pool(), WorkerName, 0),
	)

	tStathandler := th.Throttle(stathandler)
	mux.Handle("/ingest", tStathandler)

	tErrHandler := th.Throttle(errhandler)
	mux.Handle("/errors", tErrHandler)

	mux.HandleFunc("/version", artifact.VersionHandler())
	mux.HandleFunc("/healthCheck", artifact.HealthCheckHandler(WorkerName))

	port := fmt.Sprintf("%v", conf.GatherIngestor.Port)

	log.Info("Listening on server: %s", port)

	listener, err := net.Listen("tcp", ":"+port)
	if err != nil {
		log.Fatal(err.Error())
	}

	defer listener.Close()

	if err = http.Serve(listener, mux); err != nil {
		log.Fatal(err.Error())
	}
}

func write500Err(log logging.Logger, err error, w http.ResponseWriter) {
	writeErr(http.StatusInternalServerError, log, err, w)
}

func write404Err(log logging.Logger, err error, w http.ResponseWriter) {
	writeErr(http.StatusBadRequest, log, err, w)
}

func writeErr(code int, log logging.Logger, err error, w http.ResponseWriter) {
	log.Error(err.Error())

	w.WriteHeader(code)
	w.Write([]byte(err.Error()))
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package azblobbackupstorage implements the BackupStorage interface
// for Azure Blob Storage
package azblobbackupstorage

import (
	"context"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/Azure/azure-pipeline-go/pipeline"
	"github.com/Azure/azure-storage-blob-go/azblob"
	"vitess.io/vitess/go/vt/concurrency"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/mysqlctl/backupstorage"
)

var (
	// This is the account name
	accountName = flag.String("azblob_backup_account_name", "", "Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used")

	// This is the private access key
	accountKeyFile = flag.String("azblob_backup_account_key_file", "", "Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path)")

	// This is the name of the container that will store the backups
	containerName = flag.String("azblob_backup_container_name", "", "Azure Blob Container Name")

	// This is an optional prefix to prepend to all files
	storageRoot = flag.String("azblob_backup_storage_root", "", "Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/')")

	azBlobParallelism = flag.Int("azblob_backup_parallelism", 1, "Azure Blob operation parallelism (requires extra memory when increased)")
)

const (
	defaultRetryCount = 5
	delimiter         = "/"
)

// Return a Shared credential from the available credential sources.
// We will use credentials in the following order
// 1. Direct Command Line Flag (azblob_backup_account_name, azblob_backup_account_key)
// 2. Environment variables
func azInternalCredentials() (string, string, error) {
	actName := *accountName
	if actName == "" {
		// Check the Environmental Value
		actName = os.Getenv("VT_AZBLOB_ACCOUNT_NAME")
	}

	var actKey string
	if *accountKeyFile != "" {
		log.Infof("Getting Azure Storage Account key from file: %s", *accountKeyFile)
		dat, err := ioutil.ReadFile(*accountKeyFile)
		if err != nil {
			return "", "", err
		}
		actKey = string(dat)
	} else {
		actKey = os.Getenv("VT_AZBLOB_ACCOUNT_KEY")
	}

	if actName == "" || actKey == "" {
		return "", "", fmt.Errorf("Azure Storage Account credentials not found in command-line flags or environment variables")
	}
	return actName, actKey, nil
}

func azCredentials() (*azblob.SharedKeyCredential, error) {
	actName, actKey, err := azInternalCredentials()
	if err != nil {
		return nil, err
	}
	return azblob.NewSharedKeyCredential(actName, actKey)
}

func azServiceURL(credentials *azblob.SharedKeyCredential) azblob.ServiceURL {
	pipeline := azblob.NewPipeline(credentials, azblob.PipelineOptions{
		Retry: azblob.RetryOptions{
			Policy:   azblob.RetryPolicyFixed,
			MaxTries: defaultRetryCount,
			// Per https://godoc.org/github.com/Azure/azure-storage-blob-go/azblob#RetryOptions
			// this should be set to a very nigh number (they claim 60s per MB).
			// That could end up being days so we are limiting this to four hours.
			TryTimeout: 4 * time.Hour,
		},
		Log: pipeline.LogOptions{
			Log: func(level pipeline.LogLevel, message string) {
				switch level {
				case pipeline.LogFatal:
				case pipeline.LogPanic:
					log.Fatal(message)
					break
				case pipeline.LogError:
					log.Error(message)
					break
				case pipeline.LogWarning:
					log.Warning(message)
					break
				case pipeline.LogInfo:
				case pipeline.LogDebug:
					log.Info(message)
				}
			},
			ShouldLog: func(level pipeline.LogLevel) bool {
				switch level {
				case pipeline.LogFatal:
				case pipeline.LogPanic:
					return bool(log.V(3))
				case pipeline.LogError:
					return bool(log.V(3))
				case pipeline.LogWarning:
					return bool(log.V(2))
				case pipeline.LogInfo:
				case pipeline.LogDebug:
					return bool(log.V(1))
				}
				return false
			},
		},
	})
	u := url.URL{
		Scheme: "https",
		Host:   credentials.AccountName() + ".blob.core.windows.net",
		Path:   "/",
	}
	return azblob.NewServiceURL(u, pipeline)
}

// AZBlobBackupHandle implements BackupHandle for Azure Blob service.
type AZBlobBackupHandle struct {
	bs        *AZBlobBackupStorage
	dir       string
	name      string
	readOnly  bool
	waitGroup sync.WaitGroup
	errors    concurrency.AllErrorRecorder
	ctx       context.Context
	cancel    context.CancelFunc
}

// Directory implements BackupHandle.
func (bh *AZBlobBackupHandle) Directory() string {
	return bh.dir
}

// Name implements BackupHandle.
func (bh *AZBlobBackupHandle) Name() string {
	return bh.name
}

// RecordError is part of the concurrency.ErrorRecorder interface.
func (bh *AZBlobBackupHandle) RecordError(err error) {
	bh.errors.RecordError(err)
}

// HasErrors is part of the concurrency.ErrorRecorder interface.
func (bh *AZBlobBackupHandle) HasErrors() bool {
	return bh.errors.HasErrors()
}

// Error is part of the concurrency.ErrorRecorder interface.
func (bh *AZBlobBackupHandle) Error() error {
	return bh.errors.Error()
}

// AddFile implements BackupHandle.
func (bh *AZBlobBackupHandle) AddFile(ctx context.Context, filename string, filesize int64) (io.WriteCloser, error) {
	if bh.readOnly {
		return nil, fmt.Errorf("AddFile cannot be called on read-only backup")
	}
	// Error out if the file size it too large ( ~4.75 TB)
	if filesize > azblob.BlockBlobMaxStageBlockBytes*azblob.BlockBlobMaxBlocks {
		return nil, fmt.Errorf("filesize (%v) is too large to upload to az blob (max size %v)", filesize, azblob.BlockBlobMaxStageBlockBytes*azblob.BlockBlobMaxBlocks)
	}

	obj := objName(bh.dir, bh.name, filename)
	containerURL, err := bh.bs.containerURL()
	if err != nil {
		return nil, err
	}

	blockBlobURL := containerURL.NewBlockBlobURL(obj)

	reader, writer := io.Pipe()
	bh.waitGroup.Add(1)

	go func() {
		defer bh.waitGroup.Done()
		_, err := azblob.UploadStreamToBlockBlob(bh.ctx, reader, blockBlobURL, azblob.UploadStreamToBlockBlobOptions{
			BufferSize: azblob.BlockBlobMaxStageBlockBytes,
			MaxBuffers: *azBlobParallelism,
		})
		if err != nil {
			reader.CloseWithError(err)
			bh.RecordError(err)
		}
	}()

	return writer, nil
}

// EndBackup implements BackupHandle.
func (bh *AZBlobBackupHandle) EndBackup(ctx context.Context) error {
	if bh.readOnly {
		return fmt.Errorf("EndBackup cannot be called on read-only backup")
	}
	bh.waitGroup.Wait()
	return bh.Error()
}

// AbortBackup implements BackupHandle.
func (bh *AZBlobBackupHandle) AbortBackup(ctx context.Context) error {
	if bh.readOnly {
		return fmt.Errorf("AbortBackup cannot be called on read-only backup")
	}
	// Cancel the context of any uploads.
	bh.cancel()

	// Remove the backup
	return bh.bs.RemoveBackup(ctx, bh.dir, bh.name)
}

// ReadFile implements BackupHandle.
func (bh *AZBlobBackupHandle) ReadFile(ctx context.Context, filename string) (io.ReadCloser, error) {
	if !bh.readOnly {
		return nil, fmt.Errorf("ReadFile cannot be called on read-write backup")
	}

	obj := objName(bh.dir, filename)
	containerURL, err := bh.bs.containerURL()
	if err != nil {
		return nil, err
	}
	blobURL := containerURL.NewBlobURL(obj)

	resp, err := blobURL.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false)
	if err != nil {
		return nil, err
	}
	return resp.Body(azblob.RetryReaderOptions{
		MaxRetryRequests: defaultRetryCount,
		NotifyFailedRead: func(failureCount int, lastError error, offset int64, count int64, willRetry bool) {
			log.Warningf("ReadFile: [azblob] container: %s, directory: %s, filename: %s, error: %v", *containerName, objName(bh.dir, ""), filename, lastError)
		},
		TreatEarlyCloseAsError: true,
	}), nil
}

// AZBlobBackupStorage structs implements the BackupStorage interface for AZBlob
type AZBlobBackupStorage struct {
}

func (bs *AZBlobBackupStorage) containerURL() (*azblob.ContainerURL, error) {
	credentials, err := azCredentials()
	if err != nil {
		return nil, err
	}
	u := azServiceURL(credentials).NewContainerURL(*containerName)
	return &u, nil
}

// ListBackups implements BackupStorage.
func (bs *AZBlobBackupStorage) ListBackups(ctx context.Context, dir string) ([]backupstorage.BackupHandle, error) {
	log.Infof("ListBackups: [azblob] container: %s, directory: %v", *containerName, objName(dir, ""))

	containerURL, err := bs.containerURL()
	if err != nil {
		return nil, err
	}

	searchPrefix := objName(dir, "")

	result := make([]backupstorage.BackupHandle, 0)
	var subdirs []string

	for marker := (azblob.Marker{}); marker.NotDone(); {
		// This returns Blobs in sorted order so we don't need to sort them a second time.
		resp, err := containerURL.ListBlobsHierarchySegment(ctx, marker, delimiter, azblob.ListBlobsSegmentOptions{
			Prefix:     searchPrefix,
			MaxResults: 0,
		})

		if err != nil {
			return nil, err
		}

		for _, item := range resp.Segment.BlobPrefixes {
			subdir := strings.TrimPrefix(item.Name, searchPrefix)
			subdir = strings.TrimSuffix(subdir, delimiter)
			subdirs = append(subdirs, subdir)
		}

		marker = resp.NextMarker
	}

	for _, subdir := range subdirs {
		cancelableCtx, cancel := context.WithCancel(ctx)
		result = append(result, &AZBlobBackupHandle{
			bs:       bs,
			dir:      strings.Join([]string{dir, subdir}, "/"),
			name:     subdir,
			readOnly: true,
			ctx:      cancelableCtx,
			cancel:   cancel,
		})
	}

	return result, nil
}

// StartBackup implements BackupStorage.
func (bs *AZBlobBackupStorage) StartBackup(ctx context.Context, dir, name string) (backupstorage.BackupHandle, error) {
	cancelableCtx, cancel := context.WithCancel(ctx)
	return &AZBlobBackupHandle{
		bs:       bs,
		dir:      dir,
		name:     name,
		readOnly: false,
		ctx:      cancelableCtx,
		cancel:   cancel,
	}, nil
}

// RemoveBackup implements BackupStorage.
func (bs *AZBlobBackupStorage) RemoveBackup(ctx context.Context, dir, name string) error {
	log.Infof("ListBackups: [azblob] container: %s, directory: %s", *containerName, objName(dir, ""))

	containerURL, err := bs.containerURL()
	if err != nil {
		return err
	}

	searchPrefix := objName(dir, name, "")

	for marker := (azblob.Marker{}); marker.NotDone(); {
		resp, err := containerURL.ListBlobsHierarchySegment(ctx, marker, delimiter, azblob.ListBlobsSegmentOptions{
			Prefix:     searchPrefix,
			MaxResults: 0,
		})

		if err != nil {
			return err
		}

		// Right now there is no batch delete so we must iterate over all the blobs to delete them one by one
		// One day we will be able to use this https://docs.microsoft.com/en-us/rest/api/storageservices/blob-batch
		// but currently it is listed as a preview and its not in the go API
		for _, item := range resp.Segment.BlobItems {
			_, err := containerURL.NewBlobURL(item.Name).Delete(ctx, azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{})
			if err != nil {
				return err
			}
		}
		marker = resp.NextMarker
	}

	// Delete the blob representing the folder of the backup, remove any trailing slash to signify we want to remove the folder
	// NOTE: you must set DeleteSnapshotsOptionNone or this will error out with a server side error
	for retry := 0; retry < defaultRetryCount; retry = retry + 1 {
		// Since the deletion of blob's is asyncronious we may need to wait a bit before we delete the folder
		// Also refresh the client just for good measure
		time.Sleep(10 * time.Second)
		containerURL, err = bs.containerURL()

		log.Infof("Removing backup directory: %v", strings.TrimSuffix(searchPrefix, "/"))
		_, err = containerURL.NewBlobURL(strings.TrimSuffix(searchPrefix, "/")).Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
		if err == nil {
			break
		}
	}
	return err
}

// Close implements BackupStorage.
func (bs *AZBlobBackupStorage) Close() error {
	// This function is a No-op
	return nil
}

// objName joins path parts into an object name.
// Unlike path.Join, it doesn't collapse ".." or strip trailing slashes.
// It also adds the value of the -azblob_backup_storage_root flag if set.
func objName(parts ...string) string {
	if *storageRoot != "" {
		return *storageRoot + "/" + strings.Join(parts, "/")
	}
	return strings.Join(parts, "/")
}

func init() {
	backupstorage.BackupStorageMap["azblob"] = &AZBlobBackupStorage{}
}
<commit_msg>Fixing the Linter<commit_after>/*
Copyright 2020 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package azblobbackupstorage implements the BackupStorage interface
// for Azure Blob Storage
package azblobbackupstorage

import (
	"context"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/Azure/azure-pipeline-go/pipeline"
	"github.com/Azure/azure-storage-blob-go/azblob"
	"vitess.io/vitess/go/vt/concurrency"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/mysqlctl/backupstorage"
)

var (
	// This is the account name
	accountName = flag.String("azblob_backup_account_name", "", "Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used")

	// This is the private access key
	accountKeyFile = flag.String("azblob_backup_account_key_file", "", "Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path)")

	// This is the name of the container that will store the backups
	containerName = flag.String("azblob_backup_container_name", "", "Azure Blob Container Name")

	// This is an optional prefix to prepend to all files
	storageRoot = flag.String("azblob_backup_storage_root", "", "Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/')")

	azBlobParallelism = flag.Int("azblob_backup_parallelism", 1, "Azure Blob operation parallelism (requires extra memory when increased)")
)

const (
	defaultRetryCount = 5
	delimiter         = "/"
)

// Return a Shared credential from the available credential sources.
// We will use credentials in the following order
// 1. Direct Command Line Flag (azblob_backup_account_name, azblob_backup_account_key)
// 2. Environment variables
func azInternalCredentials() (string, string, error) {
	actName := *accountName
	if actName == "" {
		// Check the Environmental Value
		actName = os.Getenv("VT_AZBLOB_ACCOUNT_NAME")
	}

	var actKey string
	if *accountKeyFile != "" {
		log.Infof("Getting Azure Storage Account key from file: %s", *accountKeyFile)
		dat, err := ioutil.ReadFile(*accountKeyFile)
		if err != nil {
			return "", "", err
		}
		actKey = string(dat)
	} else {
		actKey = os.Getenv("VT_AZBLOB_ACCOUNT_KEY")
	}

	if actName == "" || actKey == "" {
		return "", "", fmt.Errorf("Azure Storage Account credentials not found in command-line flags or environment variables")
	}
	return actName, actKey, nil
}

func azCredentials() (*azblob.SharedKeyCredential, error) {
	actName, actKey, err := azInternalCredentials()
	if err != nil {
		return nil, err
	}
	return azblob.NewSharedKeyCredential(actName, actKey)
}

func azServiceURL(credentials *azblob.SharedKeyCredential) azblob.ServiceURL {
	pipeline := azblob.NewPipeline(credentials, azblob.PipelineOptions{
		Retry: azblob.RetryOptions{
			Policy:   azblob.RetryPolicyFixed,
			MaxTries: defaultRetryCount,
			// Per https://godoc.org/github.com/Azure/azure-storage-blob-go/azblob#RetryOptions
			// this should be set to a very nigh number (they claim 60s per MB).
			// That could end up being days so we are limiting this to four hours.
			TryTimeout: 4 * time.Hour,
		},
		Log: pipeline.LogOptions{
			Log: func(level pipeline.LogLevel, message string) {
				switch level {
				case pipeline.LogFatal, pipeline.LogPanic:
					log.Fatal(message)
				case pipeline.LogError:
					log.Error(message)
				case pipeline.LogWarning:
					log.Warning(message)
				case pipeline.LogInfo, pipeline.LogDebug:
					log.Info(message)
				}
			},
			ShouldLog: func(level pipeline.LogLevel) bool {
				switch level {
				case pipeline.LogFatal, pipeline.LogPanic:
					return bool(log.V(3))
				case pipeline.LogError:
					return bool(log.V(3))
				case pipeline.LogWarning:
					return bool(log.V(2))
				case pipeline.LogInfo, pipeline.LogDebug:
					return bool(log.V(1))
				}
				return false
			},
		},
	})
	u := url.URL{
		Scheme: "https",
		Host:   credentials.AccountName() + ".blob.core.windows.net",
		Path:   "/",
	}
	return azblob.NewServiceURL(u, pipeline)
}

// AZBlobBackupHandle implements BackupHandle for Azure Blob service.
type AZBlobBackupHandle struct {
	bs        *AZBlobBackupStorage
	dir       string
	name      string
	readOnly  bool
	waitGroup sync.WaitGroup
	errors    concurrency.AllErrorRecorder
	ctx       context.Context
	cancel    context.CancelFunc
}

// Directory implements BackupHandle.
func (bh *AZBlobBackupHandle) Directory() string {
	return bh.dir
}

// Name implements BackupHandle.
func (bh *AZBlobBackupHandle) Name() string {
	return bh.name
}

// RecordError is part of the concurrency.ErrorRecorder interface.
func (bh *AZBlobBackupHandle) RecordError(err error) {
	bh.errors.RecordError(err)
}

// HasErrors is part of the concurrency.ErrorRecorder interface.
func (bh *AZBlobBackupHandle) HasErrors() bool {
	return bh.errors.HasErrors()
}

// Error is part of the concurrency.ErrorRecorder interface.
func (bh *AZBlobBackupHandle) Error() error {
	return bh.errors.Error()
}

// AddFile implements BackupHandle.
func (bh *AZBlobBackupHandle) AddFile(ctx context.Context, filename string, filesize int64) (io.WriteCloser, error) {
	if bh.readOnly {
		return nil, fmt.Errorf("AddFile cannot be called on read-only backup")
	}
	// Error out if the file size it too large ( ~4.75 TB)
	if filesize > azblob.BlockBlobMaxStageBlockBytes*azblob.BlockBlobMaxBlocks {
		return nil, fmt.Errorf("filesize (%v) is too large to upload to az blob (max size %v)", filesize, azblob.BlockBlobMaxStageBlockBytes*azblob.BlockBlobMaxBlocks)
	}

	obj := objName(bh.dir, bh.name, filename)
	containerURL, err := bh.bs.containerURL()
	if err != nil {
		return nil, err
	}

	blockBlobURL := containerURL.NewBlockBlobURL(obj)

	reader, writer := io.Pipe()
	bh.waitGroup.Add(1)

	go func() {
		defer bh.waitGroup.Done()
		_, err := azblob.UploadStreamToBlockBlob(bh.ctx, reader, blockBlobURL, azblob.UploadStreamToBlockBlobOptions{
			BufferSize: azblob.BlockBlobMaxStageBlockBytes,
			MaxBuffers: *azBlobParallelism,
		})
		if err != nil {
			reader.CloseWithError(err)
			bh.RecordError(err)
		}
	}()

	return writer, nil
}

// EndBackup implements BackupHandle.
func (bh *AZBlobBackupHandle) EndBackup(ctx context.Context) error {
	if bh.readOnly {
		return fmt.Errorf("EndBackup cannot be called on read-only backup")
	}
	bh.waitGroup.Wait()
	return bh.Error()
}

// AbortBackup implements BackupHandle.
func (bh *AZBlobBackupHandle) AbortBackup(ctx context.Context) error {
	if bh.readOnly {
		return fmt.Errorf("AbortBackup cannot be called on read-only backup")
	}
	// Cancel the context of any uploads.
	bh.cancel()

	// Remove the backup
	return bh.bs.RemoveBackup(ctx, bh.dir, bh.name)
}

// ReadFile implements BackupHandle.
func (bh *AZBlobBackupHandle) ReadFile(ctx context.Context, filename string) (io.ReadCloser, error) {
	if !bh.readOnly {
		return nil, fmt.Errorf("ReadFile cannot be called on read-write backup")
	}

	obj := objName(bh.dir, filename)
	containerURL, err := bh.bs.containerURL()
	if err != nil {
		return nil, err
	}
	blobURL := containerURL.NewBlobURL(obj)

	resp, err := blobURL.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false)
	if err != nil {
		return nil, err
	}
	return resp.Body(azblob.RetryReaderOptions{
		MaxRetryRequests: defaultRetryCount,
		NotifyFailedRead: func(failureCount int, lastError error, offset int64, count int64, willRetry bool) {
			log.Warningf("ReadFile: [azblob] container: %s, directory: %s, filename: %s, error: %v", *containerName, objName(bh.dir, ""), filename, lastError)
		},
		TreatEarlyCloseAsError: true,
	}), nil
}

// AZBlobBackupStorage structs implements the BackupStorage interface for AZBlob
type AZBlobBackupStorage struct {
}

func (bs *AZBlobBackupStorage) containerURL() (*azblob.ContainerURL, error) {
	credentials, err := azCredentials()
	if err != nil {
		return nil, err
	}
	u := azServiceURL(credentials).NewContainerURL(*containerName)
	return &u, nil
}

// ListBackups implements BackupStorage.
func (bs *AZBlobBackupStorage) ListBackups(ctx context.Context, dir string) ([]backupstorage.BackupHandle, error) {
	log.Infof("ListBackups: [azblob] container: %s, directory: %v", *containerName, objName(dir, ""))

	containerURL, err := bs.containerURL()
	if err != nil {
		return nil, err
	}

	searchPrefix := objName(dir, "")

	result := make([]backupstorage.BackupHandle, 0)
	var subdirs []string

	for marker := (azblob.Marker{}); marker.NotDone(); {
		// This returns Blobs in sorted order so we don't need to sort them a second time.
		resp, err := containerURL.ListBlobsHierarchySegment(ctx, marker, delimiter, azblob.ListBlobsSegmentOptions{
			Prefix:     searchPrefix,
			MaxResults: 0,
		})

		if err != nil {
			return nil, err
		}

		for _, item := range resp.Segment.BlobPrefixes {
			subdir := strings.TrimPrefix(item.Name, searchPrefix)
			subdir = strings.TrimSuffix(subdir, delimiter)
			subdirs = append(subdirs, subdir)
		}

		marker = resp.NextMarker
	}

	for _, subdir := range subdirs {
		cancelableCtx, cancel := context.WithCancel(ctx)
		result = append(result, &AZBlobBackupHandle{
			bs:       bs,
			dir:      strings.Join([]string{dir, subdir}, "/"),
			name:     subdir,
			readOnly: true,
			ctx:      cancelableCtx,
			cancel:   cancel,
		})
	}

	return result, nil
}

// StartBackup implements BackupStorage.
func (bs *AZBlobBackupStorage) StartBackup(ctx context.Context, dir, name string) (backupstorage.BackupHandle, error) {
	cancelableCtx, cancel := context.WithCancel(ctx)
	return &AZBlobBackupHandle{
		bs:       bs,
		dir:      dir,
		name:     name,
		readOnly: false,
		ctx:      cancelableCtx,
		cancel:   cancel,
	}, nil
}

// RemoveBackup implements BackupStorage.
func (bs *AZBlobBackupStorage) RemoveBackup(ctx context.Context, dir, name string) error {
	log.Infof("ListBackups: [azblob] container: %s, directory: %s", *containerName, objName(dir, ""))

	containerURL, err := bs.containerURL()
	if err != nil {
		return err
	}

	searchPrefix := objName(dir, name, "")

	for marker := (azblob.Marker{}); marker.NotDone(); {
		resp, err := containerURL.ListBlobsHierarchySegment(ctx, marker, delimiter, azblob.ListBlobsSegmentOptions{
			Prefix:     searchPrefix,
			MaxResults: 0,
		})

		if err != nil {
			return err
		}

		// Right now there is no batch delete so we must iterate over all the blobs to delete them one by one
		// One day we will be able to use this https://docs.microsoft.com/en-us/rest/api/storageservices/blob-batch
		// but currently it is listed as a preview and its not in the go API
		for _, item := range resp.Segment.BlobItems {
			_, err := containerURL.NewBlobURL(item.Name).Delete(ctx, azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{})
			if err != nil {
				return err
			}
		}
		marker = resp.NextMarker
	}

	// Delete the blob representing the folder of the backup, remove any trailing slash to signify we want to remove the folder
	// NOTE: you must set DeleteSnapshotsOptionNone or this will error out with a server side error
	for retry := 0; retry < defaultRetryCount; retry = retry + 1 {
		// Since the deletion of blob's is asyncronious we may need to wait a bit before we delete the folder
		// Also refresh the client just for good measure
		time.Sleep(10 * time.Second)
		containerURL, err = bs.containerURL()
		if err != nil {
			return err
		}

		log.Infof("Removing backup directory: %v", strings.TrimSuffix(searchPrefix, "/"))
		_, err = containerURL.NewBlobURL(strings.TrimSuffix(searchPrefix, "/")).Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
		if err == nil {
			break
		}
	}
	return err
}

// Close implements BackupStorage.
func (bs *AZBlobBackupStorage) Close() error {
	// This function is a No-op
	return nil
}

// objName joins path parts into an object name.
// Unlike path.Join, it doesn't collapse ".." or strip trailing slashes.
// It also adds the value of the -azblob_backup_storage_root flag if set.
func objName(parts ...string) string {
	if *storageRoot != "" {
		return *storageRoot + "/" + strings.Join(parts, "/")
	}
	return strings.Join(parts, "/")
}

func init() {
	backupstorage.BackupStorageMap["azblob"] = &AZBlobBackupStorage{}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/justinas/alice"
	"net/http"
	"time"
)

var middleware = alice.New(logger, auth)

func logger(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Println("### logger begin")
		t1 := time.Now()
		next.ServeHTTP(w, r)
		t2 := time.Since(t1)
		fmt.Println("### logger request duration", t2)
		fmt.Println("### logger end")
	})

}
<commit_msg>Update middleware.go<commit_after>package main

import (
	"fmt"
	"github.com/justinas/alice"
	"net/http"
	"time"
)

var middleware = alice.New(logger, auth)

func logger(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Println("### logger begin %v\n", r.URL)
		t1 := time.Now()
		next.ServeHTTP(w, r)
		t2 := time.Since(t1)
		fmt.Println("### logger request duration", t2)
		fmt.Println("### logger end")
	})

}
<|endoftext|>


<commit_before>package lib

import "encoding/asn1"

// OidDescription returns a human-readable name, a short acronym from RFC1485, a snake_case slug suitable as a json key,
// and a boolean describing whether multiple copies can appear on an X509 cert.
type OidDescription struct {
	Name     string
	Short    string
	Slug     string
	Multiple bool
}

func describeOid(oid asn1.ObjectIdentifier) OidDescription {
	raw := oid.String()
	// Multiple should be true for any types that are []string in x509.pkix.Name. When in doubt, set it to true.
	names := map[string]OidDescription{
		"2.5.4.3":                   {"CommonName", "CN", "common_name", false},
		"2.5.4.5":                   {"EV Incorporation Registration Number", "", "ev_registration_number", false},
		"2.5.4.6":                   {"Country", "C", "country", true},
		"2.5.4.7":                   {"Locality", "L", "locality", true},
		"2.5.4.8":                   {"Province", "ST", "province", true},
		"2.5.4.9":                   {"Street", "", "street", true},
		"2.5.4.10":                  {"Organization", "O", "organization", true},
		"2.5.4.11":                  {"Organizational Unit", "OU", "organizational_unit", true},
		"2.5.4.15":                  {"Business Category", "", "business_category", true},
		"2.5.4.17":                  {"Postal Code", "", "postalcode", true},
		"1.2.840.113549.1.9.1":      {"Email Address", "", "email_address", true},
		"1.3.6.1.4.1.311.60.2.1.1":  {"EV Incorporation Locality", "", "ev_locality", true},
		"1.3.6.1.4.1.311.60.2.1.2":  {"EV Incorporation Province", "", "ev_province", true},
		"1.3.6.1.4.1.311.60.2.1.3":  {"EV Incorporation Country", "", "ev_country", true},
		"0.9.2342.19200300.100.1.1": {"User ID", "UID", "user_id", true},
	}
	if description, ok := names[raw]; ok {
		return description
	}
	return OidDescription{raw, "", raw, true}
}

func oidShort(oid asn1.ObjectIdentifier) string {
	return describeOid(oid).Short
}

func oidName(oid asn1.ObjectIdentifier) string {
	return describeOid(oid).Name
}
<commit_msg>Add DomainComponent from RFC 2247<commit_after>package lib

import "encoding/asn1"

// OidDescription returns a human-readable name, a short acronym from RFC1485, a snake_case slug suitable as a json key,
// and a boolean describing whether multiple copies can appear on an X509 cert.
type OidDescription struct {
	Name     string
	Short    string
	Slug     string
	Multiple bool
}

func describeOid(oid asn1.ObjectIdentifier) OidDescription {
	raw := oid.String()
	// Multiple should be true for any types that are []string in x509.pkix.Name. When in doubt, set it to true.
	names := map[string]OidDescription{
		"2.5.4.3":                    {"CommonName", "CN", "common_name", false},
		"2.5.4.5":                    {"EV Incorporation Registration Number", "", "ev_registration_number", false},
		"2.5.4.6":                    {"Country", "C", "country", true},
		"2.5.4.7":                    {"Locality", "L", "locality", true},
		"2.5.4.8":                    {"Province", "ST", "province", true},
		"2.5.4.9":                    {"Street", "", "street", true},
		"2.5.4.10":                   {"Organization", "O", "organization", true},
		"2.5.4.11":                   {"Organizational Unit", "OU", "organizational_unit", true},
		"2.5.4.15":                   {"Business Category", "", "business_category", true},
		"2.5.4.17":                   {"Postal Code", "", "postalcode", true},
		"1.2.840.113549.1.9.1":       {"Email Address", "", "email_address", true},
		"1.3.6.1.4.1.311.60.2.1.1":   {"EV Incorporation Locality", "", "ev_locality", true},
		"1.3.6.1.4.1.311.60.2.1.2":   {"EV Incorporation Province", "", "ev_province", true},
		"1.3.6.1.4.1.311.60.2.1.3":   {"EV Incorporation Country", "", "ev_country", true},
		"0.9.2342.19200300.100.1.1":  {"User ID", "UID", "user_id", true},
		"0.9.2342.19200300.100.1.25": {"Domain Component", "DC", "domain_component", true},
	}
	if description, ok := names[raw]; ok {
		return description
	}
	return OidDescription{raw, "", raw, true}
}

func oidShort(oid asn1.ObjectIdentifier) string {
	return describeOid(oid).Short
}

func oidName(oid asn1.ObjectIdentifier) string {
	return describeOid(oid).Name
}
<|endoftext|>


<commit_before>package actor_test

import (
	"fmt"
	"time"

	"github.com/AsynkronIT/protoactor-go/actor"
)

type setBehaviorActor struct{}

// Receive is the default message handler when an actor is started
func (f *setBehaviorActor) Receive(context actor.Context) {
	if msg, ok := context.Message().(string); ok && msg == "other" {
		// Change actor's receive message handler to Other
		context.SetBehavior(f.Other)
	}
}

func (f *setBehaviorActor) Other(context actor.Context) {
	fmt.Println(context.Message())
}

// SetBehavior allows an actor to change its Receive handler, providing basic support for state machines
func ExampleContext_setBehavior() {
	pid := actor.Spawn(actor.FromInstance(&setBehaviorActor{}))
	defer pid.Stop()

	pid.Tell("other")
	pid.RequestFuture("hello from other", 10*time.Millisecond).Wait()

	// Output: hello from other
}
<commit_msg>explicit wait<commit_after>package actor_test

import (
	"fmt"
	"sync"

	"github.com/AsynkronIT/protoactor-go/actor"
)

type setBehaviorActor struct {
	sync.WaitGroup
}

// Receive is the default message handler when an actor is started
func (f *setBehaviorActor) Receive(context actor.Context) {
	if msg, ok := context.Message().(string); ok && msg == "other" {
		// Change actor's receive message handler to Other
		context.SetBehavior(f.Other)
	}
}

func (f *setBehaviorActor) Other(context actor.Context) {
	fmt.Println(context.Message())
	f.Done()
}

// SetBehavior allows an actor to change its Receive handler, providing basic support for state machines
func ExampleContext_setBehavior() {
	a := &setBehaviorActor{}
	a.Add(1)
	pid := actor.Spawn(actor.FromInstance(a))
	defer pid.Stop()

	pid.Tell("other")
	pid.Tell("hello from other")
	a.Wait()

	// Output: hello from other
}
<|endoftext|>