<commit_before>/*
PConns are warpers on top of lower level network apis and complementing
algorithms (future: encoding, encryption, compression)

The primary purpose is to create a networking api for handling connections
between two end points, first based on TCP, that is easy to use to send data
(mostly in the Protocol Buffers format), that is also upgradable in the future
taking advantages of encryption and UDP without large api changes.

The preservation of message boundaries in PConns have several advantages vs streaming:
- Protocol Buffers does not self terminate, so that where one message end and the next start needs to be added before written to a stream such as TCP.
- Exposing message boundaries to encryption algorithms allows protection against truncation attacks. A Message is either processed in full, or not at all. To guarantee that a message is processed, an application level reply is still necessary.
- Easy of switching to a lossy transport such as UDP.

Features to manage multiple connections: a resource pool, will be somewhere else.
Features to create compatible connections with existing internet services will
probably not use this api.
*/
package pconn

import (
	"io"
)

//PConn is a generic network connection that preserves message boundaries, so
//that users don't need to worry about separating data in a stream.
//One Send function call will be matched by one Receive on the other end.
//
//Use SendBytes and ReceiveBytes for a []byte interface
//
//Multiple goroutines may NOT invoke methods on a Conn simultaneously, unless supported.
type PConn interface {

	//Sender sends a message written to the returned writer. The message must end
	//by Close, else nothing should happen on the other end.
	//Sender's Write and Close may block when network buffer is full.
	//
	//Unless supported, sender can only be called again after the last sender is closed.
	//Sender may rereturn a old writer, previously closed, to send a new message.
	//
	//Error Condtitions: those from net.Conn Write.
	//A good default error handling strategy is to close the connection (not the sender).
	Sender() io.WriteCloser

	//Receive receives a message as a reader. read EOF is the end of message.
	//Receive may block, or return reader like a future.
	//
	//Unless supported, Receive should not be called again untill read report EOF. ReceiveInWriter
	//and ReceiveBytes does this for you by blocking.
	//
	//All Errors are reported by read. Any error, other then EOF means all actions
	//taken by reader must be reverted for security purposes.
	//Error Conditions: those from net.Conn Read, or data corruption detected by
	//this PConn.
	//A good default error handling strategy is to close the connection.
	Receiver() io.Reader

	//MaxMsgLength is the max length of a msg.
	//
	//Checked by the sender to crash on design errors, when violated a panic is
	//throw. Large message need to be breaked into smaller messages.
	//
	//Checked by the receiver to prevent large resource claims. Receive will error.
	//
	//Can be used by a PConn wapper (TODO) to know what size to splite large messages.
	MaxMsgLength() int

	//Close closes the connection. see net.Conn Close()
	Close() error
}

//SendBytes sends a message using Sender
func SendBytes(p PConn, msg []byte) error {
	s := p.Sender()
	_, err := s.Write(msg)
	if err != nil {
		return err
	}
	return s.Close()
}

//ReceiveInWriter receives a full message and have decoder process it.
//Receive blocks untill decoder have finnshed processing the data unless
//error is not nil.
func ReceiveInWriter(p PConn, decoder io.Writer) error {
	r := p.Receiver()
	_, err := io.Copy(decoder, r)
	return err
}

//ReceiveBytes get bytes from the Receive function of PConn
//Returns all bytes of a message will nil error, or nil message with some error
func ReceiveBytes(p PConn) ([]byte, error) {
	b := &bytesDecoder{}
	err := ReceiveInWriter(p, b)
	if err != nil {
		return nil, err
	}
	return b.buf, nil
}

type bytesDecoder struct {
	buf []byte
}

func (b *bytesDecoder) Write(p []byte) (int, error) {
	b.buf = append(b.buf, p...)
	return len(p), nil
}
<commit_msg>comments<commit_after>/*
PConns are warpers on top of lower level network apis and complementing
algorithms (future: encoding, encryption, compression)

The primary purpose is to create a networking api for handling connections
between two end points, first based on TCP, that is easy to use to send data
(mostly in the Protocol Buffers format), that is also upgradable in the future
taking advantages of encryption and UDP without large api changes.

The preservation of message boundaries in PConns have several advantages vs streaming:

- Protocol Buffers does not self terminate, so that where one message end and the next start needs to be added before written to a stream such as TCP.

- Exposing message boundaries to encryption algorithms allows protection against truncation attacks. A Message is either processed in full, or not at all. To guarantee that a message is processed, an application level reply is still necessary.

- Ease of switching to a lossy transport such as UDP.

Features to manage multiple connections: a resource pool, will be somewhere else.
Features to create compatible connections with existing internet services will
probably not use this api.
*/
package pconn

import (
	"io"
)

//PConn is a generic network connection that preserves message boundaries, so
//that users don't need to worry about separating data in a stream.
//One Send function call will be matched by one Receive on the other end.
//
//Use SendBytes and ReceiveBytes for a []byte interface
//
//Multiple goroutines may NOT invoke methods on a Conn simultaneously, unless supported.
type PConn interface {

	//Sender sends a message written to the returned writer. The message must end
	//by Close, else nothing should happen on the other end.
	//Sender's Write and Close may block when network buffer is full.
	//
	//Unless supported, sender can only be called again after the last sender is closed.
	//Sender may re-return an old writer, previously closed, to send a new message.
	//
	//Error Condtitions: those from net.Conn Write.
	//A good default error handling strategy is to close the connection (not the sender).
	Sender() io.WriteCloser

	//Receive receives a message as a reader. read EOF is the end of message.
	//Receive may block, or return reader like a future.
	//
	//Unless supported, Receive should not be called again untill read report EOF. ReceiveInWriter
	//and ReceiveBytes does this for you by blocking.
	//
	//All Errors are reported by read. Any error, other then EOF means all actions
	//taken by reader must be reverted for security purposes.
	//Error Conditions: those from net.Conn Read, or data corruption detected by
	//this PConn.
	//A good default error handling strategy is to close the connection.
	Receiver() io.Reader

	//MaxMsgLength is the max length of a msg.
	//
	//Checked by the sender to crash on design errors, when violated a panic is
	//throw. Large message need to be breaked into smaller messages.
	//
	//Checked by the receiver to prevent large resource claims. Receive will error.
	//
	//Can be used by a PConn wapper (TODO) to know what size to splite large messages.
	MaxMsgLength() int

	//Close closes the connection. see net.Conn Close()
	Close() error
}

//SendBytes sends a message using Sender
func SendBytes(p PConn, msg []byte) error {
	s := p.Sender()
	_, err := s.Write(msg)
	if err != nil {
		return err
	}
	return s.Close()
}

//ReceiveInWriter receives a full message and have decoder process it.
//
//Receive blocks untill decoder have finnshed processing the data unless
//error is not nil.
func ReceiveInWriter(p PConn, decoder io.Writer) error {
	r := p.Receiver()
	_, err := io.Copy(decoder, r)
	return err
}

//ReceiveBytes get bytes from the Receive function of PConn.
//
//It returns all bytes of a message will nil error, or nil message with some error
func ReceiveBytes(p PConn) ([]byte, error) {
	b := &bytesDecoder{}
	err := ReceiveInWriter(p, b)
	if err != nil {
		return nil, err
	}
	return b.buf, nil
}

type bytesDecoder struct {
	buf []byte
}

func (b *bytesDecoder) Write(p []byte) (int, error) {
	b.buf = append(b.buf, p...)
	return len(p), nil
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"

	log "github.com/Sirupsen/logrus"
	"github.com/gorilla/mux"
	"github.com/syhlion/gwspack"
)

func WSConnect(w http.ResponseWriter, r *http.Request) {

	log.Info(r.RemoteAddr, " handshake start")
	params := mux.Vars(r)
	app_key := params["app_key"]
	user_tag := params["user_tag"]
	if app_key == "" || user_tag == "" {
		log.Warn(r.RemoteAddr, " app_key & user_tag empty")
		http.Error(w, "app_key || user_tag empty", 404)
		return
	}
	app := gwspack.Get(app_key)
	c, err := app.Register(user_tag, w, r, nil)
	if err != nil {
		log.Warn(r.RemoteAddr, " ", err)
		http.Error(w, err.Error(), 403)
		return
	}

	log.Info(r.RemoteAddr, " login ", app_key, " Scuess")
	c.Listen()
	defer log.Info(r.RemoteAddr, " ", user_tag, " logout", " ", app_key)
	return
}
<commit_msg>fix mulitheader write<commit_after>package main

import (
	"net/http"

	log "github.com/Sirupsen/logrus"
	"github.com/gorilla/mux"
	"github.com/syhlion/gwspack"
)

func WSConnect(w http.ResponseWriter, r *http.Request) {

	log.Info(r.RemoteAddr, " handshake start")
	params := mux.Vars(r)
	app_key := params["app_key"]
	user_tag := params["user_tag"]
	if app_key == "" || user_tag == "" {
		log.Warn(r.RemoteAddr, " app_key & user_tag empty")
		http.Error(w, "app_key || user_tag empty", 404)
		return
	}
	app := gwspack.Get(app_key)
	c, err := app.Register(user_tag, w, r, nil)
	if err != nil {
		log.Warn(r.RemoteAddr, " ", err)
		return
	}

	log.Info(r.RemoteAddr, " login ", app_key, " Scuess")
	c.Listen()
	defer log.Info(r.RemoteAddr, " ", user_tag, " logout", " ", app_key)
	return
}
<|endoftext|>


<commit_before>package google

import (
	"fmt"
	"time"

	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
	"google.golang.org/api/cloudresourcemanager/v1"
)

func resourceGoogleProjectOrganizationPolicy() *schema.Resource {
	return &schema.Resource{
		Create: resourceGoogleProjectOrganizationPolicyCreate,
		Read:   resourceGoogleProjectOrganizationPolicyRead,
		Update: resourceGoogleProjectOrganizationPolicyUpdate,
		Delete: resourceGoogleProjectOrganizationPolicyDelete,

		Importer: &schema.ResourceImporter{
			State: resourceProjectOrgPolicyImporter,
		},

		Timeouts: &schema.ResourceTimeout{
			Create: schema.DefaultTimeout(4 * time.Minute),
			Update: schema.DefaultTimeout(4 * time.Minute),
			Read:   schema.DefaultTimeout(4 * time.Minute),
			Delete: schema.DefaultTimeout(4 * time.Minute),
		},

		Schema: mergeSchemas(
			schemaOrganizationPolicy,
			map[string]*schema.Schema{
				"project": {
					Type:        schema.TypeString,
					Required:    true,
					ForceNew:    true,
					Description: `The project ID.`,
				},
			},
		),
		UseJSONNumber: true,
	}
}

func resourceProjectOrgPolicyImporter(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) {
	config := meta.(*Config)

	if err := parseImportId([]string{
		"projects/(?P<project>[^/]+):constraints/(?P<constraint>[^/]+)",
		"(?P<project>[^/]+):constraints/(?P<constraint>[^/]+)",
		"(?P<project>[^/]+):(?P<constraint>[^/]+)"},
		d, config); err != nil {
		return nil, err
	}

	if d.Get("project") == "" || d.Get("constraint") == "" {
		return nil, fmt.Errorf("unable to parse project or constraint. Check import formats")
	}

	return []*schema.ResourceData{d}, nil
}

func resourceGoogleProjectOrganizationPolicyCreate(d *schema.ResourceData, meta interface{}) error {
	d.SetId(fmt.Sprintf("%s:%s", d.Get("project"), d.Get("constraint")))

	if isOrganizationPolicyUnset(d) {
		return resourceGoogleProjectOrganizationPolicyDelete(d, meta)
	}

	if err := setProjectOrganizationPolicy(d, meta); err != nil {
		return err
	}

	return resourceGoogleProjectOrganizationPolicyRead(d, meta)
}

func resourceGoogleProjectOrganizationPolicyRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	userAgent, err := generateUserAgentString(d, config.userAgent)
	if err != nil {
		return err
	}
	project := prefixedProject(d.Get("project").(string))

	var policy *cloudresourcemanager.OrgPolicy
	err = retryTimeDuration(func() (readErr error) {
		policy, readErr = config.NewResourceManagerClient(userAgent).Projects.GetOrgPolicy(project, &cloudresourcemanager.GetOrgPolicyRequest{
			Constraint: canonicalOrgPolicyConstraint(d.Get("constraint").(string)),
		}).Do()
		return readErr
	}, d.Timeout(schema.TimeoutRead))
	if err != nil {
		return handleNotFoundError(err, d, fmt.Sprintf("Organization policy for %s", project))
	}

	if err := d.Set("constraint", policy.Constraint); err != nil {
		return fmt.Errorf("Error setting constraint: %s", err)
	}
	if err := d.Set("boolean_policy", flattenBooleanOrganizationPolicy(policy.BooleanPolicy)); err != nil {
		return fmt.Errorf("Error setting boolean_policy: %s", err)
	}
	if err := d.Set("list_policy", flattenListOrganizationPolicy(policy.ListPolicy)); err != nil {
		return fmt.Errorf("Error setting list_policy: %s", err)
	}
	if err := d.Set("restore_policy", flattenRestoreOrganizationPolicy(policy.RestoreDefault)); err != nil {
		return fmt.Errorf("Error setting restore_policy: %s", err)
	}
	if err := d.Set("version", policy.Version); err != nil {
		return fmt.Errorf("Error setting version: %s", err)
	}
	if err := d.Set("etag", policy.Etag); err != nil {
		return fmt.Errorf("Error setting etag: %s", err)
	}
	if err := d.Set("update_time", policy.UpdateTime); err != nil {
		return fmt.Errorf("Error setting update_time: %s", err)
	}

	return nil
}

func resourceGoogleProjectOrganizationPolicyUpdate(d *schema.ResourceData, meta interface{}) error {
	if isOrganizationPolicyUnset(d) {
		return resourceGoogleProjectOrganizationPolicyDelete(d, meta)
	}

	if err := setProjectOrganizationPolicy(d, meta); err != nil {
		return err
	}

	return resourceGoogleProjectOrganizationPolicyRead(d, meta)
}

func resourceGoogleProjectOrganizationPolicyDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	userAgent, err := generateUserAgentString(d, config.userAgent)
	if err != nil {
		return err
	}
	project := prefixedProject(d.Get("project").(string))

	return retryTimeDuration(func() error {
		_, err := config.NewResourceManagerClient(userAgent).Projects.ClearOrgPolicy(project, &cloudresourcemanager.ClearOrgPolicyRequest{
			Constraint: canonicalOrgPolicyConstraint(d.Get("constraint").(string)),
		}).Do()
		return err
	}, d.Timeout(schema.TimeoutDelete))
}

func setProjectOrganizationPolicy(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	userAgent, err := generateUserAgentString(d, config.userAgent)
	if err != nil {
		return err
	}

	project := prefixedProject(d.Get("project").(string))

	listPolicy, err := expandListOrganizationPolicy(d.Get("list_policy").([]interface{}))
	if err != nil {
		return err
	}

	restore_default, err := expandRestoreOrganizationPolicy(d.Get("restore_policy").([]interface{}))
	if err != nil {
		return err
	}

	return retryTimeDuration(func() error {
		_, err := config.NewResourceManagerClient(userAgent).Projects.SetOrgPolicy(project, &cloudresourcemanager.SetOrgPolicyRequest{
			Policy: &cloudresourcemanager.OrgPolicy{
				Constraint:     canonicalOrgPolicyConstraint(d.Get("constraint").(string)),
				BooleanPolicy:  expandBooleanOrganizationPolicy(d.Get("boolean_policy").([]interface{})),
				ListPolicy:     listPolicy,
				RestoreDefault: restore_default,
				Version:        int64(d.Get("version").(int)),
				Etag:           d.Get("etag").(string),
			},
		}).Do()
		return err
	}, d.Timeout(schema.TimeoutCreate))
}
<commit_msg>Fix writing to state when creation failed on `google_project_organization_policy` (#5958)<commit_after>package google

import (
	"fmt"
	"time"

	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
	"google.golang.org/api/cloudresourcemanager/v1"
)

func resourceGoogleProjectOrganizationPolicy() *schema.Resource {
	return &schema.Resource{
		Create: resourceGoogleProjectOrganizationPolicyCreate,
		Read:   resourceGoogleProjectOrganizationPolicyRead,
		Update: resourceGoogleProjectOrganizationPolicyUpdate,
		Delete: resourceGoogleProjectOrganizationPolicyDelete,

		Importer: &schema.ResourceImporter{
			State: resourceProjectOrgPolicyImporter,
		},

		Timeouts: &schema.ResourceTimeout{
			Create: schema.DefaultTimeout(4 * time.Minute),
			Update: schema.DefaultTimeout(4 * time.Minute),
			Read:   schema.DefaultTimeout(4 * time.Minute),
			Delete: schema.DefaultTimeout(4 * time.Minute),
		},

		Schema: mergeSchemas(
			schemaOrganizationPolicy,
			map[string]*schema.Schema{
				"project": {
					Type:        schema.TypeString,
					Required:    true,
					ForceNew:    true,
					Description: `The project ID.`,
				},
			},
		),
		UseJSONNumber: true,
	}
}

func resourceProjectOrgPolicyImporter(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) {
	config := meta.(*Config)

	if err := parseImportId([]string{
		"projects/(?P<project>[^/]+):constraints/(?P<constraint>[^/]+)",
		"(?P<project>[^/]+):constraints/(?P<constraint>[^/]+)",
		"(?P<project>[^/]+):(?P<constraint>[^/]+)"},
		d, config); err != nil {
		return nil, err
	}

	if d.Get("project") == "" || d.Get("constraint") == "" {
		return nil, fmt.Errorf("unable to parse project or constraint. Check import formats")
	}

	return []*schema.ResourceData{d}, nil
}

func resourceGoogleProjectOrganizationPolicyCreate(d *schema.ResourceData, meta interface{}) error {
	if isOrganizationPolicyUnset(d) {
		d.SetId(fmt.Sprintf("%s:%s", d.Get("project"), d.Get("constraint")))
		return resourceGoogleProjectOrganizationPolicyDelete(d, meta)
	}

	if err := setProjectOrganizationPolicy(d, meta); err != nil {
		return err
	}

	d.SetId(fmt.Sprintf("%s:%s", d.Get("project"), d.Get("constraint")))

	return resourceGoogleProjectOrganizationPolicyRead(d, meta)
}

func resourceGoogleProjectOrganizationPolicyRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	userAgent, err := generateUserAgentString(d, config.userAgent)
	if err != nil {
		return err
	}
	project := prefixedProject(d.Get("project").(string))

	var policy *cloudresourcemanager.OrgPolicy
	err = retryTimeDuration(func() (readErr error) {
		policy, readErr = config.NewResourceManagerClient(userAgent).Projects.GetOrgPolicy(project, &cloudresourcemanager.GetOrgPolicyRequest{
			Constraint: canonicalOrgPolicyConstraint(d.Get("constraint").(string)),
		}).Do()
		return readErr
	}, d.Timeout(schema.TimeoutRead))
	if err != nil {
		return handleNotFoundError(err, d, fmt.Sprintf("Organization policy for %s", project))
	}

	if err := d.Set("constraint", policy.Constraint); err != nil {
		return fmt.Errorf("Error setting constraint: %s", err)
	}
	if err := d.Set("boolean_policy", flattenBooleanOrganizationPolicy(policy.BooleanPolicy)); err != nil {
		return fmt.Errorf("Error setting boolean_policy: %s", err)
	}
	if err := d.Set("list_policy", flattenListOrganizationPolicy(policy.ListPolicy)); err != nil {
		return fmt.Errorf("Error setting list_policy: %s", err)
	}
	if err := d.Set("restore_policy", flattenRestoreOrganizationPolicy(policy.RestoreDefault)); err != nil {
		return fmt.Errorf("Error setting restore_policy: %s", err)
	}
	if err := d.Set("version", policy.Version); err != nil {
		return fmt.Errorf("Error setting version: %s", err)
	}
	if err := d.Set("etag", policy.Etag); err != nil {
		return fmt.Errorf("Error setting etag: %s", err)
	}
	if err := d.Set("update_time", policy.UpdateTime); err != nil {
		return fmt.Errorf("Error setting update_time: %s", err)
	}

	return nil
}

func resourceGoogleProjectOrganizationPolicyUpdate(d *schema.ResourceData, meta interface{}) error {
	if isOrganizationPolicyUnset(d) {
		return resourceGoogleProjectOrganizationPolicyDelete(d, meta)
	}

	if err := setProjectOrganizationPolicy(d, meta); err != nil {
		return err
	}

	return resourceGoogleProjectOrganizationPolicyRead(d, meta)
}

func resourceGoogleProjectOrganizationPolicyDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	userAgent, err := generateUserAgentString(d, config.userAgent)
	if err != nil {
		return err
	}
	project := prefixedProject(d.Get("project").(string))

	return retryTimeDuration(func() error {
		_, err := config.NewResourceManagerClient(userAgent).Projects.ClearOrgPolicy(project, &cloudresourcemanager.ClearOrgPolicyRequest{
			Constraint: canonicalOrgPolicyConstraint(d.Get("constraint").(string)),
		}).Do()
		return err
	}, d.Timeout(schema.TimeoutDelete))
}

func setProjectOrganizationPolicy(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	userAgent, err := generateUserAgentString(d, config.userAgent)
	if err != nil {
		return err
	}

	project := prefixedProject(d.Get("project").(string))

	listPolicy, err := expandListOrganizationPolicy(d.Get("list_policy").([]interface{}))
	if err != nil {
		return err
	}

	restore_default, err := expandRestoreOrganizationPolicy(d.Get("restore_policy").([]interface{}))
	if err != nil {
		return err
	}

	return retryTimeDuration(func() error {
		_, err := config.NewResourceManagerClient(userAgent).Projects.SetOrgPolicy(project, &cloudresourcemanager.SetOrgPolicyRequest{
			Policy: &cloudresourcemanager.OrgPolicy{
				Constraint:     canonicalOrgPolicyConstraint(d.Get("constraint").(string)),
				BooleanPolicy:  expandBooleanOrganizationPolicy(d.Get("boolean_policy").([]interface{})),
				ListPolicy:     listPolicy,
				RestoreDefault: restore_default,
				Version:        int64(d.Get("version").(int)),
				Etag:           d.Get("etag").(string),
			},
		}).Do()
		return err
	}, d.Timeout(schema.TimeoutCreate))
}
<|endoftext|>


<commit_before>package drivermanager

import (
	"context"
	"fmt"
	"sync"

	log "github.com/hashicorp/go-hclog"
	plugin "github.com/hashicorp/go-plugin"
	"github.com/hashicorp/nomad/client/pluginmanager"
	"github.com/hashicorp/nomad/client/pluginmanager/drivermanager/state"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/plugins/base"
	"github.com/hashicorp/nomad/plugins/drivers"
	"github.com/hashicorp/nomad/plugins/shared"
	"github.com/hashicorp/nomad/plugins/shared/loader"
)

// ErrDriverNotFound is returned during Dispense when the requested driver
// plugin is not found in the plugin catalog
var ErrDriverNotFound = fmt.Errorf("driver not found")

// Manager is the interface used to manage driver plugins
type Manager interface {
	pluginmanager.PluginManager

	// Dispense returns a drivers.DriverPlugin for the given driver plugin name
	// handling reattaching to an existing driver if available
	Dispense(driver string) (drivers.DriverPlugin, error)
}

// EventHandler is a callback to be called for a task.
// The handler should not block execution.
type EventHandler func(*drivers.TaskEvent)

// TaskEventHandlerFactory returns an event handler for a given allocID/task name
type TaskEventHandlerFactory func(allocID, taskName string) EventHandler

// StateStorage is used to persist the driver managers state across
// agent restarts.
type StateStorage interface {
	// GetDevicePluginState is used to retrieve the device manager's plugin
	// state.
	GetDriverPluginState() (*state.PluginState, error)

	// PutDevicePluginState is used to store the device manager's plugin
	// state.
	PutDriverPluginState(state *state.PluginState) error
}

// UpdateNodeDriverInfoFn is the callback used to update the node from
// fingerprinting
type UpdateNodeDriverInfoFn func(string, *structs.DriverInfo)

// StorePluginReattachFn is used to store plugin reattachment configurations.
type StorePluginReattachFn func(*plugin.ReattachConfig) error

// FetchPluginReattachFn is used to retrieve the stored plugin reattachment
// configuration.
type FetchPluginReattachFn func() (*plugin.ReattachConfig, bool)

// Config is used to configure a driver manager
type Config struct {
	// Logger is the logger used by the device manager
	Logger log.Logger

	// Loader is the plugin loader
	Loader loader.PluginCatalog

	// PluginConfig is the config passed to the launched plugins
	PluginConfig *base.AgentConfig

	// Updater is used to update the node when driver information changes
	Updater UpdateNodeDriverInfoFn

	// EventHandlerFactory is used to retrieve a task event handler
	EventHandlerFactory TaskEventHandlerFactory

	// State is used to manage the device managers state
	State StateStorage

	// AllowedDrivers if set will only start driver plugins for the given
	// drivers
	AllowedDrivers map[string]struct{}

	// BlockedDrivers if set will not allow the given driver plugins to start
	BlockedDrivers map[string]struct{}
}

// manager is used to manage a set of driver plugins
type manager struct {
	// logger is the logger used by the device manager
	logger log.Logger

	// state is used to manage the device managers state
	state StateStorage

	// ctx is used to shutdown the device manager
	ctx    context.Context
	cancel context.CancelFunc

	// loader is the plugin loader
	loader loader.PluginCatalog

	// pluginConfig is the config passed to the launched plugins
	pluginConfig *base.AgentConfig

	// updater is used to update the node when device information changes
	updater UpdateNodeDriverInfoFn

	// eventHandlerFactory is passed to the instance managers and used to forward
	// task events
	eventHandlerFactory TaskEventHandlerFactory

	// instances is the list of managed devices, access is serialized by instanceMu
	instances   map[string]*instanceManager
	instancesMu sync.RWMutex

	// reattachConfigs stores the plugin reattach configs
	reattachConfigs    map[loader.PluginID]*shared.ReattachConfig
	reattachConfigLock sync.Mutex

	// allows/block lists
	allowedDrivers map[string]struct{}
	blockedDrivers map[string]struct{}

	// readyCh is ticked once at the end of Run()
	readyCh chan struct{}
}

// New returns a new driver manager
func New(c *Config) *manager {
	ctx, cancel := context.WithCancel(context.Background())
	return &manager{
		logger:              c.Logger.Named("driver_mgr"),
		state:               c.State,
		ctx:                 ctx,
		cancel:              cancel,
		loader:              c.Loader,
		pluginConfig:        c.PluginConfig,
		updater:             c.Updater,
		eventHandlerFactory: c.EventHandlerFactory,
		instances:           make(map[string]*instanceManager),
		reattachConfigs:     make(map[loader.PluginID]*shared.ReattachConfig),
		allowedDrivers:      c.AllowedDrivers,
		blockedDrivers:      c.BlockedDrivers,
		readyCh:             make(chan struct{}),
	}
}

// PluginType returns the type of plugin this mananger mananges
func (*manager) PluginType() string { return base.PluginTypeDriver }

// Run starts the mananger, initializes driver plugins and blocks until Shutdown
// is called.
func (m *manager) Run() {
	// Load any previous plugin reattach configuration
	if err := m.loadReattachConfigs(); err != nil {
		m.logger.Warn("unable to load driver plugin reattach configs, a driver process may have been leaked",
			"error", err)
	}

	// Get driver plugins
	driversPlugins := m.loader.Catalog()[base.PluginTypeDriver]
	if len(driversPlugins) == 0 {
		m.logger.Debug("exiting since there are no driver plugins")
		m.cancel()
		return
	}

	var skippedDrivers []string
	for _, d := range driversPlugins {
		id := loader.PluginInfoID(d)
		if m.isDriverBlocked(id.Name) {
			skippedDrivers = append(skippedDrivers, id.Name)
			continue
		}

		storeFn := func(c *plugin.ReattachConfig) error {
			return m.storePluginReattachConfig(id, c)
		}
		fetchFn := func() (*plugin.ReattachConfig, bool) {
			return m.fetchPluginReattachConfig(id)
		}

		instance := newInstanceManager(&instanceManagerConfig{
			Logger:               m.logger,
			Ctx:                  m.ctx,
			Loader:               m.loader,
			StoreReattach:        storeFn,
			FetchReattach:        fetchFn,
			PluginConfig:         m.pluginConfig,
			ID:                   &id,
			UpdateNodeFromDriver: m.updater,
			EventHandlerFactory:  m.eventHandlerFactory,
		})

		m.instancesMu.Lock()
		m.instances[id.Name] = instance
		m.instancesMu.Unlock()
	}

	if len(skippedDrivers) > 0 {
		m.logger.Debug("drivers skipped due to allow/block list", "skipped_drivers", skippedDrivers)
	}

	// signal ready
	close(m.readyCh)
}

// Shutdown cleans up all the plugins
func (m *manager) Shutdown() {
	// Cancel the context to stop any requests
	m.cancel()

	m.instancesMu.RLock()
	defer m.instancesMu.RUnlock()

	// Go through and shut everything down
	for _, i := range m.instances {
		i.cleanup()
	}
}

func (m *manager) WaitForFirstFingerprint(ctx context.Context) <-chan struct{} {
	ctx, cancel := context.WithCancel(ctx)
	go m.waitForFirstFingerprint(ctx, cancel)
	return ctx.Done()
}

func (m *manager) waitForFirstFingerprint(ctx context.Context, cancel context.CancelFunc) {
	defer cancel()
	// We don't want to start initial fingerprint wait until Run loop has
	// finished
	select {
	case <-m.readyCh:
	case <-ctx.Done():
		// parent context canceled or timedout
		return
	case <-m.ctx.Done():
		// shutdown called
		return
	}

	var availDrivers []string
	var wg sync.WaitGroup

	// loop through instances and wait for each to finish initial fingerprint
	m.instancesMu.RLock()
	for n, i := range m.instances {
		wg.Add(1)
		go func(name string, instance *instanceManager) {
			defer wg.Done()
			instance.WaitForFirstFingerprint(ctx)
			if instance.getLastHealth() != drivers.HealthStateUndetected {
				availDrivers = append(availDrivers, name)
			}
		}(n, i)
	}
	m.instancesMu.RUnlock()
	wg.Wait()
	m.logger.Debug("detected drivers", "drivers", availDrivers)
}

func (m *manager) loadReattachConfigs() error {
	m.reattachConfigLock.Lock()
	defer m.reattachConfigLock.Unlock()

	s, err := m.state.GetDriverPluginState()
	if err != nil {
		return err
	}

	if s != nil {
		for name, c := range s.ReattachConfigs {
			if m.isDriverBlocked(name) {
				m.logger.Warn("reattach config for driver plugin found but driver is blocked due to allow/block list, killing plugin",
					"driver", name)
				m.shutdownBlockedDriver(name, c)
				continue
			}

			id := loader.PluginID{
				PluginType: base.PluginTypeDriver,
				Name:       name,
			}

			m.reattachConfigs[id] = c
		}
	}
	return nil
}

// shutdownBlockedDriver is used to forcefully shutdown a running driver plugin
// when it has been blocked due to allow/block lists
func (m *manager) shutdownBlockedDriver(name string, reattach *shared.ReattachConfig) {
	c, err := shared.ReattachConfigToGoPlugin(reattach)
	if err != nil {
		m.logger.Warn("failed to reattach and kill blocked driver plugin",
			"driver", name, "error", err)
		return

	}
	pluginInstance, err := m.loader.Reattach(name, base.PluginTypeDriver, c)
	if err != nil {
		m.logger.Warn("failed to reattach and kill blocked driver plugin",
			"driver", name, "error", err)
		return
	}

	if !pluginInstance.Exited() {
		pluginInstance.Kill()
	}
}

// storePluginReattachConfig is used as a callback to the instance managers and
// persists thhe plugin reattach configurations.
func (m *manager) storePluginReattachConfig(id loader.PluginID, c *plugin.ReattachConfig) error {
	m.reattachConfigLock.Lock()
	defer m.reattachConfigLock.Unlock()

	// Store the new reattach config
	m.reattachConfigs[id] = shared.ReattachConfigFromGoPlugin(c)

	// Persist the state
	s := &state.PluginState{
		ReattachConfigs: make(map[string]*shared.ReattachConfig, len(m.reattachConfigs)),
	}

	for id, c := range m.reattachConfigs {
		s.ReattachConfigs[id.Name] = c
	}

	return m.state.PutDriverPluginState(s)
}

// fetchPluginReattachConfig is used as a callback to the instance managers and
// retrieves the plugin reattach config. If it has not been stored it will
// return nil
func (m *manager) fetchPluginReattachConfig(id loader.PluginID) (*plugin.ReattachConfig, bool) {
	m.reattachConfigLock.Lock()
	defer m.reattachConfigLock.Unlock()

	if cfg, ok := m.reattachConfigs[id]; ok {
		c, err := shared.ReattachConfigToGoPlugin(cfg)
		if err != nil {
			m.logger.Warn("failed to read plugin reattach config", "config", cfg, "error", err)
			delete(m.reattachConfigs, id)
			return nil, false
		}
		return c, true
	}
	return nil, false
}

func (m *manager) Dispense(d string) (drivers.DriverPlugin, error) {
	m.instancesMu.RLock()
	defer m.instancesMu.RUnlock()
	if instance, ok := m.instances[d]; ok {
		return instance.dispense()
	}

	return nil, ErrDriverNotFound
}

func (m *manager) isDriverBlocked(name string) bool {
	// Block drivers that are not in the allowed list if it is set.
	if _, ok := m.allowedDrivers[name]; len(m.allowedDrivers) > 0 && !ok {
		return true
	}

	// Block drivers that are in the blocked list
	if _, ok := m.blockedDrivers[name]; ok {
		return true
	}
	return false
}
<commit_msg>drivermgr: fix race in building driver list<commit_after>package drivermanager

import (
	"context"
	"fmt"
	"sync"

	log "github.com/hashicorp/go-hclog"
	plugin "github.com/hashicorp/go-plugin"
	"github.com/hashicorp/nomad/client/pluginmanager"
	"github.com/hashicorp/nomad/client/pluginmanager/drivermanager/state"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/plugins/base"
	"github.com/hashicorp/nomad/plugins/drivers"
	"github.com/hashicorp/nomad/plugins/shared"
	"github.com/hashicorp/nomad/plugins/shared/loader"
)

// ErrDriverNotFound is returned during Dispense when the requested driver
// plugin is not found in the plugin catalog
var ErrDriverNotFound = fmt.Errorf("driver not found")

// Manager is the interface used to manage driver plugins
type Manager interface {
	pluginmanager.PluginManager

	// Dispense returns a drivers.DriverPlugin for the given driver plugin name
	// handling reattaching to an existing driver if available
	Dispense(driver string) (drivers.DriverPlugin, error)
}

// EventHandler is a callback to be called for a task.
// The handler should not block execution.
type EventHandler func(*drivers.TaskEvent)

// TaskEventHandlerFactory returns an event handler for a given allocID/task name
type TaskEventHandlerFactory func(allocID, taskName string) EventHandler

// StateStorage is used to persist the driver managers state across
// agent restarts.
type StateStorage interface {
	// GetDevicePluginState is used to retrieve the device manager's plugin
	// state.
	GetDriverPluginState() (*state.PluginState, error)

	// PutDevicePluginState is used to store the device manager's plugin
	// state.
	PutDriverPluginState(state *state.PluginState) error
}

// UpdateNodeDriverInfoFn is the callback used to update the node from
// fingerprinting
type UpdateNodeDriverInfoFn func(string, *structs.DriverInfo)

// StorePluginReattachFn is used to store plugin reattachment configurations.
type StorePluginReattachFn func(*plugin.ReattachConfig) error

// FetchPluginReattachFn is used to retrieve the stored plugin reattachment
// configuration.
type FetchPluginReattachFn func() (*plugin.ReattachConfig, bool)

// Config is used to configure a driver manager
type Config struct {
	// Logger is the logger used by the device manager
	Logger log.Logger

	// Loader is the plugin loader
	Loader loader.PluginCatalog

	// PluginConfig is the config passed to the launched plugins
	PluginConfig *base.AgentConfig

	// Updater is used to update the node when driver information changes
	Updater UpdateNodeDriverInfoFn

	// EventHandlerFactory is used to retrieve a task event handler
	EventHandlerFactory TaskEventHandlerFactory

	// State is used to manage the device managers state
	State StateStorage

	// AllowedDrivers if set will only start driver plugins for the given
	// drivers
	AllowedDrivers map[string]struct{}

	// BlockedDrivers if set will not allow the given driver plugins to start
	BlockedDrivers map[string]struct{}
}

// manager is used to manage a set of driver plugins
type manager struct {
	// logger is the logger used by the device manager
	logger log.Logger

	// state is used to manage the device managers state
	state StateStorage

	// ctx is used to shutdown the device manager
	ctx    context.Context
	cancel context.CancelFunc

	// loader is the plugin loader
	loader loader.PluginCatalog

	// pluginConfig is the config passed to the launched plugins
	pluginConfig *base.AgentConfig

	// updater is used to update the node when device information changes
	updater UpdateNodeDriverInfoFn

	// eventHandlerFactory is passed to the instance managers and used to forward
	// task events
	eventHandlerFactory TaskEventHandlerFactory

	// instances is the list of managed devices, access is serialized by instanceMu
	instances   map[string]*instanceManager
	instancesMu sync.RWMutex

	// reattachConfigs stores the plugin reattach configs
	reattachConfigs    map[loader.PluginID]*shared.ReattachConfig
	reattachConfigLock sync.Mutex

	// allows/block lists
	allowedDrivers map[string]struct{}
	blockedDrivers map[string]struct{}

	// readyCh is ticked once at the end of Run()
	readyCh chan struct{}
}

// New returns a new driver manager
func New(c *Config) *manager {
	ctx, cancel := context.WithCancel(context.Background())
	return &manager{
		logger:              c.Logger.Named("driver_mgr"),
		state:               c.State,
		ctx:                 ctx,
		cancel:              cancel,
		loader:              c.Loader,
		pluginConfig:        c.PluginConfig,
		updater:             c.Updater,
		eventHandlerFactory: c.EventHandlerFactory,
		instances:           make(map[string]*instanceManager),
		reattachConfigs:     make(map[loader.PluginID]*shared.ReattachConfig),
		allowedDrivers:      c.AllowedDrivers,
		blockedDrivers:      c.BlockedDrivers,
		readyCh:             make(chan struct{}),
	}
}

// PluginType returns the type of plugin this mananger mananges
func (*manager) PluginType() string { return base.PluginTypeDriver }

// Run starts the mananger, initializes driver plugins and blocks until Shutdown
// is called.
func (m *manager) Run() {
	// Load any previous plugin reattach configuration
	if err := m.loadReattachConfigs(); err != nil {
		m.logger.Warn("unable to load driver plugin reattach configs, a driver process may have been leaked",
			"error", err)
	}

	// Get driver plugins
	driversPlugins := m.loader.Catalog()[base.PluginTypeDriver]
	if len(driversPlugins) == 0 {
		m.logger.Debug("exiting since there are no driver plugins")
		m.cancel()
		return
	}

	var skippedDrivers []string
	for _, d := range driversPlugins {
		id := loader.PluginInfoID(d)
		if m.isDriverBlocked(id.Name) {
			skippedDrivers = append(skippedDrivers, id.Name)
			continue
		}

		storeFn := func(c *plugin.ReattachConfig) error {
			return m.storePluginReattachConfig(id, c)
		}
		fetchFn := func() (*plugin.ReattachConfig, bool) {
			return m.fetchPluginReattachConfig(id)
		}

		instance := newInstanceManager(&instanceManagerConfig{
			Logger:               m.logger,
			Ctx:                  m.ctx,
			Loader:               m.loader,
			StoreReattach:        storeFn,
			FetchReattach:        fetchFn,
			PluginConfig:         m.pluginConfig,
			ID:                   &id,
			UpdateNodeFromDriver: m.updater,
			EventHandlerFactory:  m.eventHandlerFactory,
		})

		m.instancesMu.Lock()
		m.instances[id.Name] = instance
		m.instancesMu.Unlock()
	}

	if len(skippedDrivers) > 0 {
		m.logger.Debug("drivers skipped due to allow/block list", "skipped_drivers", skippedDrivers)
	}

	// signal ready
	close(m.readyCh)
}

// Shutdown cleans up all the plugins
func (m *manager) Shutdown() {
	// Cancel the context to stop any requests
	m.cancel()

	m.instancesMu.RLock()
	defer m.instancesMu.RUnlock()

	// Go through and shut everything down
	for _, i := range m.instances {
		i.cleanup()
	}
}

func (m *manager) WaitForFirstFingerprint(ctx context.Context) <-chan struct{} {
	ctx, cancel := context.WithCancel(ctx)
	go m.waitForFirstFingerprint(ctx, cancel)
	return ctx.Done()
}

func (m *manager) waitForFirstFingerprint(ctx context.Context, cancel context.CancelFunc) {
	defer cancel()
	// We don't want to start initial fingerprint wait until Run loop has
	// finished
	select {
	case <-m.readyCh:
	case <-ctx.Done():
		// parent context canceled or timedout
		return
	case <-m.ctx.Done():
		// shutdown called
		return
	}

	var mu sync.Mutex
	var availDrivers []string
	var wg sync.WaitGroup

	// loop through instances and wait for each to finish initial fingerprint
	m.instancesMu.RLock()
	for n, i := range m.instances {
		wg.Add(1)
		go func(name string, instance *instanceManager) {
			defer wg.Done()
			instance.WaitForFirstFingerprint(ctx)
			if instance.getLastHealth() != drivers.HealthStateUndetected {
				mu.Lock()
				availDrivers = append(availDrivers, name)
				mu.Unlock()
			}
		}(n, i)
	}
	m.instancesMu.RUnlock()
	wg.Wait()
	m.logger.Debug("detected drivers", "drivers", availDrivers)
}

func (m *manager) loadReattachConfigs() error {
	m.reattachConfigLock.Lock()
	defer m.reattachConfigLock.Unlock()

	s, err := m.state.GetDriverPluginState()
	if err != nil {
		return err
	}

	if s != nil {
		for name, c := range s.ReattachConfigs {
			if m.isDriverBlocked(name) {
				m.logger.Warn("reattach config for driver plugin found but driver is blocked due to allow/block list, killing plugin",
					"driver", name)
				m.shutdownBlockedDriver(name, c)
				continue
			}

			id := loader.PluginID{
				PluginType: base.PluginTypeDriver,
				Name:       name,
			}

			m.reattachConfigs[id] = c
		}
	}
	return nil
}

// shutdownBlockedDriver is used to forcefully shutdown a running driver plugin
// when it has been blocked due to allow/block lists
func (m *manager) shutdownBlockedDriver(name string, reattach *shared.ReattachConfig) {
	c, err := shared.ReattachConfigToGoPlugin(reattach)
	if err != nil {
		m.logger.Warn("failed to reattach and kill blocked driver plugin",
			"driver", name, "error", err)
		return

	}
	pluginInstance, err := m.loader.Reattach(name, base.PluginTypeDriver, c)
	if err != nil {
		m.logger.Warn("failed to reattach and kill blocked driver plugin",
			"driver", name, "error", err)
		return
	}

	if !pluginInstance.Exited() {
		pluginInstance.Kill()
	}
}

// storePluginReattachConfig is used as a callback to the instance managers and
// persists thhe plugin reattach configurations.
func (m *manager) storePluginReattachConfig(id loader.PluginID, c *plugin.ReattachConfig) error {
	m.reattachConfigLock.Lock()
	defer m.reattachConfigLock.Unlock()

	// Store the new reattach config
	m.reattachConfigs[id] = shared.ReattachConfigFromGoPlugin(c)

	// Persist the state
	s := &state.PluginState{
		ReattachConfigs: make(map[string]*shared.ReattachConfig, len(m.reattachConfigs)),
	}

	for id, c := range m.reattachConfigs {
		s.ReattachConfigs[id.Name] = c
	}

	return m.state.PutDriverPluginState(s)
}

// fetchPluginReattachConfig is used as a callback to the instance managers and
// retrieves the plugin reattach config. If it has not been stored it will
// return nil
func (m *manager) fetchPluginReattachConfig(id loader.PluginID) (*plugin.ReattachConfig, bool) {
	m.reattachConfigLock.Lock()
	defer m.reattachConfigLock.Unlock()

	if cfg, ok := m.reattachConfigs[id]; ok {
		c, err := shared.ReattachConfigToGoPlugin(cfg)
		if err != nil {
			m.logger.Warn("failed to read plugin reattach config", "config", cfg, "error", err)
			delete(m.reattachConfigs, id)
			return nil, false
		}
		return c, true
	}
	return nil, false
}

func (m *manager) Dispense(d string) (drivers.DriverPlugin, error) {
	m.instancesMu.RLock()
	defer m.instancesMu.RUnlock()
	if instance, ok := m.instances[d]; ok {
		return instance.dispense()
	}

	return nil, ErrDriverNotFound
}

func (m *manager) isDriverBlocked(name string) bool {
	// Block drivers that are not in the allowed list if it is set.
	if _, ok := m.allowedDrivers[name]; len(m.allowedDrivers) > 0 && !ok {
		return true
	}

	// Block drivers that are in the blocked list
	if _, ok := m.blockedDrivers[name]; ok {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package sisparse

import (
	"bufio"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/yhat/scrape"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"

	"github.com/iamwave/samorozvrh/backend/cache"
)

// The year in which semester 1 begins
const schoolYear = 2018

// 1 for winter, 2 for summer
const semester = 1

const sisUrl = "https://is.cuni.cz/studium/predmety/index.php?do=predmet&kod=%s&skr=%d&sem=%d"
const scheduleBaseUrl = "https://is.cuni.cz/studium/rozvrhng/"

const DEBUG = false

// Returns a two-dimensional array containing groups of events.
// Each group is a slice of events which must be enrolled together,
// the groups represent different times/teachers of the same course.
// Also, lectures and seminars/practicals are in separate groups.
func GetCourseEvents(courseCode string) ([][]Event, error) {
	subjectUrl := fmt.Sprintf(sisUrl, courseCode, schoolYear, semester)

	subjectHtmlRoot, err := getHtml(subjectUrl)
	if err != nil {
		return nil, err
	}
	// It is difficult to directly convert an event code to a schedule link,
	// because SIS requires the faculty number. Therefore we first open the course
	// in the "Subjects" SIS module and then go to a link which takes
	// us to the schedule.
	scheduleUrl, err := getScheduleUrl(subjectHtmlRoot)
	if err != nil {
		return nil, err
	}

	// For some subjects (ASZLJ3010), the link has the wrong semester for whatever reason
	// (even though the correct semester is specified in the original URL).
	// Let's fix this manually.
	scheduleUrl = strings.Replace(
		scheduleUrl,
		fmt.Sprintf("sem=%d", 3-semester),
		fmt.Sprintf("sem=%d", semester),
		-1,
	)

	scheduleHtmlRoot, err := getHtml(scheduleUrl)
	if err != nil {
		return nil, err
	}
	return parseCourseEvents(scheduleHtmlRoot)
}

func getScheduleUrl(root *html.Node) (string, error) {
	const scheduleLinkText = "Rozvrh"

	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A {
			return scrape.Text(n) == scheduleLinkText
		}
		return false
	}

	scheduleLink, ok := scrape.Find(root, matcher)
	if !ok {
		return "", errors.New("Couldn't find schedule URL")
	}
	relativeUrl := scrape.Attr(scheduleLink, "href")
	return getAbsoluteUrl(sisUrl, relativeUrl)
}

func parseCourseEvents(root *html.Node) ([][]Event, error) {
	if DEBUG {
		f, err := os.Create("/tmp/samorozvrh_debug.html")
		if err != nil {
			return nil, err
		}
		w := bufio.NewWriter(f)
		html.Render(w, root)
		w.Flush()
	}

	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Tr && n.Parent != nil && n.Parent.Parent != nil {
			return (scrape.Attr(n.Parent.Parent, "id") == "table1" ||
				scrape.Attr(n.Parent.Parent, "id") == "roz_predmet_macro1") &&
				scrape.Attr(n, "class") != "head1" // ignore table header
		}
		return false
	}

	eventsTable := scrape.FindAll(root, matcher)
	if len(eventsTable) == 0 {
		// The event table is not present at all (possibly SIS returned an error message)
		return nil, errors.New("Couldn't find the event table")
	}

	res := [][]Event{}
	group := []Event{}
	for _, row := range eventsTable {
		event, err := parseEvent(row)
		if err != nil {
			continue
		}
		// A non-empty name means the start of a new group;
		// names are omitted in all but the first event of a group.
		if event.Name != "" {
			if len(group) > 0 {
				res = append(res, group)
			}
			group = []Event{}
		} else {
			// Add the missing fields based on the group's first event
			event.Name = group[0].Name
			event.Teacher = group[0].Teacher
		}
		group = append(group, event)
	}
	if len(group) > 0 {
		res = append(res, group)
	}

	if len(res) == 0 {
		return nil, errors.New("The course has no scheduled events")
	}

	return res, nil
}

func parseEvent(event *html.Node) (Event, error) {
	var cols []string
	for col := event.FirstChild; col != nil; col = col.NextSibling {
		// For some reason we also get siblings with no tag and no data?
		if len(strings.TrimSpace(col.Data)) > 0 {
			if col.DataAtom == atom.A {
				cols = append(cols, scrape.Attr(col, "href"))
			} else {
				cols = append(cols, scrape.Text(col))
			}
		}
	}

	relativeEventUrl := scrape.Attr(event.FirstChild.NextSibling.FirstChild, "href")
	eventUrl, err := getAbsoluteUrl(scheduleBaseUrl, relativeEventUrl)
	if err != nil {
		return Event{}, err
	}

	e := Event{
		Type:     cols[1],
		Name:     cols[2],
		Teacher:  cols[3],
		Room:     cols[5],
		Language: cols[7],
	}

	addEventBuilding(&e, eventUrl)

	err = addEventScheduling(&e, cols[4], cols[6])
	return e, err
}

func addEventBuilding(e *Event, eventUrl string) error {
	cacheName := []string{"rooms", e.Room}

	if cache.Has(cacheName) {
		building, err := cache.Get(cacheName)
		if err == nil {
			e.Building = building
		}
		return err
	} else {
		root, err := getHtml(eventUrl)
		if err != nil {
			return err
		}

		matcher := func(n *html.Node) bool {
			if n.DataAtom == atom.Td &&
				n.Parent != nil &&
				n.Parent.FirstChild != n &&
				scrape.Text(n.Parent.FirstChild) == "Místo výuky:" {
				return true
			}
			return false
		}

		rooms := scrape.FindAll(root, matcher)
		if len(rooms) != 1 {
			return errors.New(fmt.Sprintf("Found %d rooms, expected 1", len(rooms)))
		}
		building, err := roomToBuilding(scrape.Text(rooms[0]))
		if err != nil {
			return err
		}
		cache.Set(cacheName, building)
		e.Building = building
	}
	return nil
}

func addEventScheduling(e *Event, daytime string, dur string) error {
	// For strings such as "Út 12:20"
	if len(daytime) == 0 {
		return errors.New("The daytime field is empty")
	}

	daytimeRunes := []rune(daytime)
	var err error
	e.Day, err = parseDay(string(daytimeRunes[:2]))
	if err != nil {
		return err
	}

	timeFrom, err := time.Parse("15:04", string(daytimeRunes[3:]))
	if err != nil {
		return err
	}

	d, parity, err := parseDurationAndWeekParity(dur)
	if err != nil {
		return err
	}

	e.TimeFrom = timeFrom
	e.TimeTo = timeFrom.Add(time.Minute * time.Duration(d))
	e.WeekParity = parity
	return nil
}

func parseDurationAndWeekParity(dur string) (int, int, error) {
	// Strings like "90" or "240 Sudé týdny (liché kalendářní)"
	w := strings.Fields(dur)
	d, err := strconv.Atoi(w[0])
	if err != nil {
		return -1, -1, errors.New(fmt.Sprintf("Unable to parse duration: %s", err))
	}
	parity := 0
	if len(w) > 1 {
		if w[1] == "Liché" {
			parity = 1
		} else {
			parity = 2
		}
	}
	return d, parity, nil
}

func parseDay(day string) (int, error) {
	days := []string{"Po", "Út", "St", "Čt", "Pá"}
	for i, d := range days {
		if d == day {
			return i, nil
		}
	}
	return -1, errors.New(fmt.Sprintf("Unknown day \"%s\"", day))
}

func getAbsoluteUrl(base, relative string) (string, error) {
	baseUrl, err := url.Parse(base)
	if err != nil {
		return "", err
	}
	relativeUrl, err := url.Parse(relative)
	if err != nil {
		return "", err
	}
	return baseUrl.ResolveReference(relativeUrl).String(), nil
}

func roomToBuilding(room string) (string, error) {
	p := strings.LastIndex(room, ") ")
	if p == -1 {
		return "", errors.New(fmt.Sprintf("Could not parse room: %s", room))
	}
	return room[p+2:], nil
}

func getHtml(url string) (*html.Node, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	return html.Parse(resp.Body)
}
<commit_msg>Improve sisparse error handling<commit_after>package sisparse

import (
	"bufio"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/yhat/scrape"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"

	"github.com/iamwave/samorozvrh/backend/cache"
)

// The year in which semester 1 begins
const schoolYear = 2018

// 1 for winter, 2 for summer
const semester = 1

const sisUrl = "https://is.cuni.cz/studium/predmety/index.php?do=predmet&kod=%s&skr=%d&sem=%d"
const scheduleBaseUrl = "https://is.cuni.cz/studium/rozvrhng/"

const DEBUG = false

// Returns a two-dimensional array containing groups of events.
// Each group is a slice of events which must be enrolled together,
// the groups represent different times/teachers of the same course.
// Also, lectures and seminars/practicals are in separate groups.
func GetCourseEvents(courseCode string) ([][]Event, error) {
	subjectUrl := fmt.Sprintf(sisUrl, courseCode, schoolYear, semester)

	subjectHtmlRoot, err := getHtml(subjectUrl)
	if err != nil {
		return nil, err
	}
	// It is difficult to directly convert an event code to a schedule link,
	// because SIS requires the faculty number. Therefore we first open the course
	// in the "Subjects" SIS module and then go to a link which takes
	// us to the schedule.
	scheduleUrl, err := getScheduleUrl(subjectHtmlRoot)
	if err != nil {
		return nil, err
	}

	// For some subjects (ASZLJ3010), the link has the wrong semester for whatever reason
	// (even though the correct semester is specified in the original URL).
	// Let's fix this manually.
	scheduleUrl = strings.Replace(
		scheduleUrl,
		fmt.Sprintf("sem=%d", 3-semester),
		fmt.Sprintf("sem=%d", semester),
		-1,
	)

	scheduleHtmlRoot, err := getHtml(scheduleUrl)
	if err != nil {
		return nil, err
	}
	return parseCourseEvents(scheduleHtmlRoot)
}

func getScheduleUrl(root *html.Node) (string, error) {
	const scheduleLinkText = "Rozvrh"

	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A {
			return scrape.Text(n) == scheduleLinkText
		}
		return false
	}

	scheduleLink, ok := scrape.Find(root, matcher)
	if !ok {
		return "", errors.New("Couldn't find schedule URL")
	}
	relativeUrl := scrape.Attr(scheduleLink, "href")
	return getAbsoluteUrl(sisUrl, relativeUrl)
}

func parseCourseEvents(root *html.Node) ([][]Event, error) {
	if DEBUG {
		f, err := os.Create("/tmp/samorozvrh_debug.html")
		if err != nil {
			return nil, err
		}
		w := bufio.NewWriter(f)
		html.Render(w, root)
		w.Flush()
	}

	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Tr && n.Parent != nil && n.Parent.Parent != nil {
			return (scrape.Attr(n.Parent.Parent, "id") == "table1" ||
				scrape.Attr(n.Parent.Parent, "id") == "roz_predmet_macro1") &&
				scrape.Attr(n, "class") != "head1" // ignore table header
		}
		return false
	}

	eventsTable := scrape.FindAll(root, matcher)
	if len(eventsTable) == 0 {
		// The event table is not present at all (possibly SIS returned an error message)
		return nil, errors.New("Couldn't find the event table")
	}

	res := [][]Event{}
	group := []Event{}
	for _, row := range eventsTable {
		event, err := parseEvent(row)
		if err != nil {
			return nil, err
		}
		if event == nil {
			// This could happen if an event is not scheduled, see:
			// https://is.cuni.cz/studium/rozvrhng/roz_predmet_macro.php?fak=11320&skr=2018&sem=1&predmet=NAIL062
			continue
		}
		// A non-empty name means the start of a new group;
		// names are omitted in all but the first event of a group.
		if event.Name != "" {
			if len(group) > 0 {
				res = append(res, group)
			}
			group = []Event{}
		} else {
			// Add the missing fields based on the group's first event
			event.Name = group[0].Name
			event.Teacher = group[0].Teacher
		}
		group = append(group, *event)
	}
	if len(group) > 0 {
		res = append(res, group)
	}

	if len(res) == 0 {
		return nil, errors.New("The course has no scheduled events")
	}

	return res, nil
}

func parseEvent(event *html.Node) (*Event, error) {
	var cols []string
	for col := event.FirstChild; col != nil; col = col.NextSibling {
		// For some reason we also get siblings with no tag and no data?
		if len(strings.TrimSpace(col.Data)) > 0 {
			if col.DataAtom == atom.A {
				cols = append(cols, scrape.Attr(col, "href"))
			} else {
				cols = append(cols, scrape.Text(col))
			}
		}
	}

	e := Event{
		Type:     cols[1],
		Name:     cols[2],
		Teacher:  cols[3],
		Room:     cols[5],
		Language: cols[7],
	}

	err := addEventScheduling(&e, cols[4], cols[6])
	if err != nil {
		// The event is not scheduled - this is ok
		return nil, nil
	}

	relativeEventUrl := scrape.Attr(event.FirstChild.NextSibling.FirstChild, "href")
	eventUrl, err := getAbsoluteUrl(scheduleBaseUrl, relativeEventUrl)
	if err != nil {
		return nil, err
	}

	err = addEventBuilding(&e, eventUrl)
	if err != nil {
		return nil, err
	}

	return &e, err
}

func addEventBuilding(e *Event, eventUrl string) error {
	cacheName := []string{"rooms", e.Room}

	if cache.Has(cacheName) {
		building, err := cache.Get(cacheName)
		if err == nil {
			e.Building = building
		}
		return err
	} else {
		root, err := getHtml(eventUrl)
		if err != nil {
			return err
		}

		matcher := func(n *html.Node) bool {
			if n.DataAtom == atom.Td &&
				n.Parent != nil &&
				n.Parent.FirstChild != n &&
				scrape.Text(n.Parent.FirstChild) == "Místo výuky:" {
				return true
			}
			return false
		}

		rooms := scrape.FindAll(root, matcher)
		if len(rooms) != 1 {
			return errors.New(fmt.Sprintf("Found %d rooms, expected 1", len(rooms)))
		}
		building, err := roomToBuilding(scrape.Text(rooms[0]))
		if err != nil {
			return err
		}
		cache.Set(cacheName, building)
		e.Building = building
	}
	return nil
}

func addEventScheduling(e *Event, daytime string, dur string) error {
	// For strings such as "Út 12:20"
	if len(daytime) == 0 {
		return errors.New("The daytime field is empty")
	}

	daytimeRunes := []rune(daytime)
	var err error
	e.Day, err = parseDay(string(daytimeRunes[:2]))
	if err != nil {
		return err
	}

	timeFrom, err := time.Parse("15:04", string(daytimeRunes[3:]))
	if err != nil {
		return err
	}

	d, parity, err := parseDurationAndWeekParity(dur)
	if err != nil {
		return err
	}

	e.TimeFrom = timeFrom
	e.TimeTo = timeFrom.Add(time.Minute * time.Duration(d))
	e.WeekParity = parity
	return nil
}

func parseDurationAndWeekParity(dur string) (int, int, error) {
	// Strings like "90" or "240 Sudé týdny (liché kalendářní)"
	w := strings.Fields(dur)
	d, err := strconv.Atoi(w[0])
	if err != nil {
		return -1, -1, errors.New(fmt.Sprintf("Unable to parse duration: %s", err))
	}
	parity := 0
	if len(w) > 1 {
		if w[1] == "Liché" {
			parity = 1
		} else {
			parity = 2
		}
	}
	return d, parity, nil
}

func parseDay(day string) (int, error) {
	days := []string{"Po", "Út", "St", "Čt", "Pá"}
	for i, d := range days {
		if d == day {
			return i, nil
		}
	}
	return -1, errors.New(fmt.Sprintf("Unknown day \"%s\"", day))
}

func getAbsoluteUrl(base, relative string) (string, error) {
	baseUrl, err := url.Parse(base)
	if err != nil {
		return "", err
	}
	relativeUrl, err := url.Parse(relative)
	if err != nil {
		return "", err
	}
	return baseUrl.ResolveReference(relativeUrl).String(), nil
}

func roomToBuilding(room string) (string, error) {
	p := strings.LastIndex(room, ") ")
	if p == -1 {
		return "", errors.New(fmt.Sprintf("Could not parse room: %s", room))
	}
	return room[p+2:], nil
}

func getHtml(url string) (*html.Node, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	return html.Parse(resp.Body)
}
<|endoftext|>


<commit_before>package logger

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"net"
	"os"
	"strings"
	"sync/atomic"
	"time"
	"unsafe"
)

// The csyslog function is necessary here because cgo does not appear
// to be able to call a variadic function directly and syslog has the
// same signature as printf.

// #include <stdlib.h>
// #include <syslog.h>
// void csyslog(int p, const char *m) {
//     syslog(p, "%s", m);
// }
import "C"

const (
	NumMessages   = 10 * 1024 // number of allowed log messages
	STDOUT_FORMAT = "2006-01-02T15:04:05.000 "
)

// container for a pending log message
type logMessage struct {
	bytes.Buffer
	level C.int
	time  time.Time
}

var (
	ErrLogFullBuf           = errors.New("Log message queue is full")
	ErrFreeMessageOverflow  = errors.New("Too many free messages. Overflow of fixed	set.")
	ErrFreeMessageUnderflow = errors.New("Too few free messages. Underflow of fixed	set.")

	// the logName object for syslog to use
	logName       *C.char
	logNameString string

	// the message queue of pending or free messages
	// since only one can be full at a time, the total size will be about 10MB
	messages     chan *logMessage
	freeMessages chan *logMessage

	// mapping of our levels to syslog values
	levelSysLog = map[Level]C.int{
		Levels.Access: C.LOG_INFO,
		Levels.Off:    C.LOG_DEBUG,
		Levels.Panic:  C.LOG_ERR,
		Levels.Error:  C.LOG_ERR,
		Levels.Warn:   C.LOG_WARNING,
		Levels.Info:   C.LOG_INFO,
		Levels.Debug:  C.LOG_DEBUG,
	}

	// mirror of levelMap used to avoid making a new string with '[]' on every log
	// call
	levelMapFmt = map[Level][]byte{
		Levels.Access: []byte("[Access] "),
		Levels.Off:    []byte("[Off] "),
		Levels.Panic:  []byte("[Panic] "),
		Levels.Error:  []byte("[Error] "),
		Levels.Warn:   []byte("[Warn] "),
		Levels.Info:   []byte("[Info] "),
		Levels.Debug:  []byte("[Debug] "),
	}

	customSock net.Conn = nil

	logWriterFinished chan struct{}

	stdhdl io.Writer
)

// When called, this will switch over to writting log messages to the defined socket.
func SetCustomSocket(address, network string) (err error) {
	customSock, err = net.Dial(network, address)

	return err
}

func SetStdOut() {
	stdhdl = io.Writer(os.Stdout)
}

func SetStdErr() {
	stdhdl = io.Writer(os.Stderr)
}

// SetLogName sets the indentifier used by syslog for this program
func SetLogName(p string) (err error) {

	logNameString = p
	if stdhdl != nil {
		return
	}

	if logName != nil {
		C.free(unsafe.Pointer(logName))
	}
	logName = C.CString(p)
	_, err = C.openlog(logName, C.LOG_NDELAY|C.LOG_NOWAIT|C.LOG_PID, C.LOG_USER)
	if err != nil {
		atomic.AddUint64(&errCount, 1)
	}

	return err
}

// freeMsg releases the message back to be reused
func freeMsg(msg *logMessage) (err error) {
	msg.Reset()
	select {
	case freeMessages <- msg: // no-op
	default:
		atomic.AddUint64(&errCount, 1)
		return ErrFreeMessageOverflow
	}

	return
}

// queueMsg adds a message to the pending messages channel. It will drop the
// message and return an error if the channel is full.
func queueMsg(lvl Level, prefix, format string, v ...interface{}) (err error) {
	atomic.AddUint64(&logCount, 1)

	var msg *logMessage

	// get a message if possible
	select {
	case msg = <-freeMessages: // got a message-struct; proceed
	default:
		// no messages left, drop
		atomic.AddUint64(&dropCount, 1)
		return
	}

	msg.time = time.Now()

	// render the message: level prefix, message body, C null terminator
	msg.level = levelSysLog[lvl]
	if _, err = msg.Write(levelMapFmt[lvl]); err != nil {
		atomic.AddUint64(&errCount, 1)
		freeMsg(msg)
		return
	}
	if _, err = fmt.Fprintf(msg, "%s", prefix); err != nil {
		atomic.AddUint64(&errCount, 1)
		freeMsg(msg)
		return
	}
	if _, err = fmt.Fprintf(msg, format, v...); err != nil {
		atomic.AddUint64(&errCount, 1)
		freeMsg(msg)
		return
	}
	if err = msg.WriteByte(0); err != nil {
		atomic.AddUint64(&errCount, 1)
		freeMsg(msg)
		return
	}

	// queue the message
	select {
	case messages <- msg:
		// no-op
	default:
		// this should never happen since there is an exact number of messages
		atomic.AddUint64(&errCount, 1)
		return ErrLogFullBuf
	}

	return
}

// Just print mesg to stdout
func printStd(msg *logMessage) (err error) {
	// remove C null-termination byte
	message := string(msg.Bytes()[:msg.Len()-1])
	message = strings.TrimRight(message, "\n")
	fmt.Fprintf(stdhdl, "%s%s%s\n", msg.time.Format(STDOUT_FORMAT), logNameString, message)
	return
}

// write a message to syslog. This is a concrete, blocking event.
func write(msg *logMessage) (err error) {
	start := (*C.char)(unsafe.Pointer(&msg.Bytes()[0]))
	if _, err = C.csyslog(C.LOG_USER|msg.level, start); err != nil {
		atomic.AddUint64(&errCount, 1)
	}
	return
}

// write a message to a pre-defined custom socket. This is a concrete, blocking event.
// Writes out using the syslog rfc5424 format.
func writeCustomSocket(msg *logMessage) (err error) {
	if _, err = customSock.Write(bytes.Join([][]byte{[]byte(fmt.Sprintf("<%d>", C.LOG_USER|msg.level)),
		msg.Bytes()}, []byte(""))); err != nil {
		atomic.AddUint64(&errCount, 1)
	}
	return
}

// logWriter will write out messages to syslog. It may block if something breaks
// within the syslog call.
func logWriter() {
	for msg := range messages {
		if stdhdl != nil {
			printStd(msg)
		} else if customSock == nil {
			write(msg)
		} else {
			writeCustomSocket(msg)
		}
		freeMsg(msg)
	}
	if customSock != nil {
		customSock.Close()
	}
	close(logWriterFinished)
}

// Close shuts down the logger system. After Close is called, any additional
// logs will panic. Only call this if you are completely done.
func Close(ctx context.Context) error {
	close(messages)
	select {
	case <-logWriterFinished:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

// DrainContext blocks until it sees no pending messages or the context is canceled.
// Pending messages may never run out if another goroutine is constantly
// writing.
func DrainContext(ctx context.Context) error {
	for ctx.Err() == nil && len(messages) > 0 {
		innerCtx, cancel := context.WithTimeout(ctx, 10*time.Millisecond)
		<-innerCtx.Done() // Wait for 10ms and check len(messages) again
		cancel()
	}
	return ctx.Err()
}

// Drain is like DrainContext, but you didn't want to write context.Background().
// Outside of tests, you want to use DrainContext.
func Drain() {
	_ = DrainContext(context.Background())
}

func setup() {
	stdhdl = nil
	messages = make(chan *logMessage, NumMessages)
	freeMessages = make(chan *logMessage, NumMessages)
	msgArr := make([]logMessage, NumMessages)
	for i := range msgArr {
		if err := freeMsg(&msgArr[i]); err != nil {
			break
		}
	}

	logWriterFinished = make(chan struct{}, 1)
	go logWriter()
}

func init() {
	setup()
}
<commit_msg>Adding a way to tee message out of the low level log system<commit_after>package logger

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"net"
	"os"
	"strings"
	"sync/atomic"
	"time"
	"unsafe"
)

// The csyslog function is necessary here because cgo does not appear
// to be able to call a variadic function directly and syslog has the
// same signature as printf.

// #include <stdlib.h>
// #include <syslog.h>
// void csyslog(int p, const char *m) {
//     syslog(p, "%s", m);
// }
import "C"

const (
	NumMessages   = 10 * 1024 // number of allowed log messages
	STDOUT_FORMAT = "2006-01-02T15:04:05.000 "
)

// container for a pending log message
type logMessage struct {
	bytes.Buffer
	level C.int
	time  time.Time
}

var (
	ErrLogFullBuf           = errors.New("Log message queue is full")
	ErrFreeMessageOverflow  = errors.New("Too many free messages. Overflow of fixed	set.")
	ErrFreeMessageUnderflow = errors.New("Too few free messages. Underflow of fixed	set.")

	// the logName object for syslog to use
	logName       *C.char
	logNameString string

	// the message queue of pending or free messages
	// since only one can be full at a time, the total size will be about 10MB
	messages     chan *logMessage
	freeMessages chan *logMessage

	// mapping of our levels to syslog values
	levelSysLog = map[Level]C.int{
		Levels.Access: C.LOG_INFO,
		Levels.Off:    C.LOG_DEBUG,
		Levels.Panic:  C.LOG_ERR,
		Levels.Error:  C.LOG_ERR,
		Levels.Warn:   C.LOG_WARNING,
		Levels.Info:   C.LOG_INFO,
		Levels.Debug:  C.LOG_DEBUG,
	}

	// mirror of levelMap used to avoid making a new string with '[]' on every log
	// call
	levelMapFmt = map[Level][]byte{
		Levels.Access: []byte("[Access] "),
		Levels.Off:    []byte("[Off] "),
		Levels.Panic:  []byte("[Panic] "),
		Levels.Error:  []byte("[Error] "),
		Levels.Warn:   []byte("[Warn] "),
		Levels.Info:   []byte("[Info] "),
		Levels.Debug:  []byte("[Debug] "),
	}

	customSock net.Conn = nil

	logWriterFinished chan struct{}

	stdhdl io.Writer

	logTee chan string
)

// When called, this will switch over to writting log messages to the defined socket.
func SetCustomSocket(address, network string) (err error) {
	customSock, err = net.Dial(network, address)

	return err
}

func SetStdOut() {
	stdhdl = io.Writer(os.Stdout)
}

func SetStdErr() {
	stdhdl = io.Writer(os.Stderr)
}

func SetTee(tee chan string) {
	logTee = tee
}

// SetLogName sets the indentifier used by syslog for this program
func SetLogName(p string) (err error) {

	logNameString = p
	if stdhdl != nil {
		return
	}

	if logName != nil {
		C.free(unsafe.Pointer(logName))
	}
	logName = C.CString(p)
	_, err = C.openlog(logName, C.LOG_NDELAY|C.LOG_NOWAIT|C.LOG_PID, C.LOG_USER)
	if err != nil {
		atomic.AddUint64(&errCount, 1)
	}

	return err
}

// freeMsg releases the message back to be reused
func freeMsg(msg *logMessage) (err error) {
	msg.Reset()
	select {
	case freeMessages <- msg: // no-op
	default:
		atomic.AddUint64(&errCount, 1)
		return ErrFreeMessageOverflow
	}

	return
}

// queueMsg adds a message to the pending messages channel. It will drop the
// message and return an error if the channel is full.
func queueMsg(lvl Level, prefix, format string, v ...interface{}) (err error) {
	atomic.AddUint64(&logCount, 1)

	var msg *logMessage

	// get a message if possible
	select {
	case msg = <-freeMessages: // got a message-struct; proceed
	default:
		// no messages left, drop
		atomic.AddUint64(&dropCount, 1)
		return
	}

	msg.time = time.Now()

	// render the message: level prefix, message body, C null terminator
	msg.level = levelSysLog[lvl]
	if _, err = msg.Write(levelMapFmt[lvl]); err != nil {
		atomic.AddUint64(&errCount, 1)
		freeMsg(msg)
		return
	}
	if _, err = fmt.Fprintf(msg, "%s", prefix); err != nil {
		atomic.AddUint64(&errCount, 1)
		freeMsg(msg)
		return
	}
	if _, err = fmt.Fprintf(msg, format, v...); err != nil {
		atomic.AddUint64(&errCount, 1)
		freeMsg(msg)
		return
	}
	if err = msg.WriteByte(0); err != nil {
		atomic.AddUint64(&errCount, 1)
		freeMsg(msg)
		return
	}

	// queue the message
	select {
	case messages <- msg:
		// no-op
	default:
		// this should never happen since there is an exact number of messages
		atomic.AddUint64(&errCount, 1)
		return ErrLogFullBuf
	}

	return
}

// Send to a tee
func printTee(msg *logMessage) (err error) {
	// remove C null-termination byte
	message := string(msg.Bytes()[:msg.Len()-1])
	message = strings.TrimRight(message, "\n")
	select {
	case logTee <- fmt.Sprintf("%s%s%s", msg.time.Format(STDOUT_FORMAT), logNameString, message):
	default:
		err = fmt.Errorf("Log Tee Full")
	}
	return
}

// Just print mesg to stdout
func printStd(msg *logMessage) (err error) {
	// remove C null-termination byte
	message := string(msg.Bytes()[:msg.Len()-1])
	message = strings.TrimRight(message, "\n")
	fmt.Fprintf(stdhdl, "%s%s%s\n", msg.time.Format(STDOUT_FORMAT), logNameString, message)
	return
}

// write a message to syslog. This is a concrete, blocking event.
func write(msg *logMessage) (err error) {
	start := (*C.char)(unsafe.Pointer(&msg.Bytes()[0]))
	if _, err = C.csyslog(C.LOG_USER|msg.level, start); err != nil {
		atomic.AddUint64(&errCount, 1)
	}
	return
}

// write a message to a pre-defined custom socket. This is a concrete, blocking event.
// Writes out using the syslog rfc5424 format.
func writeCustomSocket(msg *logMessage) (err error) {
	if _, err = customSock.Write(bytes.Join([][]byte{[]byte(fmt.Sprintf("<%d>", C.LOG_USER|msg.level)),
		msg.Bytes()}, []byte(""))); err != nil {
		atomic.AddUint64(&errCount, 1)
	}
	return
}

// logWriter will write out messages to syslog. It may block if something breaks
// within the syslog call.
func logWriter() {
	for msg := range messages {
		if logTee != nil {
			printTee(msg)
		}

		if stdhdl != nil {
			printStd(msg)
		} else if customSock == nil {
			write(msg)
		} else {
			writeCustomSocket(msg)
		}
		freeMsg(msg)
	}
	if customSock != nil {
		customSock.Close()
	}
	close(logWriterFinished)
}

// Close shuts down the logger system. After Close is called, any additional
// logs will panic. Only call this if you are completely done.
func Close(ctx context.Context) error {
	close(messages)
	select {
	case <-logWriterFinished:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

// DrainContext blocks until it sees no pending messages or the context is canceled.
// Pending messages may never run out if another goroutine is constantly
// writing.
func DrainContext(ctx context.Context) error {
	for ctx.Err() == nil && len(messages) > 0 {
		innerCtx, cancel := context.WithTimeout(ctx, 10*time.Millisecond)
		<-innerCtx.Done() // Wait for 10ms and check len(messages) again
		cancel()
	}
	return ctx.Err()
}

// Drain is like DrainContext, but you didn't want to write context.Background().
// Outside of tests, you want to use DrainContext.
func Drain() {
	_ = DrainContext(context.Background())
}

func setup() {
	stdhdl = nil
	messages = make(chan *logMessage, NumMessages)
	freeMessages = make(chan *logMessage, NumMessages)
	msgArr := make([]logMessage, NumMessages)
	for i := range msgArr {
		if err := freeMsg(&msgArr[i]); err != nil {
			break
		}
	}

	logWriterFinished = make(chan struct{}, 1)
	go logWriter()
}

func init() {
	setup()
}
<|endoftext|>


<commit_before>package plugin

import (
	"crypto/tls"

	"github.com/hashicorp/go-plugin"
	"github.com/hashicorp/vault/helper/pluginutil"
	"github.com/hashicorp/vault/logical"
)

// BackendPluginName is the name of the plugin that can be
// dispensed rom the plugin server.
const BackendPluginName = "backend"

type BackendFactoryFunc func(*logical.BackendConfig) (logical.Backend, error)
type TLSProdiverFunc func() (*tls.Config, error)

type ServeOpts struct {
	BackendFactoryFunc BackendFactoryFunc
	TLSProviderFunc    TLSProdiverFunc
}

// Serve is a helper function used to serve a backend plugin. This
// should be ran on the plugin's main process.
func Serve(opts *ServeOpts) error {
	// pluginMap is the map of plugins we can dispense.
	var pluginMap = map[string]plugin.Plugin{
		"backend": &BackendPlugin{
			Factory: opts.BackendFactoryFunc,
		},
	}

	err := pluginutil.OptionallyEnableMlock()
	if err != nil {
		return err
	}

	// If FetchMetadata is true, run without TLSProvider
	plugin.Serve(&plugin.ServeConfig{
		HandshakeConfig: handshakeConfig,
		Plugins:         pluginMap,
		TLSProvider:     opts.TLSProviderFunc,
	})

	return nil
}

// handshakeConfigs are used to just do a basic handshake between
// a plugin and host. If the handshake fails, a user friendly error is shown.
// This prevents users from executing bad plugins or executing a plugin
// directory. It is a UX feature, not a security feature.
var handshakeConfig = plugin.HandshakeConfig{
	ProtocolVersion:  2,
	MagicCookieKey:   "VAULT_BACKEND_PLUGIN",
	MagicCookieValue: "6669da05-b1c8-4f49-97d9-c8e5bed98e20",
}
<commit_msg>plugin version bump (#3423)<commit_after>package plugin

import (
	"crypto/tls"

	"github.com/hashicorp/go-plugin"
	"github.com/hashicorp/vault/helper/pluginutil"
	"github.com/hashicorp/vault/logical"
)

// BackendPluginName is the name of the plugin that can be
// dispensed rom the plugin server.
const BackendPluginName = "backend"

type BackendFactoryFunc func(*logical.BackendConfig) (logical.Backend, error)
type TLSProdiverFunc func() (*tls.Config, error)

type ServeOpts struct {
	BackendFactoryFunc BackendFactoryFunc
	TLSProviderFunc    TLSProdiverFunc
}

// Serve is a helper function used to serve a backend plugin. This
// should be ran on the plugin's main process.
func Serve(opts *ServeOpts) error {
	// pluginMap is the map of plugins we can dispense.
	var pluginMap = map[string]plugin.Plugin{
		"backend": &BackendPlugin{
			Factory: opts.BackendFactoryFunc,
		},
	}

	err := pluginutil.OptionallyEnableMlock()
	if err != nil {
		return err
	}

	// If FetchMetadata is true, run without TLSProvider
	plugin.Serve(&plugin.ServeConfig{
		HandshakeConfig: handshakeConfig,
		Plugins:         pluginMap,
		TLSProvider:     opts.TLSProviderFunc,
	})

	return nil
}

// handshakeConfigs are used to just do a basic handshake between
// a plugin and host. If the handshake fails, a user friendly error is shown.
// This prevents users from executing bad plugins or executing a plugin
// directory. It is a UX feature, not a security feature.
var handshakeConfig = plugin.HandshakeConfig{
	ProtocolVersion:  3,
	MagicCookieKey:   "VAULT_BACKEND_PLUGIN",
	MagicCookieValue: "6669da05-b1c8-4f49-97d9-c8e5bed98e20",
}
<|endoftext|>


<commit_before>package lxdatabase_test

import (
	"github.com/layer-x/layerx-commons/lxdatabase"

	"github.com/layer-x/layerx-mesos-tpi/test_helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Lxdb", func() {
	test_helpers.StartETCD()

	Describe("Init(etcd_url)", func(){
		It("initializes without error", func() {
			err := lxdatabase.Init([]string{"http://127.0.0.1:2379"})
			Expect(err).To(BeNil())
		})
	})

	Describe("lxdatabase.Set(key, val)", func() {
		It("sets keys", func() {
			err := lxdatabase.Set("foo", "bar")
			Expect(err).To(BeNil())
		})
	})
	Describe("lxdatabase.Get(key)", func() {
		It("gets values", func() {
			val, err := lxdatabase.Get("foo")
			Expect(err).To(BeNil())
			Expect(val).To(Equal("bar"))
		})
	})
	Describe("lxdatabase.Rm(key)", func() {
		It("deletes keys", func() {
			err := lxdatabase.Rm("foo")
			Expect(err).To(BeNil())
			val, err := lxdatabase.Get("foo")
			Expect(val).To(Equal(""))
			Expect(err).To(Not(BeNil()))
			Expect(err.Error()).To(ContainSubstring("Key not found"))
		})
	})
	Describe("lxdatabase.Mkdir(key)", func() {
		It("makes directories", func() {
			err := lxdatabase.Mkdir("foo_dir")
			Expect(err).To(BeNil())
		})
	})
	Describe("lxdatabase.GetKeys(key)", func() {
		It("lists keys in directory", func() {
			keys, err := lxdatabase.GetKeys("foo_dir")
			Expect(err).To(BeNil())
			Expect(keys).To(BeEmpty())
			err = lxdatabase.Set("foo_dir/foo", "bar")
			Expect(err).To(BeNil())
			keys, err = lxdatabase.GetKeys("foo_dir")
			Expect(err).To(BeNil())
			Expect(keys).To(ContainElement("bar"))
		})
	})
	Describe("lxdatabase.GetSubdirectories(dir)", func() {
		It("lists sbudirs in directory", func() {
			dirs, err := lxdatabase.GetSubdirectories("foo_dir")
			Expect(err).To(BeNil())
			Expect(dirs).To(BeEmpty())
			err = lxdatabase.Mkdir("foo_dir/bar_dir")
			Expect(err).To(BeNil())
			dirs, err = lxdatabase.GetSubdirectories("foo_dir")
			Expect(err).To(BeNil())
			Expect(dirs).To(ContainElement("/foo_dir/bar_dir"))
		})
	})
	Describe("lxdatabase.Rmdir(key)", func() {
		It("makes directories", func() {
			err := lxdatabase.Rmdir("/foo_dir", true)
			Expect(err).To(BeNil())
		})
	})
	Describe("cleanup", func() {
		It("cleans up etcd", func() {
			test_helpers.CleanupETCD()
		})
	})
})
<commit_msg>start etcd on your own<commit_after>package lxdatabase_test

import (
	"github.com/layer-x/layerx-commons/lxdatabase"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Lxdb", func() {

	Describe("Init(etcd_url)", func(){
		It("initializes without error", func() {
			err := lxdatabase.Init([]string{"http://127.0.0.1:2379"})
			Expect(err).To(BeNil())
		})
	})

	Describe("lxdatabase.Set(key, val)", func() {
		It("sets keys", func() {
			err := lxdatabase.Set("foo", "bar")
			Expect(err).To(BeNil())
		})
	})
	Describe("lxdatabase.Get(key)", func() {
		It("gets values", func() {
			val, err := lxdatabase.Get("foo")
			Expect(err).To(BeNil())
			Expect(val).To(Equal("bar"))
		})
	})
	Describe("lxdatabase.Rm(key)", func() {
		It("deletes keys", func() {
			err := lxdatabase.Rm("foo")
			Expect(err).To(BeNil())
			val, err := lxdatabase.Get("foo")
			Expect(val).To(Equal(""))
			Expect(err).ToNot(BeNil())
			Expect(err.Error()).To(ContainSubstring("Key not found"))
		})
	})
	Describe("lxdatabase.Mkdir(key)", func() {
		It("makes directories", func() {
			err := lxdatabase.Mkdir("foo_dir")
			Expect(err).To(BeNil())
		})
	})
	Describe("lxdatabase.GetKeys(key)", func() {
		It("lists keys in directory", func() {
			keys, err := lxdatabase.GetKeys("foo_dir")
			Expect(err).To(BeNil())
			Expect(keys).To(BeEmpty())
			err = lxdatabase.Set("foo_dir/foo", "bar")
			Expect(err).To(BeNil())
			keys, err = lxdatabase.GetKeys("foo_dir")
			Expect(err).To(BeNil())
			Expect(keys).To(ContainElement("bar"))
		})
	})
	Describe("lxdatabase.GetSubdirectories(dir)", func() {
		It("lists sbudirs in directory", func() {
			dirs, err := lxdatabase.GetSubdirectories("foo_dir")
			Expect(err).To(BeNil())
			Expect(dirs).To(BeEmpty())
			err = lxdatabase.Mkdir("foo_dir/bar_dir")
			Expect(err).To(BeNil())
			dirs, err = lxdatabase.GetSubdirectories("foo_dir")
			Expect(err).To(BeNil())
			Expect(dirs).To(ContainElement("/foo_dir/bar_dir"))
		})
	})
	Describe("lxdatabase.Rmdir(key)", func() {
		It("makes directories", func() {
			err := lxdatabase.Rmdir("/foo_dir", true)
			Expect(err).To(BeNil())
		})
	})
})
<|endoftext|>


<commit_before>package errorcollector

import (
	"fmt"
	"strings"
)

// ErrorCollector is used to collect multiple error
// values while still satisfying the error interface
type ErrorCollector []error

type unwrapper interface {
	unwrap() []error
}

// New returns a new ErrorCollector
func New() ErrorCollector {
	var ec ErrorCollector
	return ec
}

func (ec ErrorCollector) unwrap() []error {
	collection := []error{}
	for _, err := range ec {
		collection = append(collection, err)
	}
	return collection
}

// Collect adds a single error or another collector to
// the collection of errors
func (ec *ErrorCollector) Collect(err error) {
	if err == nil {
		return
	}
	if castCollector, ok := err.(unwrapper); ok {
		*ec = append(*ec, castCollector.unwrap()...)
	} else {
		*ec = append(*ec, err)
	}
}

func (ec ErrorCollector) Error() string {
	collection := []string{}
	for _, err := range ec {
		collection = append(collection, err.Error())
	}
	return fmt.Sprintf("collected errors: %s", strings.Join(collection, ", "))
}
<commit_msg>improve godoc comment<commit_after>package errorcollector

import (
	"fmt"
	"strings"
)

// ErrorCollector is used to collect multiple error
// values while satisfying the common error interface
type ErrorCollector []error

type unwrapper interface {
	unwrap() []error
}

// New returns a new ErrorCollector
func New() ErrorCollector {
	var ec ErrorCollector
	return ec
}

func (ec ErrorCollector) unwrap() []error {
	collection := []error{}
	for _, err := range ec {
		collection = append(collection, err)
	}
	return collection
}

// Collect adds a single error or another collector to
// the collection of errors
func (ec *ErrorCollector) Collect(err error) {
	if err == nil {
		return
	}
	if castCollector, ok := err.(unwrapper); ok {
		*ec = append(*ec, castCollector.unwrap()...)
	} else {
		*ec = append(*ec, err)
	}
}

func (ec ErrorCollector) Error() string {
	collection := []string{}
	for _, err := range ec {
		collection = append(collection, err.Error())
	}
	return fmt.Sprintf("collected errors: %s", strings.Join(collection, ", "))
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package examples_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/validation"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/golang/glog"
)

func validateObject(obj runtime.Object) (errors []error) {
	ctx := api.NewDefaultContext()
	switch t := obj.(type) {
	case *api.ReplicationController:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		errors = validation.ValidateReplicationController(t)
	case *api.ReplicationControllerList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Service:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		api.ValidNamespace(ctx, &t.ObjectMeta)
		errors = validation.ValidateService(t)
	case *api.ServiceList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Pod:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		api.ValidNamespace(ctx, &t.ObjectMeta)
		errors = validation.ValidatePod(t)
	case *api.PodList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	default:
		return []error{fmt.Errorf("no validation defined for %#v", obj)}
	}
	return errors
}

func walkJSONFiles(inDir string, fn func(name, path string, data []byte)) error {
	return filepath.Walk(inDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.IsDir() && path != inDir {
			return filepath.SkipDir
		}

		file := filepath.Base(path)
		if ext := filepath.Ext(file); ext == ".json" || ext == ".yaml" {
			glog.Infof("Testing %s", path)
			data, err := ioutil.ReadFile(path)
			if err != nil {
				return err
			}
			name := strings.TrimSuffix(file, ext)
			fn(name, path, data)
		}
		return nil
	})
}

func TestExampleObjectSchemas(t *testing.T) {
	cases := map[string]map[string]runtime.Object{
		"../docs/getting-started-guides": {
			"pod": &api.Pod{},
		},
		"../cmd/integration": {
			"controller": &api.ReplicationController{},
		},
		"../examples/guestbook": {
			"frontend-controller":     &api.ReplicationController{},
			"redis-slave-controller":  &api.ReplicationController{},
			"redis-master-controller": &api.ReplicationController{},
			"frontend-service":        &api.Service{},
			"redis-master-service":    &api.Service{},
			"redis-slave-service":     &api.Service{},
		},
		"../examples/guestbook/v1beta3": {
			"frontend-controller":    &api.ReplicationController{},
			"redis-slave-controller": &api.ReplicationController{},
			"redis-master":           &api.ReplicationController{},
			"frontend-service":       &api.Service{},
			"redis-master-service":   &api.Service{},
			"redis-slave-service":    &api.Service{},
		},
		"../examples/guestbook-go": {
			"guestbook-controller":    &api.ReplicationController{},
			"redis-slave-controller":  &api.ReplicationController{},
			"redis-master-controller": &api.ReplicationController{},
			"guestbook-service":       &api.Service{},
			"redis-master-service":    &api.Service{},
			"redis-slave-service":     &api.Service{},
		},
		"../examples/guestbook-go/v1beta3": {
			"guestbook-controller":    &api.ReplicationController{},
			"redis-slave-controller":  &api.ReplicationController{},
			"redis-master-controller": &api.ReplicationController{},
			"guestbook-service":       &api.Service{},
			"redis-master-service":    &api.Service{},
			"redis-slave-service":     &api.Service{},
		},
		"../examples/walkthrough": {
			"pod1": &api.Pod{},
			"pod2": &api.Pod{},
			"pod-with-http-healthcheck": &api.Pod{},
			"service":                   &api.Service{},
			"replication-controller":    &api.ReplicationController{},
		},
		"../examples/update-demo/v1beta1": {
			"kitten-rc":   &api.ReplicationController{},
			"nautilus-rc": &api.ReplicationController{},
		},
		"../examples/update-demo/v1beta3": {
			"kitten-rc":   &api.ReplicationController{},
			"nautilus-rc": &api.ReplicationController{},
		},
	}

	for path, expected := range cases {
		tested := 0
		err := walkJSONFiles(path, func(name, path string, data []byte) {
			expectedType, found := expected[name]
			if !found {
				t.Errorf("%s does not have a test case defined", path)
				return
			}
			tested += 1
			if err := latest.Codec.DecodeInto(data, expectedType); err != nil {
				t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
				return
			}
			if errors := validateObject(expectedType); len(errors) > 0 {
				t.Errorf("%s did not validate correctly: %v", path, errors)
			}
		})
		if err != nil {
			t.Errorf("Expected no error, Got %v", err)
		}
		if tested != len(expected) {
			t.Errorf("Expected %d examples, Got %d", len(expected), tested)
		}
	}
}

var sampleRegexp = regexp.MustCompile("(?ms)^```(?:(?P<type>yaml)\\w*\\n(?P<content>.+?)|\\w*\\n(?P<content>\\{.+?\\}))\\w*\\n^```")
var subsetRegexp = regexp.MustCompile("(?ms)\\.{3}")

func TestReadme(t *testing.T) {
	paths := []string{
		"../README.md",
		"../examples/walkthrough/README.md",
	}

	for _, path := range paths {
		data, err := ioutil.ReadFile(path)
		if err != nil {
			t.Errorf("Unable to read file %s: %v", path, err)
			continue
		}

		matches := sampleRegexp.FindAllStringSubmatch(string(data), -1)
		if matches == nil {
			continue
		}
		for _, match := range matches {
			var content, subtype string
			for i, name := range sampleRegexp.SubexpNames() {
				if name == "type" {
					subtype = match[i]
				}
				if name == "content" && match[i] != "" {
					content = match[i]
				}
			}
			if subtype == "yaml" && subsetRegexp.FindString(content) != "" {
				t.Logf("skipping (%s): \n%s", subtype, content)
				continue
			}

			//t.Logf("testing (%s): \n%s", subtype, content)
			expectedType := &api.Pod{}
			if err := latest.Codec.DecodeInto([]byte(content), expectedType); err != nil {
				t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(content))
				continue
			}
			if errors := validateObject(expectedType); len(errors) > 0 {
				t.Errorf("%s did not validate correctly: %v", path, errors)
			}
			_, err := latest.Codec.Encode(expectedType)
			if err != nil {
				t.Errorf("Could not encode object: %v", err)
				continue
			}
		}
	}
}
<commit_msg>Updating integration tests to test both API versions - v1beta1 and 3<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package examples_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/validation"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/golang/glog"
)

func validateObject(obj runtime.Object) (errors []error) {
	ctx := api.NewDefaultContext()
	switch t := obj.(type) {
	case *api.ReplicationController:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		errors = validation.ValidateReplicationController(t)
	case *api.ReplicationControllerList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Service:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		api.ValidNamespace(ctx, &t.ObjectMeta)
		errors = validation.ValidateService(t)
	case *api.ServiceList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Pod:
		if t.Namespace == "" {
			t.Namespace = api.NamespaceDefault
		}
		api.ValidNamespace(ctx, &t.ObjectMeta)
		errors = validation.ValidatePod(t)
	case *api.PodList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	default:
		return []error{fmt.Errorf("no validation defined for %#v", obj)}
	}
	return errors
}

func walkJSONFiles(inDir string, fn func(name, path string, data []byte)) error {
	return filepath.Walk(inDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.IsDir() && path != inDir {
			return filepath.SkipDir
		}

		file := filepath.Base(path)
		if ext := filepath.Ext(file); ext == ".json" || ext == ".yaml" {
			glog.Infof("Testing %s", path)
			data, err := ioutil.ReadFile(path)
			if err != nil {
				return err
			}
			name := strings.TrimSuffix(file, ext)
			fn(name, path, data)
		}
		return nil
	})
}

func TestExampleObjectSchemas(t *testing.T) {
	cases := map[string]map[string]runtime.Object{
		"../docs/getting-started-guides": {
			"pod": &api.Pod{},
		},
		"../cmd/integration": {
			"v1beta1-controller": &api.ReplicationController{},
			"v1beta3-controller": &api.ReplicationController{},
		},
		"../examples/guestbook": {
			"frontend-controller":     &api.ReplicationController{},
			"redis-slave-controller":  &api.ReplicationController{},
			"redis-master-controller": &api.ReplicationController{},
			"frontend-service":        &api.Service{},
			"redis-master-service":    &api.Service{},
			"redis-slave-service":     &api.Service{},
		},
		"../examples/guestbook/v1beta3": {
			"frontend-controller":    &api.ReplicationController{},
			"redis-slave-controller": &api.ReplicationController{},
			"redis-master":           &api.ReplicationController{},
			"frontend-service":       &api.Service{},
			"redis-master-service":   &api.Service{},
			"redis-slave-service":    &api.Service{},
		},
		"../examples/guestbook-go": {
			"guestbook-controller":    &api.ReplicationController{},
			"redis-slave-controller":  &api.ReplicationController{},
			"redis-master-controller": &api.ReplicationController{},
			"guestbook-service":       &api.Service{},
			"redis-master-service":    &api.Service{},
			"redis-slave-service":     &api.Service{},
		},
		"../examples/guestbook-go/v1beta3": {
			"guestbook-controller":    &api.ReplicationController{},
			"redis-slave-controller":  &api.ReplicationController{},
			"redis-master-controller": &api.ReplicationController{},
			"guestbook-service":       &api.Service{},
			"redis-master-service":    &api.Service{},
			"redis-slave-service":     &api.Service{},
		},
		"../examples/walkthrough": {
			"pod1": &api.Pod{},
			"pod2": &api.Pod{},
			"pod-with-http-healthcheck": &api.Pod{},
			"service":                   &api.Service{},
			"replication-controller":    &api.ReplicationController{},
		},
		"../examples/update-demo/v1beta1": {
			"kitten-rc":   &api.ReplicationController{},
			"nautilus-rc": &api.ReplicationController{},
		},
		"../examples/update-demo/v1beta3": {
			"kitten-rc":   &api.ReplicationController{},
			"nautilus-rc": &api.ReplicationController{},
		},
	}

	for path, expected := range cases {
		tested := 0
		err := walkJSONFiles(path, func(name, path string, data []byte) {
			expectedType, found := expected[name]
			if !found {
				t.Errorf("%s does not have a test case defined", path)
				return
			}
			tested += 1
			if err := latest.Codec.DecodeInto(data, expectedType); err != nil {
				t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
				return
			}
			if errors := validateObject(expectedType); len(errors) > 0 {
				t.Errorf("%s did not validate correctly: %v", path, errors)
			}
		})
		if err != nil {
			t.Errorf("Expected no error, Got %v", err)
		}
		if tested != len(expected) {
			t.Errorf("Expected %d examples, Got %d", len(expected), tested)
		}
	}
}

var sampleRegexp = regexp.MustCompile("(?ms)^```(?:(?P<type>yaml)\\w*\\n(?P<content>.+?)|\\w*\\n(?P<content>\\{.+?\\}))\\w*\\n^```")
var subsetRegexp = regexp.MustCompile("(?ms)\\.{3}")

func TestReadme(t *testing.T) {
	paths := []string{
		"../README.md",
		"../examples/walkthrough/README.md",
	}

	for _, path := range paths {
		data, err := ioutil.ReadFile(path)
		if err != nil {
			t.Errorf("Unable to read file %s: %v", path, err)
			continue
		}

		matches := sampleRegexp.FindAllStringSubmatch(string(data), -1)
		if matches == nil {
			continue
		}
		for _, match := range matches {
			var content, subtype string
			for i, name := range sampleRegexp.SubexpNames() {
				if name == "type" {
					subtype = match[i]
				}
				if name == "content" && match[i] != "" {
					content = match[i]
				}
			}
			if subtype == "yaml" && subsetRegexp.FindString(content) != "" {
				t.Logf("skipping (%s): \n%s", subtype, content)
				continue
			}

			//t.Logf("testing (%s): \n%s", subtype, content)
			expectedType := &api.Pod{}
			if err := latest.Codec.DecodeInto([]byte(content), expectedType); err != nil {
				t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(content))
				continue
			}
			if errors := validateObject(expectedType); len(errors) > 0 {
				t.Errorf("%s did not validate correctly: %v", path, errors)
			}
			_, err := latest.Codec.Encode(expectedType)
			if err != nil {
				t.Errorf("Could not encode object: %v", err)
				continue
			}
		}
	}
}
<|endoftext|>