<commit_before>package base

import (
	"archive/tar"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"reflect"
)

const (
	SerializationFormatVersion = "golearn 1.0"
)

// FunctionalTarReader allows you to read anything in a tar file in any order, rather than just
// sequentially.
type FunctionalTarReader struct {
	Regenerate func() *tar.Reader
}

// NewFunctionalTarReader creates a new FunctionalTarReader using a function that it can call
// to get a tar.Reader at the beginning of the file.
func NewFunctionalTarReader(regenFunc func() *tar.Reader) *FunctionalTarReader {
	return &FunctionalTarReader{
		regenFunc,
	}
}

// GetNamedFile returns a file named a given thing from the tar file. If there's more than one
// entry, the most recent is returned.
func (f *FunctionalTarReader) GetNamedFile(name string) ([]byte, error) {
	tr := f.Regenerate()

	var returnCandidate []byte = nil
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}

		if hdr.Name == name {
			ret, err := ioutil.ReadAll(tr)
			if err != nil {
				return nil, WrapError(err)
			}
			if int64(len(ret)) != hdr.Size {
				if int64(len(ret)) < hdr.Size {
					log.Printf("Size mismatch, got %d byte(s) for %s, expected %d (err was %s)", len(ret), hdr.Name, hdr.Size, err)
				} else {
					return nil, WrapError(fmt.Errorf("Size mismatch, expected %d byte(s) for %s, got %d", len(ret), hdr.Name, hdr.Size))
				}
			}
			if err != nil {
				return nil, err
			}
			returnCandidate = ret
		}
	}
	if returnCandidate == nil {
		return nil, WrapError(fmt.Errorf("Not found (looking for %s)", name))
	}
	return returnCandidate, nil
}

func tarPrefix(prefix string, suffix string) string {
	if prefix == "" {
		return suffix
	}
	return fmt.Sprintf("%s/%s")
}

// ClassifierMetadataV1 is what gets written into METADATA
// in a classification file format.
type ClassifierMetadataV1 struct {
	// FormatVersion should always be 1 for this structure
	FormatVersion int `json:"format_version"`
	// Uses the classifier name (provided by the classifier)
	ClassifierName string `json:"classifier"`
	// ClassifierVersion is also provided by the classifier
	// and checks whether this version of GoLearn can read what's
	// be written.
	ClassifierVersion string `json"classifier_version"`
	// This is a custom metadata field, provided by the classifier
	ClassifierMetadata map[string]interface{} `json:"classifier_metadata"`
}

// ClassifierDeserializer attaches helper functions useful for reading classificatiers. (UNSTABLE).
type ClassifierDeserializer struct {
	gzipReader io.Reader
	fileReader io.ReadCloser
	tarReader  *FunctionalTarReader
	Metadata   *ClassifierMetadataV1
}

// Prefix outputs a string in the right format for TAR
func (c *ClassifierDeserializer) Prefix(prefix string, suffix string) string {
	if prefix == "" {
		return suffix
	}
	return fmt.Sprintf("%s/%s", prefix, suffix)
}

// ReadMetadataAtPrefix reads the METADATA file after prefix. If an error is returned, the first value is undefined.
func (c *ClassifierDeserializer) ReadMetadataAtPrefix(prefix string) (ClassifierMetadataV1, error) {
	var ret ClassifierMetadataV1
	err := c.GetJSONForKey(c.Prefix(prefix, "METADATA"), &ret)
	return ret, err
}

// ReadSerializedClassifierStub is the counterpart of CreateSerializedClassifierStub.
// It's used inside SaveableClassifiers to read information from a perviously saved
// model file.
func ReadSerializedClassifierStub(filePath string) (*ClassifierDeserializer, error) {

	f, err := os.Open(filePath)
	if err != nil {
		return nil, DescribeError("Can't open file", err)
	}

	gzr, err := gzip.NewReader(f)
	if err != nil {
		return nil, DescribeError("Can't decompress", err)
	}

	regenerateFunc := func() *tar.Reader {
		f.Seek(0, os.SEEK_SET)
		gzr.Reset(f)
		tz := tar.NewReader(gzr)
		return tz
	}

	tz := NewFunctionalTarReader(regenerateFunc)

	// Check that the serialization format is right
	// Retrieve the MANIFEST and verify
	manifestBytes, err := tz.GetNamedFile("CLS_MANIFEST")
	if err != nil {
		return nil, DescribeError("Error reading CLS_MANIFEST", err)
	}
	if !reflect.DeepEqual(manifestBytes, []byte(SerializationFormatVersion)) {
		return nil, fmt.Errorf("Unsupported CLS_MANIFEST: %s", string(manifestBytes))
	}

	//
	// Parse METADATA
	//
	var metadata ClassifierMetadataV1
	ret := &ClassifierDeserializer{
		f,
		gzr,
		tz,
		&metadata,
	}

	metadata, err = ret.ReadMetadataAtPrefix("")
	if err != nil {
		return nil, fmt.Errorf("Error whilst reading METADATA: %s", err)
	}
	ret.Metadata = &metadata

	// Check that we can understand this archive
	if metadata.FormatVersion != 1 {
		return nil, fmt.Errorf("METADATA: wrong format_version for this version of golearn")
	}

	return ret, nil
}

// GetBytesForKey returns the bytes at a given location in the output.
func (c *ClassifierDeserializer) GetBytesForKey(key string) ([]byte, error) {
	return c.tarReader.GetNamedFile(key)
}

func (c *ClassifierDeserializer) GetStringForKey(key string) (string, error) {
	b, err := c.GetBytesForKey(key)
	if err != nil {
		return "", err
	}
	return string(b), err
}

// GetJSONForKey deserializes a JSON key in the output file.
func (c *ClassifierDeserializer) GetJSONForKey(key string, v interface{}) error {
	b, err := c.GetBytesForKey(key)
	if err != nil {
		return err
	}
	return json.Unmarshal(b, v)
}

// GetInstancesForKey deserializes some instances stored in a classifier output file
func (c *ClassifierDeserializer) GetInstancesForKey(key string) (FixedDataGrid, error) {
	return DeserializeInstancesFromTarReader(c.tarReader, key)
}

// GetUInt64ForKey returns a int64 stored at a given key
func (c *ClassifierDeserializer) GetU64ForKey(key string) (uint64, error) {
	b, err := c.GetBytesForKey(key)
	if err != nil {
		return 0, err
	}
	return UnpackBytesToU64(b), nil
}

// GetAttributeForKey returns an Attribute stored at a given key
func (c *ClassifierDeserializer) GetAttributeForKey(key string) (Attribute, error) {
	b, err := c.GetBytesForKey(key)
	if err != nil {
		return nil, WrapError(err)
	}
	attr, err := DeserializeAttribute(b)
	if err != nil {
		return nil, WrapError(err)
	}
	return attr, nil
}

// GetAttributesForKey returns an Attribute list stored at a given key
func (c *ClassifierDeserializer) GetAttributesForKey(key string) ([]Attribute, error) {

	attrCountKey := c.Prefix(key, "ATTR_COUNT")
	attrCount, err := c.GetU64ForKey(attrCountKey)
	if err != nil {
		return nil, DescribeError("Unable to read ATTR_COUNT", err)
	}

	ret := make([]Attribute, attrCount)

	for i := range ret {
		attrKey := c.Prefix(key, fmt.Sprintf("%d", i))
		ret[i], err = c.GetAttributeForKey(attrKey)
		if err != nil {
			return nil, DescribeError("Unable to read Attribute", err)
		}
	}
	return ret, nil
}

// Close cleans up everything.
func (c *ClassifierDeserializer) Close() {
	c.fileReader.Close()
}

// ClassifierSerializer is an object used by SaveableClassifiers.
type ClassifierSerializer struct {
	gzipWriter *gzip.Writer
	fileWriter *os.File
	tarWriter  *tar.Writer
	f          *os.File
	filePath   string
}

// Close finalizes the Classifier serialization session.
func (c *ClassifierSerializer) Close() error {

	// Finally, close and flush the various levels
	if err := c.tarWriter.Flush(); err != nil {
		return fmt.Errorf("Could not flush tar: %s", err)
	}

	if err := c.tarWriter.Close(); err != nil {
		return fmt.Errorf("Could not close tar: %s", err)
	}

	if err := c.gzipWriter.Flush(); err != nil {
		return fmt.Errorf("Could not flush gz: %s", err)
	}

	if err := c.gzipWriter.Close(); err != nil {
		return fmt.Errorf("Could not close gz: %s", err)
	}

	if err := c.fileWriter.Sync(); err != nil {
		return fmt.Errorf("Could not close file writer: %s", err)
	}

	if err := c.fileWriter.Close(); err != nil {
		return fmt.Errorf("Could not close file writer: %s", err)
	}

	return nil
}

// WriteBytesForKey creates a new entry in the serializer file with some user-defined bytes.
func (c *ClassifierSerializer) WriteBytesForKey(key string, b []byte) error {

	//
	// Write header for key
	//
	hdr := &tar.Header{
		Name: key,
		Size: int64(len(b)),
	}

	if err := c.tarWriter.WriteHeader(hdr); err != nil {
		return fmt.Errorf("Could not write header for '%s': %s", key, err)
	}
	//
	// Write data
	//
	if _, err := c.tarWriter.Write(b); err != nil {
		return fmt.Errorf("Could not write data for '%s': %s", key, err)
	}

	c.tarWriter.Flush()
	c.gzipWriter.Flush()
	c.fileWriter.Sync()
	return nil
}

// WriteU64ForKey creates a new entry in the serializer file with the bytes of a uint64
func (c *ClassifierSerializer) WriteU64ForKey(key string, v uint64) error {
	b := PackU64ToBytes(v)
	return c.WriteBytesForKey(key, b)
}

// WriteJSONForKey creates a new entry in the file with an interface serialized as JSON.
func (c *ClassifierSerializer) WriteJSONForKey(key string, v interface{}) error {

	b, err := json.Marshal(v)
	if err != nil {
		return err
	}

	return c.WriteBytesForKey(key, b)

}

// WriteAttributeForKey creates a new entry in the file containing a serialized representation of Attribute
func (c *ClassifierSerializer) WriteAttributeForKey(key string, a Attribute) error {
	b, err := SerializeAttribute(a)
	if err != nil {
		return WrapError(err)
	}
	return c.WriteBytesForKey(key, b)
}

// WriteAttributesForKey does the same as WriteAttributeForKey, just with more than one Attribute.
func (c *ClassifierSerializer) WriteAttributesForKey(key string, attrs []Attribute) error {

	attrCountKey := c.Prefix(key, "ATTR_COUNT")
	err := c.WriteU64ForKey(attrCountKey, uint64(len(attrs)))
	if err != nil {
		return DescribeError("Unable to write ATTR_COUNT", err)
	}
	for i, a := range attrs {
		attrKey := c.Prefix(key, fmt.Sprintf("%d", i))
		err = c.WriteAttributeForKey(attrKey, a)
		if err != nil {
			return DescribeError("Unable to write Attribute", err)
		}
	}
	return nil
}

// WriteInstances for key creates a new entry in the file containing some training instances
func (c *ClassifierSerializer) WriteInstancesForKey(key string, g FixedDataGrid, includeData bool) error {
	fmt.Sprintf("%v", c)
	return SerializeInstancesToTarWriter(g, c.tarWriter, key, includeData)
}

// Prefix outputs a string in the right format for TAR
func (c *ClassifierSerializer) Prefix(prefix string, suffix string) string {
	if prefix == "" {
		return suffix
	}
	return fmt.Sprintf("%s/%s", prefix, suffix)
}

// WriteMetadataAtPrefix outputs a METADATA entry in the right place
func (c *ClassifierSerializer) WriteMetadataAtPrefix(prefix string, metadata ClassifierMetadataV1) error {
	return c.WriteJSONForKey(c.Prefix(prefix, "METADATA"), &metadata)
}

// CreateSerializedClassifierStub generates a file to serialize into
// and writes the METADATA header.
func CreateSerializedClassifierStub(filePath string, metadata ClassifierMetadataV1) (*ClassifierSerializer, error) {

	// Open the filePath
	f, err := os.OpenFile(filePath, os.O_RDWR|os.O_TRUNC|os.O_CREAT, 0600)
	if err != nil {
		return nil, err
	}

	var hdr *tar.Header
	gzWriter := gzip.NewWriter(f)
	tw := tar.NewWriter(gzWriter)

	ret := ClassifierSerializer{
		gzipWriter: gzWriter,
		fileWriter: f,
		tarWriter:  tw,
	}

	//
	// Write the MANIFEST entry
	//
	hdr = &tar.Header{
		Name: "CLS_MANIFEST",
		Size: int64(len(SerializationFormatVersion)),
	}
	if err := tw.WriteHeader(hdr); err != nil {
		return nil, fmt.Errorf("Could not write CLS_MANIFEST header: %s", err)
	}

	if _, err := tw.Write([]byte(SerializationFormatVersion)); err != nil {
		return nil, fmt.Errorf("Could not write CLS_MANIFEST contents: %s", err)
	}

	//
	// Write the METADATA entry
	//
	err = ret.WriteMetadataAtPrefix("", metadata)
	if err != nil {
		return nil, fmt.Errorf("JSON marshal error: %s", err)
	}

	return &ret, nil

}
<commit_msg>base: CREAT => CREATE<commit_after>package base

import (
	"archive/tar"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"reflect"
)

const (
	SerializationFormatVersion = "golearn 1.0"
)

// FunctionalTarReader allows you to read anything in a tar file in any order, rather than just
// sequentially.
type FunctionalTarReader struct {
	Regenerate func() *tar.Reader
}

// NewFunctionalTarReader creates a new FunctionalTarReader using a function that it can call
// to get a tar.Reader at the beginning of the file.
func NewFunctionalTarReader(regenFunc func() *tar.Reader) *FunctionalTarReader {
	return &FunctionalTarReader{
		regenFunc,
	}
}

// GetNamedFile returns a file named a given thing from the tar file. If there's more than one
// entry, the most recent is returned.
func (f *FunctionalTarReader) GetNamedFile(name string) ([]byte, error) {
	tr := f.Regenerate()

	var returnCandidate []byte = nil
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}

		if hdr.Name == name {
			ret, err := ioutil.ReadAll(tr)
			if err != nil {
				return nil, WrapError(err)
			}
			if int64(len(ret)) != hdr.Size {
				if int64(len(ret)) < hdr.Size {
					log.Printf("Size mismatch, got %d byte(s) for %s, expected %d (err was %s)", len(ret), hdr.Name, hdr.Size, err)
				} else {
					return nil, WrapError(fmt.Errorf("Size mismatch, expected %d byte(s) for %s, got %d", len(ret), hdr.Name, hdr.Size))
				}
			}
			if err != nil {
				return nil, err
			}
			returnCandidate = ret
		}
	}
	if returnCandidate == nil {
		return nil, WrapError(fmt.Errorf("Not found (looking for %s)", name))
	}
	return returnCandidate, nil
}

func tarPrefix(prefix string, suffix string) string {
	if prefix == "" {
		return suffix
	}
	return fmt.Sprintf("%s/%s")
}

// ClassifierMetadataV1 is what gets written into METADATA
// in a classification file format.
type ClassifierMetadataV1 struct {
	// FormatVersion should always be 1 for this structure
	FormatVersion int `json:"format_version"`
	// Uses the classifier name (provided by the classifier)
	ClassifierName string `json:"classifier"`
	// ClassifierVersion is also provided by the classifier
	// and checks whether this version of GoLearn can read what's
	// be written.
	ClassifierVersion string `json"classifier_version"`
	// This is a custom metadata field, provided by the classifier
	ClassifierMetadata map[string]interface{} `json:"classifier_metadata"`
}

// ClassifierDeserializer attaches helper functions useful for reading classificatiers. (UNSTABLE).
type ClassifierDeserializer struct {
	gzipReader io.Reader
	fileReader io.ReadCloser
	tarReader  *FunctionalTarReader
	Metadata   *ClassifierMetadataV1
}

// Prefix outputs a string in the right format for TAR
func (c *ClassifierDeserializer) Prefix(prefix string, suffix string) string {
	if prefix == "" {
		return suffix
	}
	return fmt.Sprintf("%s/%s", prefix, suffix)
}

// ReadMetadataAtPrefix reads the METADATA file after prefix. If an error is returned, the first value is undefined.
func (c *ClassifierDeserializer) ReadMetadataAtPrefix(prefix string) (ClassifierMetadataV1, error) {
	var ret ClassifierMetadataV1
	err := c.GetJSONForKey(c.Prefix(prefix, "METADATA"), &ret)
	return ret, err
}

// ReadSerializedClassifierStub is the counterpart of CreateSerializedClassifierStub.
// It's used inside SaveableClassifiers to read information from a perviously saved
// model file.
func ReadSerializedClassifierStub(filePath string) (*ClassifierDeserializer, error) {

	f, err := os.Open(filePath)
	if err != nil {
		return nil, DescribeError("Can't open file", err)
	}

	gzr, err := gzip.NewReader(f)
	if err != nil {
		return nil, DescribeError("Can't decompress", err)
	}

	regenerateFunc := func() *tar.Reader {
		f.Seek(0, os.SEEK_SET)
		gzr.Reset(f)
		tz := tar.NewReader(gzr)
		return tz
	}

	tz := NewFunctionalTarReader(regenerateFunc)

	// Check that the serialization format is right
	// Retrieve the MANIFEST and verify
	manifestBytes, err := tz.GetNamedFile("CLS_MANIFEST")
	if err != nil {
		return nil, DescribeError("Error reading CLS_MANIFEST", err)
	}
	if !reflect.DeepEqual(manifestBytes, []byte(SerializationFormatVersion)) {
		return nil, fmt.Errorf("Unsupported CLS_MANIFEST: %s", string(manifestBytes))
	}

	//
	// Parse METADATA
	//
	var metadata ClassifierMetadataV1
	ret := &ClassifierDeserializer{
		f,
		gzr,
		tz,
		&metadata,
	}

	metadata, err = ret.ReadMetadataAtPrefix("")
	if err != nil {
		return nil, fmt.Errorf("Error whilst reading METADATA: %s", err)
	}
	ret.Metadata = &metadata

	// Check that we can understand this archive
	if metadata.FormatVersion != 1 {
		return nil, fmt.Errorf("METADATA: wrong format_version for this version of golearn")
	}

	return ret, nil
}

// GetBytesForKey returns the bytes at a given location in the output.
func (c *ClassifierDeserializer) GetBytesForKey(key string) ([]byte, error) {
	return c.tarReader.GetNamedFile(key)
}

func (c *ClassifierDeserializer) GetStringForKey(key string) (string, error) {
	b, err := c.GetBytesForKey(key)
	if err != nil {
		return "", err
	}
	return string(b), err
}

// GetJSONForKey deserializes a JSON key in the output file.
func (c *ClassifierDeserializer) GetJSONForKey(key string, v interface{}) error {
	b, err := c.GetBytesForKey(key)
	if err != nil {
		return err
	}
	return json.Unmarshal(b, v)
}

// GetInstancesForKey deserializes some instances stored in a classifier output file
func (c *ClassifierDeserializer) GetInstancesForKey(key string) (FixedDataGrid, error) {
	return DeserializeInstancesFromTarReader(c.tarReader, key)
}

// GetUInt64ForKey returns a int64 stored at a given key
func (c *ClassifierDeserializer) GetU64ForKey(key string) (uint64, error) {
	b, err := c.GetBytesForKey(key)
	if err != nil {
		return 0, err
	}
	return UnpackBytesToU64(b), nil
}

// GetAttributeForKey returns an Attribute stored at a given key
func (c *ClassifierDeserializer) GetAttributeForKey(key string) (Attribute, error) {
	b, err := c.GetBytesForKey(key)
	if err != nil {
		return nil, WrapError(err)
	}
	attr, err := DeserializeAttribute(b)
	if err != nil {
		return nil, WrapError(err)
	}
	return attr, nil
}

// GetAttributesForKey returns an Attribute list stored at a given key
func (c *ClassifierDeserializer) GetAttributesForKey(key string) ([]Attribute, error) {

	attrCountKey := c.Prefix(key, "ATTR_COUNT")
	attrCount, err := c.GetU64ForKey(attrCountKey)
	if err != nil {
		return nil, DescribeError("Unable to read ATTR_COUNT", err)
	}

	ret := make([]Attribute, attrCount)

	for i := range ret {
		attrKey := c.Prefix(key, fmt.Sprintf("%d", i))
		ret[i], err = c.GetAttributeForKey(attrKey)
		if err != nil {
			return nil, DescribeError("Unable to read Attribute", err)
		}
	}
	return ret, nil
}

// Close cleans up everything.
func (c *ClassifierDeserializer) Close() {
	c.fileReader.Close()
}

// ClassifierSerializer is an object used by SaveableClassifiers.
type ClassifierSerializer struct {
	gzipWriter *gzip.Writer
	fileWriter *os.File
	tarWriter  *tar.Writer
	f          *os.File
	filePath   string
}

// Close finalizes the Classifier serialization session.
func (c *ClassifierSerializer) Close() error {

	// Finally, close and flush the various levels
	if err := c.tarWriter.Flush(); err != nil {
		return fmt.Errorf("Could not flush tar: %s", err)
	}

	if err := c.tarWriter.Close(); err != nil {
		return fmt.Errorf("Could not close tar: %s", err)
	}

	if err := c.gzipWriter.Flush(); err != nil {
		return fmt.Errorf("Could not flush gz: %s", err)
	}

	if err := c.gzipWriter.Close(); err != nil {
		return fmt.Errorf("Could not close gz: %s", err)
	}

	if err := c.fileWriter.Sync(); err != nil {
		return fmt.Errorf("Could not close file writer: %s", err)
	}

	if err := c.fileWriter.Close(); err != nil {
		return fmt.Errorf("Could not close file writer: %s", err)
	}

	return nil
}

// WriteBytesForKey creates a new entry in the serializer file with some user-defined bytes.
func (c *ClassifierSerializer) WriteBytesForKey(key string, b []byte) error {

	//
	// Write header for key
	//
	hdr := &tar.Header{
		Name: key,
		Size: int64(len(b)),
	}

	if err := c.tarWriter.WriteHeader(hdr); err != nil {
		return fmt.Errorf("Could not write header for '%s': %s", key, err)
	}
	//
	// Write data
	//
	if _, err := c.tarWriter.Write(b); err != nil {
		return fmt.Errorf("Could not write data for '%s': %s", key, err)
	}

	c.tarWriter.Flush()
	c.gzipWriter.Flush()
	c.fileWriter.Sync()
	return nil
}

// WriteU64ForKey creates a new entry in the serializer file with the bytes of a uint64
func (c *ClassifierSerializer) WriteU64ForKey(key string, v uint64) error {
	b := PackU64ToBytes(v)
	return c.WriteBytesForKey(key, b)
}

// WriteJSONForKey creates a new entry in the file with an interface serialized as JSON.
func (c *ClassifierSerializer) WriteJSONForKey(key string, v interface{}) error {

	b, err := json.Marshal(v)
	if err != nil {
		return err
	}

	return c.WriteBytesForKey(key, b)

}

// WriteAttributeForKey creates a new entry in the file containing a serialized representation of Attribute
func (c *ClassifierSerializer) WriteAttributeForKey(key string, a Attribute) error {
	b, err := SerializeAttribute(a)
	if err != nil {
		return WrapError(err)
	}
	return c.WriteBytesForKey(key, b)
}

// WriteAttributesForKey does the same as WriteAttributeForKey, just with more than one Attribute.
func (c *ClassifierSerializer) WriteAttributesForKey(key string, attrs []Attribute) error {

	attrCountKey := c.Prefix(key, "ATTR_COUNT")
	err := c.WriteU64ForKey(attrCountKey, uint64(len(attrs)))
	if err != nil {
		return DescribeError("Unable to write ATTR_COUNT", err)
	}
	for i, a := range attrs {
		attrKey := c.Prefix(key, fmt.Sprintf("%d", i))
		err = c.WriteAttributeForKey(attrKey, a)
		if err != nil {
			return DescribeError("Unable to write Attribute", err)
		}
	}
	return nil
}

// WriteInstances for key creates a new entry in the file containing some training instances
func (c *ClassifierSerializer) WriteInstancesForKey(key string, g FixedDataGrid, includeData bool) error {
	fmt.Sprintf("%v", c)
	return SerializeInstancesToTarWriter(g, c.tarWriter, key, includeData)
}

// Prefix outputs a string in the right format for TAR
func (c *ClassifierSerializer) Prefix(prefix string, suffix string) string {
	if prefix == "" {
		return suffix
	}
	return fmt.Sprintf("%s/%s", prefix, suffix)
}

// WriteMetadataAtPrefix outputs a METADATA entry in the right place
func (c *ClassifierSerializer) WriteMetadataAtPrefix(prefix string, metadata ClassifierMetadataV1) error {
	return c.WriteJSONForKey(c.Prefix(prefix, "METADATA"), &metadata)
}

// CreateSerializedClassifierStub generates a file to serialize into
// and writes the METADATA header.
func CreateSerializedClassifierStub(filePath string, metadata ClassifierMetadataV1) (*ClassifierSerializer, error) {

	// Open the filePath
	f, err := os.OpenFile(filePath, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0600)
	if err != nil {
		return nil, err
	}

	var hdr *tar.Header
	gzWriter := gzip.NewWriter(f)
	tw := tar.NewWriter(gzWriter)

	ret := ClassifierSerializer{
		gzipWriter: gzWriter,
		fileWriter: f,
		tarWriter:  tw,
	}

	//
	// Write the MANIFEST entry
	//
	hdr = &tar.Header{
		Name: "CLS_MANIFEST",
		Size: int64(len(SerializationFormatVersion)),
	}
	if err := tw.WriteHeader(hdr); err != nil {
		return nil, fmt.Errorf("Could not write CLS_MANIFEST header: %s", err)
	}

	if _, err := tw.Write([]byte(SerializationFormatVersion)); err != nil {
		return nil, fmt.Errorf("Could not write CLS_MANIFEST contents: %s", err)
	}

	//
	// Write the METADATA entry
	//
	err = ret.WriteMetadataAtPrefix("", metadata)
	if err != nil {
		return nil, fmt.Errorf("JSON marshal error: %s", err)
	}

	return &ret, nil

}
<|endoftext|>


<commit_before>// Copyright 2013 Landon Wainwright. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Starts up the blog system using the default values
package main

import (
	"flag"
	"github.com/landonia/simplegoblog/blog"
	"log"
	"os"
)

// Starts a new simple go blog server
func main() {

	// Define flags
	var postsdir, templatesdir, assetsdir string
	flag.StringVar(&postsdir, "pdir", "../posts", "the directory for storing the posts")
	flag.StringVar(&templatesdir, "tdir", "../templates", "the directory containing the templates")
	flag.StringVar(&assetsdir, "adir", "../assets", "the directory containing the assets")
	flag.Parse()

	// Create a new configuration containing the info
	config := &blog.Configuration{Title: "Life thru a Lando", DevelopmentMode: true, Postsdir: postsdir, Templatesdir: templatesdir, Assetsdir: assetsdir}

	// Create a new data structure for storing the data
	b := blog.New(config)

	// Start the blog server
	err := b.Start(":8080")
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}
}
<commit_msg>Allow the port to be defined<commit_after>// Copyright 2013 Landon Wainwright. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Starts up the blog system using the default values
package main

import (
	"flag"
	"github.com/landonia/simplegoblog/blog"
	"log"
	"os"
)

// Starts a new simple go blog server
func main() {

	// Define flags
	var postsdir, templatesdir, assetsdir, port string
	flag.StringVar(&postsdir, "pdir", "../posts", "the directory for storing the posts")
	flag.StringVar(&templatesdir, "tdir", "../templates", "the directory containing the templates")
	flag.StringVar(&assetsdir, "adir", "../assets", "the directory containing the assets")
	flag.StringVar(&port, "port", "8080", "the port to run the blog on")
	flag.Parse()

	// Create a new configuration containing the info
	config := &blog.Configuration{Title: "Life thru a Lando", DevelopmentMode: true, Postsdir: postsdir, Templatesdir: templatesdir, Assetsdir: assetsdir}

	// Create a new data structure for storing the data
	b := blog.New(config)

	// Start the blog server
	err := b.Start(port)
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package lang_test;

import (
    "fmt"
    "math/rand"
    "testing"
    "github.com/stretchr/testify/assert"
    "github.com/michael-golfi/rules/lang"
)

func TestLexIdentifier(t *testing.T) {
    assertLex(t, "t", "Identifier(t)")
    assertLex(t, "t12", "Identifier(t12)")
    assertLex(t, "_", "Identifier(_)")
    assertLex(t, "_12", "Identifier(_12)")
    assertLex(t, "test", "Identifier(test)")
}

func TestLexIndentation(t *testing.T) {
    assertLex(t, "    test", "Indentation(    )", "Identifier(test)")
    assertLex(t, "\ntest", "Identifier(test)")
    assertLex(t, "\r\ntest", "Identifier(test)")
    assertLex(t, "\r    test", "Indentation(    )", "Identifier(test)")
    assertLex(t, "\r\ttest", "Indentation(\t)", "Identifier(test)")
    assertLex(t, "test\nyou", "Identifier(test)", "Identifier(you)")
    assertLex(t, "hello\n you", "Identifier(hello)", "Indentation( )", "Identifier(you)")
    assertLex(t, " hello\n you", "Indentation( )", "Identifier(hello)", "Indentation( )", "Identifier(you)")
    assertLex(t, " \\\ntest", "Indentation( )", "Identifier(test)")
}

func TestLexTerminator(t *testing.T) {
    assertLex(t, "test;that", "Identifier(test)", "Terminator(;)", "Identifier(that)")
}

func TestLexKeyword(t *testing.T) {
    for _, keyword := range lang.KEYWORDS {
        stringKeword := string(keyword)
        assertLex(t, stringKeword, fmt.Sprintf("Keyword(%s)", stringKeword))
    }
}

func TestLexSymbol(t *testing.T) {
    for _, symbol := range lang.SYMBOLS {
        compositeSymbol := []rune{symbol}
        length := rand.Intn(3)
        for i := 0; i < length; i++ {
            compositeSymbol = append(compositeSymbol, lang.SYMBOLS[rand.Intn(len(lang.SYMBOLS))])
        }
        stringSymbol := string(compositeSymbol)
        assertLex(t, stringSymbol, fmt.Sprintf("Symbol(%s)", stringSymbol))
    }
}

func TestLexBooleanLiteral(t *testing.T) {
    assertLex(t, "false", "BooleanLiteral(false)")
    assertLex(t, "true", "BooleanLiteral(true)")
}

func TestLextStringLiteral(t *testing.T) {
    assertLex(t, "\"test\"", "StringLiteral(\"test\")")
    assertLex(t, "\"  \"", "StringLiteral(\"  \")")
    assertLex(t, "\"  t  \"", "StringLiteral(\"  t  \")")
    assertLex(t, "\"te\\nst\"", "StringLiteral(\"te\\nst\")")
    assertLex(t, "\"te\\\"nst\"", "StringLiteral(\"te\\\"nst\")")
    assertLex(t, "\"te\\\\st\"", "StringLiteral(\"te\\\\st\")")
    assertLex(t, "\"te\\\\nst\"", "StringLiteral(\"te\\\\nst\")")
    assertLex(t, "\"\\u00000000\"", "StringLiteral(\"\\u00000000\")")
    assertLex(t, "\"\\u0\"", "StringLiteral(\"\\u0\")")
    assertLex(t, "\"\\u214ade\"", "StringLiteral(\"\\u214ade\")")
    assertLex(t, "\"\\u214ader\"", "StringLiteral(\"\\u214ader\")")
}

func TestLexBinaryIntegerLiteral(t *testing.T) {
    assertLex(t, "0b0", "BinaryIntegerLiteral(0b0)")
    assertLex(t, "0b11", "BinaryIntegerLiteral(0b11)")
    assertLex(t, "0B110101", "BinaryIntegerLiteral(0B110101)")
    assertLex(t, "0b1101_0001", "BinaryIntegerLiteral(0b1101_0001)")
    assertLex(t, "0b1101__0001", "BinaryIntegerLiteral(0b1101__0001)")
    assertLex(t, "0b1101_0100_0001", "BinaryIntegerLiteral(0b1101_0100_0001)")
    assertLex(t, "0b1101_0100____0001", "BinaryIntegerLiteral(0b1101_0100____0001)")
}

func TestLexDecimalIntegerLiteral(t *testing.T) {
    assertLex(t, "0", "DecimalIntegerLiteral(0)")
    assertLex(t, "012", "DecimalIntegerLiteral(012)")
    assertLex(t, "564", "DecimalIntegerLiteral(564)")
    assertLex(t, "5_000", "DecimalIntegerLiteral(5_000)")
    assertLex(t, "5__000", "DecimalIntegerLiteral(5__000)")
    assertLex(t, "5_000_000", "DecimalIntegerLiteral(5_000_000)")
    assertLex(t, "5_000____000", "DecimalIntegerLiteral(5_000____000)")
}

func TestLexHexadecimalIntegerLiteral(t *testing.T) {
    assertLex(t, "0x0", "HexadecimalIntegerLiteral(0x0)")
    assertLex(t, "0xAf", "HexadecimalIntegerLiteral(0xAf)")
    assertLex(t, "0XA24E4", "HexadecimalIntegerLiteral(0XA24E4)")
    assertLex(t, "0xDEAD_BEEF", "HexadecimalIntegerLiteral(0xDEAD_BEEF)")
    assertLex(t, "0x2192__CAFE", "HexadecimalIntegerLiteral(0x2192__CAFE)")
    assertLex(t, "0xBABE_291c_13b2", "HexadecimalIntegerLiteral(0xBABE_291c_13b2)")
    assertLex(t, "0x4235_1232____54fd3", "HexadecimalIntegerLiteral(0x4235_1232____54fd3)")
}

func TestLexFloatLiteral(t *testing.T) {
    assertLex(t, "1.", "FloatLiteral(1.)");
    assertLex(t, "12_345_689.", "FloatLiteral(12_345_689.)");
    assertLex(t, "1.0", "FloatLiteral(1.0)");
    assertLex(t, ".1", "FloatLiteral(.1)");
    assertLex(t, ".14321", "FloatLiteral(.14321)");
    assertLex(t, "1e2", "FloatLiteral(1e2)");
    assertLex(t, "1.e2", "FloatLiteral(1.e2)");
    assertLex(t, "1.0e2", "FloatLiteral(1.0e2)");
    assertLex(t, "113.211e21", "FloatLiteral(113.211e21)");
    assertLex(t, ".1e2", "FloatLiteral(.1e2)");
    assertLex(t, "1e-2", "FloatLiteral(1e-2)");
    assertLex(t, "1.e+2", "FloatLiteral(1.e+2)");
    assertLex(t, "1.0e-2", "FloatLiteral(1.0e-2)");
    assertLex(t, ".1e+2", "FloatLiteral(.1e+2)");
    assertLex(t, "1_113.291_121e9", "FloatLiteral(1_113.291_121e9)");
}

func TestLexIgnored(t *testing.T) {
    assertLex(t, "test\\\nyou", "Identifier(test)", "Identifier(you)")
    assertLex(t, "#A \tcomment!\ntest", "Identifier(test)")
    assertLex(t, "#Testing #some characters in \\comments\ntest", "Identifier(test)")
    assertLex(t, "## Comment\\test ## test", "Identifier(test)")
    assertLex(t, "## Two ## #Comments\n test", "Indentation( )", "Identifier(test)")
    assertLex(t, "## Hello \n World \r\t\n ##test", "Identifier(test)")
    assertLex(t, "### You\nCan ## Nest\nComments # like this ## ###test", "Identifier(test)")
}

func assertLex(t *testing.T, source string, expected ...string) {
    tokenizer := lang.StringTokenizer(source)
    tokens := []string{}
    for tokenizer.Has() {
        tokens = append(tokens, string(tokenizer.Head().String()))
        tokenizer.Advance()
    }
    assert.Equal(t, expected, tokens)
}
<commit_msg>Add test for lexing a generic program<commit_after>package lang_test

import (
    "fmt"
    "math/rand"
    "testing"
    "github.com/stretchr/testify/assert"
    "github.com/michael-golfi/rules/lang"
)

func TestLexIdentifier(t *testing.T) {
    assertLex(t, "t", "Identifier(t)")
    assertLex(t, "t12", "Identifier(t12)")
    assertLex(t, "_", "Identifier(_)")
    assertLex(t, "_12", "Identifier(_12)")
    assertLex(t, "test", "Identifier(test)")
}

func TestLexIndentation(t *testing.T) {
    assertLex(t, "    test", "Indentation(    )", "Identifier(test)")
    assertLex(t, "\ntest", "Identifier(test)")
    assertLex(t, "\r\ntest", "Identifier(test)")
    assertLex(t, "\r    test", "Indentation(    )", "Identifier(test)")
    assertLex(t, "\r\ttest", "Indentation(\t)", "Identifier(test)")
    assertLex(t, "test\nyou", "Identifier(test)", "Identifier(you)")
    assertLex(t, "hello\n you", "Identifier(hello)", "Indentation( )", "Identifier(you)")
    assertLex(t, " hello\n you", "Indentation( )", "Identifier(hello)", "Indentation( )", "Identifier(you)")
    assertLex(t, " \\\ntest", "Indentation( )", "Identifier(test)")
}

func TestLexTerminator(t *testing.T) {
    assertLex(t, "test;that", "Identifier(test)", "Terminator(;)", "Identifier(that)")
}

func TestLexKeyword(t *testing.T) {
    for _, keyword := range lang.KEYWORDS {
        stringKeword := string(keyword)
        assertLex(t, stringKeword, fmt.Sprintf("Keyword(%s)", stringKeword))
    }
}

func TestLexSymbol(t *testing.T) {
    for _, symbol := range lang.SYMBOLS {
        compositeSymbol := []rune{symbol}
        length := rand.Intn(3)
        for i := 0; i < length; i++ {
            compositeSymbol = append(compositeSymbol, lang.SYMBOLS[rand.Intn(len(lang.SYMBOLS))])
        }
        stringSymbol := string(compositeSymbol)
        assertLex(t, stringSymbol, fmt.Sprintf("Symbol(%s)", stringSymbol))
    }
}

func TestLexBooleanLiteral(t *testing.T) {
    assertLex(t, "false", "BooleanLiteral(false)")
    assertLex(t, "true", "BooleanLiteral(true)")
}

func TestLextStringLiteral(t *testing.T) {
    assertLex(t, "\"test\"", "StringLiteral(\"test\")")
    assertLex(t, "\"  \"", "StringLiteral(\"  \")")
    assertLex(t, "\"  t  \"", "StringLiteral(\"  t  \")")
    assertLex(t, "\"te\\nst\"", "StringLiteral(\"te\\nst\")")
    assertLex(t, "\"te\\\"nst\"", "StringLiteral(\"te\\\"nst\")")
    assertLex(t, "\"te\\\\st\"", "StringLiteral(\"te\\\\st\")")
    assertLex(t, "\"te\\\\nst\"", "StringLiteral(\"te\\\\nst\")")
    assertLex(t, "\"\\u00000000\"", "StringLiteral(\"\\u00000000\")")
    assertLex(t, "\"\\u0\"", "StringLiteral(\"\\u0\")")
    assertLex(t, "\"\\u214ade\"", "StringLiteral(\"\\u214ade\")")
    assertLex(t, "\"\\u214ader\"", "StringLiteral(\"\\u214ader\")")
}

func TestLexBinaryIntegerLiteral(t *testing.T) {
    assertLex(t, "0b0", "BinaryIntegerLiteral(0b0)")
    assertLex(t, "0b11", "BinaryIntegerLiteral(0b11)")
    assertLex(t, "0B110101", "BinaryIntegerLiteral(0B110101)")
    assertLex(t, "0b1101_0001", "BinaryIntegerLiteral(0b1101_0001)")
    assertLex(t, "0b1101__0001", "BinaryIntegerLiteral(0b1101__0001)")
    assertLex(t, "0b1101_0100_0001", "BinaryIntegerLiteral(0b1101_0100_0001)")
    assertLex(t, "0b1101_0100____0001", "BinaryIntegerLiteral(0b1101_0100____0001)")
}

func TestLexDecimalIntegerLiteral(t *testing.T) {
    assertLex(t, "0", "DecimalIntegerLiteral(0)")
    assertLex(t, "012", "DecimalIntegerLiteral(012)")
    assertLex(t, "564", "DecimalIntegerLiteral(564)")
    assertLex(t, "5_000", "DecimalIntegerLiteral(5_000)")
    assertLex(t, "5__000", "DecimalIntegerLiteral(5__000)")
    assertLex(t, "5_000_000", "DecimalIntegerLiteral(5_000_000)")
    assertLex(t, "5_000____000", "DecimalIntegerLiteral(5_000____000)")
}

func TestLexHexadecimalIntegerLiteral(t *testing.T) {
    assertLex(t, "0x0", "HexadecimalIntegerLiteral(0x0)")
    assertLex(t, "0xAf", "HexadecimalIntegerLiteral(0xAf)")
    assertLex(t, "0XA24E4", "HexadecimalIntegerLiteral(0XA24E4)")
    assertLex(t, "0xDEAD_BEEF", "HexadecimalIntegerLiteral(0xDEAD_BEEF)")
    assertLex(t, "0x2192__CAFE", "HexadecimalIntegerLiteral(0x2192__CAFE)")
    assertLex(t, "0xBABE_291c_13b2", "HexadecimalIntegerLiteral(0xBABE_291c_13b2)")
    assertLex(t, "0x4235_1232____54fd3", "HexadecimalIntegerLiteral(0x4235_1232____54fd3)")
}

func TestLexFloatLiteral(t *testing.T) {
    assertLex(t, "1.", "FloatLiteral(1.)");
    assertLex(t, "12_345_689.", "FloatLiteral(12_345_689.)");
    assertLex(t, "1.0", "FloatLiteral(1.0)");
    assertLex(t, ".1", "FloatLiteral(.1)");
    assertLex(t, ".14321", "FloatLiteral(.14321)");
    assertLex(t, "1e2", "FloatLiteral(1e2)");
    assertLex(t, "1.e2", "FloatLiteral(1.e2)");
    assertLex(t, "1.0e2", "FloatLiteral(1.0e2)");
    assertLex(t, "113.211e21", "FloatLiteral(113.211e21)");
    assertLex(t, ".1e2", "FloatLiteral(.1e2)");
    assertLex(t, "1e-2", "FloatLiteral(1e-2)");
    assertLex(t, "1.e+2", "FloatLiteral(1.e+2)");
    assertLex(t, "1.0e-2", "FloatLiteral(1.0e-2)");
    assertLex(t, ".1e+2", "FloatLiteral(.1e+2)");
    assertLex(t, "1_113.291_121e9", "FloatLiteral(1_113.291_121e9)");
}

func TestLexIgnored(t *testing.T) {
    assertLex(t, "test\\\nyou", "Identifier(test)", "Identifier(you)")
    assertLex(t, "#A \tcomment!\ntest", "Identifier(test)")
    assertLex(t, "#Testing #some characters in \\comments\ntest", "Identifier(test)")
    assertLex(t, "## Comment\\test ## test", "Identifier(test)")
    assertLex(t, "## Two ## #Comments\n test", "Indentation( )", "Identifier(test)")
    assertLex(t, "## Hello \n World \r\t\n ##test", "Identifier(test)")
    assertLex(t, "### You\nCan ## Nest\nComments # like this ## ###test", "Identifier(test)")
}

func TestLexGenericProgram(t *testing.T) {
    // Not necessarily representative of the actual language, just to test the lexer
    assertLex(t,
        "# Computes and prints the factorial of n\n" +
        "let n := 12; var fact := 1\n" +
        "for var i := 2; i <= n; i += 1\n" +
        "    fact *= i\n" +
        "printfln(\"%d! is %d\", n, fact)\n" +
        "## Random block comment ##",
        "Keyword(let)", "Identifier(n)", "Symbol(:=)", "DecimalIntegerLiteral(12)", "Terminator(;)",
        "Keyword(var)", "Identifier(fact)", "Symbol(:=)", "DecimalIntegerLiteral(1)",
        "Keyword(for)", "Keyword(var)", "Identifier(i)", "Symbol(:=)", "DecimalIntegerLiteral(2)", "Terminator(;)",
        "Identifier(i)", "Symbol(<=)", "Identifier(n)", "Terminator(;)",
        "Identifier(i)", "Symbol(+=)", "DecimalIntegerLiteral(1)",
        "Indentation(    )", "Identifier(fact)", "Symbol(*=)", "Identifier(i)",
        "Identifier(printfln)", "Symbol(()", "StringLiteral(\"%d! is %d\")", "Symbol(,)", "Identifier(n)", "Symbol(,)", "Identifier(fact)", "Symbol())",
    )
}

func assertLex(t *testing.T, source string, expected ...string) {
    tokenizer := lang.StringTokenizer(source)
    tokens := []string{}
    for tokenizer.Has() {
        tokens = append(tokens, string(tokenizer.Head().String()))
        tokenizer.Advance()
    }
    assert.Equal(t, expected, tokens)
}
<|endoftext|>


<commit_before>// Copyright ©2016 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gonum

import (
	"gonum.org/v1/gonum/blas"
	"gonum.org/v1/gonum/blas/blas64"
	"gonum.org/v1/gonum/lapack"
)

// Dgehrd reduces a block of a real n×n general matrix A to upper Hessenberg
// form H by an orthogonal similarity transformation Q^T * A * Q = H.
//
// The matrix Q is represented as a product of (ihi-ilo) elementary
// reflectors
//  Q = H_{ilo} H_{ilo+1} ... H_{ihi-1}.
// Each H_i has the form
//  H_i = I - tau[i] * v * v^T
// where v is a real vector with v[0:i+1] = 0, v[i+1] = 1 and v[ihi+1:n] = 0.
// v[i+2:ihi+1] is stored on exit in A[i+2:ihi+1,i].
//
// On entry, a contains the n×n general matrix to be reduced. On return, the
// upper triangle and the first subdiagonal of A will be overwritten with the
// upper Hessenberg matrix H, and the elements below the first subdiagonal, with
// the slice tau, represent the orthogonal matrix Q as a product of elementary
// reflectors.
//
// The contents of a are illustrated by the following example, with n = 7, ilo =
// 1 and ihi = 5.
// On entry,
//  [ a   a   a   a   a   a   a ]
//  [     a   a   a   a   a   a ]
//  [     a   a   a   a   a   a ]
//  [     a   a   a   a   a   a ]
//  [     a   a   a   a   a   a ]
//  [     a   a   a   a   a   a ]
//  [                         a ]
// on return,
//  [ a   a   h   h   h   h   a ]
//  [     a   h   h   h   h   a ]
//  [     h   h   h   h   h   h ]
//  [     v1  h   h   h   h   h ]
//  [     v1  v2  h   h   h   h ]
//  [     v1  v2  v3  h   h   h ]
//  [                         a ]
// where a denotes an element of the original matrix A, h denotes a
// modified element of the upper Hessenberg matrix H, and vi denotes an
// element of the vector defining H_i.
//
// ilo and ihi determine the block of A that will be reduced to upper Hessenberg
// form. It must hold that 0 <= ilo <= ihi < n if n > 0, and ilo == 0 and ihi ==
// -1 if n == 0, otherwise Dgehrd will panic.
//
// On return, tau will contain the scalar factors of the elementary reflectors.
// Elements tau[:ilo] and tau[ihi:] will be set to zero. tau must have length
// equal to n-1 if n > 0, otherwise Dgehrd will panic.
//
// work must have length at least lwork and lwork must be at least max(1,n),
// otherwise Dgehrd will panic. On return, work[0] contains the optimal value of
// lwork.
//
// If lwork == -1, instead of performing Dgehrd, only the optimal value of lwork
// will be stored in work[0].
//
// Dgehrd is an internal routine. It is exported for testing purposes.
func (impl Implementation) Dgehrd(n, ilo, ihi int, a []float64, lda int, tau, work []float64, lwork int) {
	switch {
	case ilo < 0 || max(0, n-1) < ilo:
		panic(badIlo)
	case ihi < min(ilo, n-1) || n <= ihi:
		panic(badIhi)
	case lwork < max(1, n) && lwork != -1:
		panic(badWork)
	case len(work) < lwork:
		panic(shortWork)
	}
	if lwork != -1 {
		checkMatrix(n, n, a, lda)
		if len(tau) != n-1 && n > 0 {
			panic(badTau)
		}
	}

	const (
		nbmax = 64
		ldt   = nbmax + 1
		tsize = ldt * nbmax
	)
	// Compute the workspace requirements.
	nb := min(nbmax, impl.Ilaenv(1, "DGEHRD", " ", n, ilo, ihi, -1))
	lwkopt := n*nb + tsize
	if lwork == -1 {
		work[0] = float64(lwkopt)
		return
	}

	// Set tau[:ilo] and tau[ihi:] to zero.
	for i := 0; i < ilo; i++ {
		tau[i] = 0
	}
	for i := ihi; i < n-1; i++ {
		tau[i] = 0
	}

	// Quick return if possible.
	nh := ihi - ilo + 1
	if nh <= 1 {
		work[0] = 1
		return
	}

	// Determine the block size.
	nbmin := 2
	var nx int
	if 1 < nb && nb < nh {
		// Determine when to cross over from blocked to unblocked code
		// (last block is always handled by unblocked code).
		nx = max(nb, impl.Ilaenv(3, "DGEHRD", " ", n, ilo, ihi, -1))
		if nx < nh {
			// Determine if workspace is large enough for blocked code.
			if lwork < n*nb+tsize {
				// Not enough workspace to use optimal nb:
				// determine the minimum value of nb, and reduce
				// nb or force use of unblocked code.
				nbmin = max(2, impl.Ilaenv(2, "DGEHRD", " ", n, ilo, ihi, -1))
				if lwork >= n*nbmin+tsize {
					nb = (lwork - tsize) / n
				} else {
					nb = 1
				}
			}
		}
	}
	ldwork := nb // work is used as an n×nb matrix.

	var i int
	if nb < nbmin || nh <= nb {
		// Use unblocked code below.
		i = ilo
	} else {
		// Use blocked code.
		bi := blas64.Implementation()
		iwt := n * nb // Size of the matrix Y and index where the matrix T starts in work.
		for i = ilo; i < ihi-nx; i += nb {
			ib := min(nb, ihi-i)

			// Reduce columns [i:i+ib] to Hessenberg form, returning the
			// matrices V and T of the block reflector H = I - V*T*V^T
			// which performs the reduction, and also the matrix Y = A*V*T.
			impl.Dlahr2(ihi+1, i+1, ib, a[i:], lda, tau[i:], work[iwt:], ldt, work, ldwork)

			// Apply the block reflector H to A[:ihi+1,i+ib:ihi+1] from the
			// right, computing  A := A - Y * V^T. V[i+ib,i+ib-1] must be set
			// to 1.
			ei := a[(i+ib)*lda+i+ib-1]
			a[(i+ib)*lda+i+ib-1] = 1
			bi.Dgemm(blas.NoTrans, blas.Trans, ihi+1, ihi-i-ib+1, ib,
				-1, work, ldwork,
				a[(i+ib)*lda+i:], lda,
				1, a[i+ib:], lda)
			a[(i+ib)*lda+i+ib-1] = ei

			// Apply the block reflector H to A[0:i+1,i+1:i+ib-1] from the
			// right.
			bi.Dtrmm(blas.Right, blas.Lower, blas.Trans, blas.Unit, i+1, ib-1,
				1, a[(i+1)*lda+i:], lda, work, ldwork)
			for j := 0; j <= ib-2; j++ {
				bi.Daxpy(i+1, -1, work[j:], ldwork, a[i+j+1:], lda)
			}

			// Apply the block reflector H to A[i+1:ihi+1,i+ib:n] from the
			// left.
			impl.Dlarfb(blas.Left, blas.Trans, lapack.Forward, lapack.ColumnWise,
				ihi-i, n-i-ib, ib,
				a[(i+1)*lda+i:], lda, work[iwt:], ldt, a[(i+1)*lda+i+ib:], lda, work, ldwork)
		}
	}
	// Use unblocked code to reduce the rest of the matrix.
	impl.Dgehd2(n, i, ihi, a, lda, tau, work)
	work[0] = float64(lwkopt)
}
<commit_msg>lapack/gonum: clean up parameter checks in Dgehrd<commit_after>// Copyright ©2016 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gonum

import (
	"gonum.org/v1/gonum/blas"
	"gonum.org/v1/gonum/blas/blas64"
	"gonum.org/v1/gonum/lapack"
)

// Dgehrd reduces a block of a real n×n general matrix A to upper Hessenberg
// form H by an orthogonal similarity transformation Q^T * A * Q = H.
//
// The matrix Q is represented as a product of (ihi-ilo) elementary
// reflectors
//  Q = H_{ilo} H_{ilo+1} ... H_{ihi-1}.
// Each H_i has the form
//  H_i = I - tau[i] * v * v^T
// where v is a real vector with v[0:i+1] = 0, v[i+1] = 1 and v[ihi+1:n] = 0.
// v[i+2:ihi+1] is stored on exit in A[i+2:ihi+1,i].
//
// On entry, a contains the n×n general matrix to be reduced. On return, the
// upper triangle and the first subdiagonal of A will be overwritten with the
// upper Hessenberg matrix H, and the elements below the first subdiagonal, with
// the slice tau, represent the orthogonal matrix Q as a product of elementary
// reflectors.
//
// The contents of a are illustrated by the following example, with n = 7, ilo =
// 1 and ihi = 5.
// On entry,
//  [ a   a   a   a   a   a   a ]
//  [     a   a   a   a   a   a ]
//  [     a   a   a   a   a   a ]
//  [     a   a   a   a   a   a ]
//  [     a   a   a   a   a   a ]
//  [     a   a   a   a   a   a ]
//  [                         a ]
// on return,
//  [ a   a   h   h   h   h   a ]
//  [     a   h   h   h   h   a ]
//  [     h   h   h   h   h   h ]
//  [     v1  h   h   h   h   h ]
//  [     v1  v2  h   h   h   h ]
//  [     v1  v2  v3  h   h   h ]
//  [                         a ]
// where a denotes an element of the original matrix A, h denotes a
// modified element of the upper Hessenberg matrix H, and vi denotes an
// element of the vector defining H_i.
//
// ilo and ihi determine the block of A that will be reduced to upper Hessenberg
// form. It must hold that 0 <= ilo <= ihi < n if n > 0, and ilo == 0 and ihi ==
// -1 if n == 0, otherwise Dgehrd will panic.
//
// On return, tau will contain the scalar factors of the elementary reflectors.
// Elements tau[:ilo] and tau[ihi:] will be set to zero. tau must have length
// equal to n-1 if n > 0, otherwise Dgehrd will panic.
//
// work must have length at least lwork and lwork must be at least max(1,n),
// otherwise Dgehrd will panic. On return, work[0] contains the optimal value of
// lwork.
//
// If lwork == -1, instead of performing Dgehrd, only the optimal value of lwork
// will be stored in work[0].
//
// Dgehrd is an internal routine. It is exported for testing purposes.
func (impl Implementation) Dgehrd(n, ilo, ihi int, a []float64, lda int, tau, work []float64, lwork int) {
	switch {
	case n < 0:
		panic(nLT0)
	case ilo < 0 || max(0, n-1) < ilo:
		panic(badIlo)
	case ihi < min(ilo, n-1) || n <= ihi:
		panic(badIhi)
	case lda < max(1, n):
		panic(badLdA)
	case lwork < max(1, n) && lwork != -1:
		panic(badWork)
	case len(work) < lwork:
		panic(shortWork)
	}

	const (
		nbmax = 64
		ldt   = nbmax + 1
		tsize = ldt * nbmax
	)
	// Compute the workspace requirements.
	nb := min(nbmax, impl.Ilaenv(1, "DGEHRD", " ", n, ilo, ihi, -1))
	lwkopt := n*nb + tsize
	if lwork == -1 {
		work[0] = float64(lwkopt)
		return
	}

	if len(a) < (n-1)*lda+n {
		panic("lapack: insufficient length of a")
	}
	if len(tau) != n-1 && n > 0 {
		panic(badTau)
	}

	// Set tau[:ilo] and tau[ihi:] to zero.
	for i := 0; i < ilo; i++ {
		tau[i] = 0
	}
	for i := ihi; i < n-1; i++ {
		tau[i] = 0
	}

	// Quick return if possible.
	nh := ihi - ilo + 1
	if nh <= 1 {
		work[0] = 1
		return
	}

	// Determine the block size.
	nbmin := 2
	var nx int
	if 1 < nb && nb < nh {
		// Determine when to cross over from blocked to unblocked code
		// (last block is always handled by unblocked code).
		nx = max(nb, impl.Ilaenv(3, "DGEHRD", " ", n, ilo, ihi, -1))
		if nx < nh {
			// Determine if workspace is large enough for blocked code.
			if lwork < n*nb+tsize {
				// Not enough workspace to use optimal nb:
				// determine the minimum value of nb, and reduce
				// nb or force use of unblocked code.
				nbmin = max(2, impl.Ilaenv(2, "DGEHRD", " ", n, ilo, ihi, -1))
				if lwork >= n*nbmin+tsize {
					nb = (lwork - tsize) / n
				} else {
					nb = 1
				}
			}
		}
	}
	ldwork := nb // work is used as an n×nb matrix.

	var i int
	if nb < nbmin || nh <= nb {
		// Use unblocked code below.
		i = ilo
	} else {
		// Use blocked code.
		bi := blas64.Implementation()
		iwt := n * nb // Size of the matrix Y and index where the matrix T starts in work.
		for i = ilo; i < ihi-nx; i += nb {
			ib := min(nb, ihi-i)

			// Reduce columns [i:i+ib] to Hessenberg form, returning the
			// matrices V and T of the block reflector H = I - V*T*V^T
			// which performs the reduction, and also the matrix Y = A*V*T.
			impl.Dlahr2(ihi+1, i+1, ib, a[i:], lda, tau[i:], work[iwt:], ldt, work, ldwork)

			// Apply the block reflector H to A[:ihi+1,i+ib:ihi+1] from the
			// right, computing  A := A - Y * V^T. V[i+ib,i+ib-1] must be set
			// to 1.
			ei := a[(i+ib)*lda+i+ib-1]
			a[(i+ib)*lda+i+ib-1] = 1
			bi.Dgemm(blas.NoTrans, blas.Trans, ihi+1, ihi-i-ib+1, ib,
				-1, work, ldwork,
				a[(i+ib)*lda+i:], lda,
				1, a[i+ib:], lda)
			a[(i+ib)*lda+i+ib-1] = ei

			// Apply the block reflector H to A[0:i+1,i+1:i+ib-1] from the
			// right.
			bi.Dtrmm(blas.Right, blas.Lower, blas.Trans, blas.Unit, i+1, ib-1,
				1, a[(i+1)*lda+i:], lda, work, ldwork)
			for j := 0; j <= ib-2; j++ {
				bi.Daxpy(i+1, -1, work[j:], ldwork, a[i+j+1:], lda)
			}

			// Apply the block reflector H to A[i+1:ihi+1,i+ib:n] from the
			// left.
			impl.Dlarfb(blas.Left, blas.Trans, lapack.Forward, lapack.ColumnWise,
				ihi-i, n-i-ib, ib,
				a[(i+1)*lda+i:], lda, work[iwt:], ldt, a[(i+1)*lda+i+ib:], lda, work, ldwork)
		}
	}
	// Use unblocked code to reduce the rest of the matrix.
	impl.Dgehd2(n, i, ihi, a, lda, tau, work)
	work[0] = float64(lwkopt)
}
<|endoftext|>


<commit_before>// +build linux lxc

package lxcadapter

import (
	"flag"
	"fmt"
	"github.com/dropbox/changes-client/client"
	"github.com/dropbox/changes-client/client/adapter"
)

var (
	preLaunch  string
	postLaunch string
	s3Bucket   string
)

type Adapter struct {
	config    *client.Config
	container *Container
}

func formatUUID(uuid string) string {
	return fmt.Sprintf("%s-%s-%s-%s-%s", uuid[0:8], uuid[8:12], uuid[12:16], uuid[16:20], uuid[20:])
}

func (a *Adapter) Init(config *client.Config) error {
	container := &Container{
		Name:       formatUUID(config.JobstepID),
		Arch:       "amd64",
		Dist:       "ubuntu",
		Release:    "precise",
		PreLaunch:  preLaunch,
		PostLaunch: postLaunch,
		Snapshot:   formatUUID(config.Snapshot.ID),
		S3Bucket:   s3Bucket,
	}

	a.config = config
	a.container = container

	return nil
}

// Prepare the environment for future commands. This is run before any
// commands are processed and is run once.
func (a *Adapter) Prepare(clientLog *client.Log) error {
	return a.container.Launch(clientLog)
}

// Runs a given command. This may be called multiple times depending
func (a *Adapter) Run(cmd *client.Command, clientLog *client.Log) (*client.CommandResult, error) {
	return a.container.RunCommandInContainer(cmd, clientLog, "ubuntu")
}

// Perform any cleanup actions within the environment.
func (a *Adapter) Shutdown(clientLog *client.Log) error {
	return a.container.Destroy()
}

func init() {
	flag.StringVar(&preLaunch, "pre-launch", "", "Container pre-launch script")
	flag.StringVar(&postLaunch, "post-launch", "", "Container post-launch script")
	flag.StringVar(&s3Bucket, "s3-bucket", "", "S3 bucket name")

	adapter.Register("lxc", &Adapter{})
}
<commit_msg>Correct empty snapshot behavior<commit_after>// +build linux lxc

package lxcadapter

import (
	"flag"
	"fmt"
	"github.com/dropbox/changes-client/client"
	"github.com/dropbox/changes-client/client/adapter"
)

var (
	preLaunch  string
	postLaunch string
	s3Bucket   string
)

type Adapter struct {
	config    *client.Config
	container *Container
}

func formatUUID(uuid string) string {
	return fmt.Sprintf("%s-%s-%s-%s-%s", uuid[0:8], uuid[8:12], uuid[12:16], uuid[16:20], uuid[20:])
}

func (a *Adapter) Init(config *client.Config) error {
	var snapshot string = config.Snapshot.ID
	if snapshot != "" {
		snapshot = formatUUID(snapshot)
	}

	container := &Container{
		Name:       formatUUID(config.JobstepID),
		Arch:       "amd64",
		Dist:       "ubuntu",
		Release:    "precise",
		PreLaunch:  preLaunch,
		PostLaunch: postLaunch,
		Snapshot:   snapshot,
		S3Bucket:   s3Bucket,
	}

	a.config = config
	a.container = container

	return nil
}

// Prepare the environment for future commands. This is run before any
// commands are processed and is run once.
func (a *Adapter) Prepare(clientLog *client.Log) error {
	return a.container.Launch(clientLog)
}

// Runs a given command. This may be called multiple times depending
func (a *Adapter) Run(cmd *client.Command, clientLog *client.Log) (*client.CommandResult, error) {
	return a.container.RunCommandInContainer(cmd, clientLog, "ubuntu")
}

// Perform any cleanup actions within the environment.
func (a *Adapter) Shutdown(clientLog *client.Log) error {
	return a.container.Destroy()
}

func init() {
	flag.StringVar(&preLaunch, "pre-launch", "", "Container pre-launch script")
	flag.StringVar(&postLaunch, "post-launch", "", "Container post-launch script")
	flag.StringVar(&s3Bucket, "s3-bucket", "", "S3 bucket name")

	adapter.Register("lxc", &Adapter{})
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package tests_test

import (
	"flag"
	"strings"
	"time"

	expect "github.com/google/goexpect"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"

	v1 "kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/tests"
)

var _ = Describe("ContainerDisk", func() {

	flag.Parse()

	virtClient, err := kubecli.GetKubevirtClient()
	tests.PanicOnError(err)

	BeforeEach(func() {
		tests.BeforeTestCleanup()
	})

	LaunchVMI := func(vmi *v1.VirtualMachineInstance) runtime.Object {
		By("Starting a VirtualMachineInstance")
		obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
		Expect(err).To(BeNil())
		return obj
	}

	VerifyContainerDiskVMI := func(vmi *v1.VirtualMachineInstance, obj runtime.Object, ignoreWarnings bool) {
		_, ok := obj.(*v1.VirtualMachineInstance)
		Expect(ok).To(BeTrue(), "Object is not of type *v1.VirtualMachineInstance")
		if ignoreWarnings == true {
			tests.WaitForSuccessfulVMIStartIgnoreWarnings(obj)
		} else {
			tests.WaitForSuccessfulVMIStart(obj)
		}

		// Verify Registry Disks are Online
		pods, err := virtClient.CoreV1().Pods(tests.NamespaceTestDefault).List(tests.UnfinishedVMIPodSelector(vmi))
		Expect(err).To(BeNil())

		By("Checking the number of VirtualMachineInstance disks")
		disksFound := 0
		for _, pod := range pods.Items {
			if pod.ObjectMeta.DeletionTimestamp != nil {
				continue
			}
			for _, containerStatus := range pod.Status.ContainerStatuses {
				if strings.HasPrefix(containerStatus.Name, "volume") == false {
					// only check readiness of disk containers
					continue
				}
				disksFound++
			}
			break
		}
		Expect(disksFound).To(Equal(1))
	}

	Describe("Starting and stopping the same VirtualMachineInstance", func() {
		Context("with ephemeral registry disk", func() {
			It("should success multiple times", func() {
				vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(tests.ContainerDiskFor(tests.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
				num := 2
				for i := 0; i < num; i++ {
					By("Starting the VirtualMachineInstance")
					obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
					Expect(err).To(BeNil())
					tests.WaitForSuccessfulVMIStart(obj)

					By("Stopping the VirtualMachineInstance")
					_, err = virtClient.RestClient().Delete().Resource("virtualmachineinstances").Namespace(vmi.GetObjectMeta().GetNamespace()).Name(vmi.GetObjectMeta().GetName()).Do().Get()
					Expect(err).To(BeNil())
					By("Waiting until the VirtualMachineInstance is gone")
					tests.WaitForVirtualMachineToDisappearWithTimeout(vmi, 120)
				}
			})
		})
	})

	Describe("Starting a VirtualMachineInstance", func() {
		Context("with ephemeral registry disk", func() {
			It("should not modify the spec on status update", func() {
				vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(tests.ContainerDiskFor(tests.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
				v1.SetObjectDefaults_VirtualMachineInstance(vmi)

				By("Starting the VirtualMachineInstance")
				vmi, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
				Expect(err).To(BeNil())
				tests.WaitForSuccessfulVMIStart(vmi)
				startedVMI, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Get(vmi.ObjectMeta.Name, &metav1.GetOptions{})
				Expect(err).To(BeNil())
				By("Checking that the VirtualMachineInstance spec did not change")
				Expect(startedVMI.Spec).To(Equal(vmi.Spec))
			})
		})
	})

	Describe("Starting multiple VMIs", func() {
		Context("with ephemeral registry disk", func() {
			It("should success", func() {
				num := 5
				vmis := make([]*v1.VirtualMachineInstance, 0, num)
				objs := make([]runtime.Object, 0, num)
				for i := 0; i < num; i++ {
					vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(tests.ContainerDiskFor(tests.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
					// FIXME if we give too much ram, the vmis really boot and eat all our memory (cache?)
					vmi.Spec.Domain.Resources.Requests[k8sv1.ResourceMemory] = resource.MustParse("1M")
					obj := LaunchVMI(vmi)
					vmis = append(vmis, vmi)
					objs = append(objs, obj)
				}

				for idx, vmi := range vmis {
					// TODO once networking is implemented properly set ignoreWarnings == false here.
					// We have to ignore warnings because VMIs started in parallel
					// may cause libvirt to fail to create the macvtap device in
					// the host network.
					// The new network implementation we're working on should resolve this.
					// NOTE the VirtualMachineInstance still starts successfully regardless of this warning.
					// It just requires virt-handler to retry the Start command at the moment.
					VerifyContainerDiskVMI(vmi, objs[idx], true)
				}
			}) // Timeout is long because this test involves multiple parallel VirtualMachineInstance launches.
		})
	})

	Describe("Starting from custom image location", func() {
		Context("with disk at /custom-disk/boot.img", func() {
			It("should boot normally", func() {
				vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(tests.ContainerDiskFor(tests.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
				for ind, volume := range vmi.Spec.Volumes {
					if volume.ContainerDisk != nil {
						vmi.Spec.Volumes[ind].ContainerDisk.Path = "/custom-disk/boot.img"
					}
				}
				By("Starting the VirtualMachineInstance")
				obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
				Expect(err).To(BeNil())
				tests.WaitForSuccessfulVMIStart(obj)
			})
		})
	})

	Describe("Starting with virtio-win", func() {
		Context("with virtio-win as secondary disk", func() {
			It("should boot and have the virtio as sata CDROM", func() {
				vmi := tests.NewRandomVMIWithEphemeralDisk(tests.ContainerDiskFor(tests.ContainerDiskAlpine))
				tests.AddEphemeralCdrom(vmi, "disk4", "sata", tests.ContainerDiskFor(tests.ContainerDiskVirtio))

				By("Starting the VirtualMachineInstance")
				obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
				Expect(err).To(BeNil(), "expected vmi to start with no problem")
				tests.WaitForSuccessfulVMIStart(obj)

				By("Checking whether the second disk really contains virtio drivers")
				expecter, err := tests.LoggedInAlpineExpecter(vmi)
				Expect(err).ToNot(HaveOccurred(), "expected alpine to login properly")
				defer expecter.Close()

				_, err = expecter.ExpectBatch([]expect.Batcher{
					// mount virtio cdrom and check files are there
					&expect.BSnd{S: "mount -t iso9600 /dev/cdrom /mnt\n"},
					&expect.BSnd{S: "echo $?\n"},
					&expect.BExp{R: "0"},
					&expect.BSnd{S: "ls virtio-win_license.txt guest-agent\n"},
					&expect.BSnd{S: "echo $?\n"},
					&expect.BExp{R: "0"},
				}, 200*time.Second)
				Expect(err).ToNot(HaveOccurred(), "expected virtio files to be mounted properly")
			})
		})
	})
})
<commit_msg>Fix flaky virtio-win test<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package tests_test

import (
	"flag"
	"strings"
	"time"

	expect "github.com/google/goexpect"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"

	v1 "kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/tests"
)

var _ = Describe("ContainerDisk", func() {

	flag.Parse()

	virtClient, err := kubecli.GetKubevirtClient()
	tests.PanicOnError(err)

	BeforeEach(func() {
		tests.BeforeTestCleanup()
	})

	LaunchVMI := func(vmi *v1.VirtualMachineInstance) runtime.Object {
		By("Starting a VirtualMachineInstance")
		obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
		Expect(err).To(BeNil())
		return obj
	}

	VerifyContainerDiskVMI := func(vmi *v1.VirtualMachineInstance, obj runtime.Object, ignoreWarnings bool) {
		_, ok := obj.(*v1.VirtualMachineInstance)
		Expect(ok).To(BeTrue(), "Object is not of type *v1.VirtualMachineInstance")
		if ignoreWarnings == true {
			tests.WaitForSuccessfulVMIStartIgnoreWarnings(obj)
		} else {
			tests.WaitForSuccessfulVMIStart(obj)
		}

		// Verify Registry Disks are Online
		pods, err := virtClient.CoreV1().Pods(tests.NamespaceTestDefault).List(tests.UnfinishedVMIPodSelector(vmi))
		Expect(err).To(BeNil())

		By("Checking the number of VirtualMachineInstance disks")
		disksFound := 0
		for _, pod := range pods.Items {
			if pod.ObjectMeta.DeletionTimestamp != nil {
				continue
			}
			for _, containerStatus := range pod.Status.ContainerStatuses {
				if strings.HasPrefix(containerStatus.Name, "volume") == false {
					// only check readiness of disk containers
					continue
				}
				disksFound++
			}
			break
		}
		Expect(disksFound).To(Equal(1))
	}

	Describe("Starting and stopping the same VirtualMachineInstance", func() {
		Context("with ephemeral registry disk", func() {
			It("should success multiple times", func() {
				vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(tests.ContainerDiskFor(tests.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
				num := 2
				for i := 0; i < num; i++ {
					By("Starting the VirtualMachineInstance")
					obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
					Expect(err).To(BeNil())
					tests.WaitForSuccessfulVMIStart(obj)

					By("Stopping the VirtualMachineInstance")
					_, err = virtClient.RestClient().Delete().Resource("virtualmachineinstances").Namespace(vmi.GetObjectMeta().GetNamespace()).Name(vmi.GetObjectMeta().GetName()).Do().Get()
					Expect(err).To(BeNil())
					By("Waiting until the VirtualMachineInstance is gone")
					tests.WaitForVirtualMachineToDisappearWithTimeout(vmi, 120)
				}
			})
		})
	})

	Describe("Starting a VirtualMachineInstance", func() {
		Context("with ephemeral registry disk", func() {
			It("should not modify the spec on status update", func() {
				vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(tests.ContainerDiskFor(tests.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
				v1.SetObjectDefaults_VirtualMachineInstance(vmi)

				By("Starting the VirtualMachineInstance")
				vmi, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
				Expect(err).To(BeNil())
				tests.WaitForSuccessfulVMIStart(vmi)
				startedVMI, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Get(vmi.ObjectMeta.Name, &metav1.GetOptions{})
				Expect(err).To(BeNil())
				By("Checking that the VirtualMachineInstance spec did not change")
				Expect(startedVMI.Spec).To(Equal(vmi.Spec))
			})
		})
	})

	Describe("Starting multiple VMIs", func() {
		Context("with ephemeral registry disk", func() {
			It("should success", func() {
				num := 5
				vmis := make([]*v1.VirtualMachineInstance, 0, num)
				objs := make([]runtime.Object, 0, num)
				for i := 0; i < num; i++ {
					vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(tests.ContainerDiskFor(tests.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
					// FIXME if we give too much ram, the vmis really boot and eat all our memory (cache?)
					vmi.Spec.Domain.Resources.Requests[k8sv1.ResourceMemory] = resource.MustParse("1M")
					obj := LaunchVMI(vmi)
					vmis = append(vmis, vmi)
					objs = append(objs, obj)
				}

				for idx, vmi := range vmis {
					// TODO once networking is implemented properly set ignoreWarnings == false here.
					// We have to ignore warnings because VMIs started in parallel
					// may cause libvirt to fail to create the macvtap device in
					// the host network.
					// The new network implementation we're working on should resolve this.
					// NOTE the VirtualMachineInstance still starts successfully regardless of this warning.
					// It just requires virt-handler to retry the Start command at the moment.
					VerifyContainerDiskVMI(vmi, objs[idx], true)
				}
			}) // Timeout is long because this test involves multiple parallel VirtualMachineInstance launches.
		})
	})

	Describe("Starting from custom image location", func() {
		Context("with disk at /custom-disk/boot.img", func() {
			It("should boot normally", func() {
				vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(tests.ContainerDiskFor(tests.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
				for ind, volume := range vmi.Spec.Volumes {
					if volume.ContainerDisk != nil {
						vmi.Spec.Volumes[ind].ContainerDisk.Path = "/custom-disk/boot.img"
					}
				}
				By("Starting the VirtualMachineInstance")
				obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
				Expect(err).To(BeNil())
				tests.WaitForSuccessfulVMIStart(obj)
			})
		})
	})

	Describe("Starting with virtio-win", func() {
		Context("with virtio-win as secondary disk", func() {
			It("should boot and have the virtio as sata CDROM", func() {
				vmi := tests.NewRandomVMIWithEphemeralDisk(tests.ContainerDiskFor(tests.ContainerDiskAlpine))
				tests.AddEphemeralCdrom(vmi, "disk4", "sata", tests.ContainerDiskFor(tests.ContainerDiskVirtio))

				By("Starting the VirtualMachineInstance")
				obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
				Expect(err).To(BeNil(), "expected vmi to start with no problem")
				tests.WaitForSuccessfulVMIStart(obj)

				By("Checking whether the second disk really contains virtio drivers")
				expecter, err := tests.LoggedInAlpineExpecter(vmi)
				Expect(err).ToNot(HaveOccurred(), "expected alpine to login properly")
				defer expecter.Close()

				_, err = expecter.ExpectBatch([]expect.Batcher{
					// mount virtio cdrom and check files are there
					&expect.BSnd{S: "mount -t iso9600 /dev/cdrom\n"},
					&expect.BSnd{S: "echo $?\n"},
					&expect.BExp{R: "0"},
					&expect.BSnd{S: "cd /media/cdrom\n"},
					&expect.BSnd{S: "ls virtio-win_license.txt guest-agent\n"},
					&expect.BSnd{S: "echo $?\n"},
					&expect.BExp{R: "0"},
				}, 200*time.Second)
				Expect(err).ToNot(HaveOccurred(), "expected virtio files to be mounted properly")
			})
		})
	})
})
<|endoftext|>


<commit_before>// Copyright (c) 2016 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// firewall_test.go contains test cases for firewall.go
package agent

// Some comments on use of mocking framework in helpers_test.go.

import (
	"errors"
	"net"
	"strings"
	"testing"
)

// TestNewChains is checking that detectMissingChains correctly detects which
// Romana chains must be created for given NetIf.
func TestNewChains(t *testing.T) {
	agent := mockAgent()
	// when

	// detectMissingChains calls isChainExist which is reading FakeExecutor
	// isChainExist doesn't care for output but must receive not nil error
	// otherwise it would decide that chain exist already and skip
	E := &FakeExecutor{nil, errors.New("bla"), nil}
	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	newChains := fw.detectMissingChains()

	// expect
	if len(newChains) != 3 {
		t.Error("TestNewChains failed")
	}

	// TODO test case with some chains already exist requires support for
	// stack of output in FakeExecutor
}

// TestCreateChains is checking that CreateChains generates correct OS commands
// for iptables to create firewall chains.
func TestCreateChains(t *testing.T) {
	agent := mockAgent()
	// when

	// CreateChains doesn't care for output and we don't any errors
	// we only want to analize which command generated by the function
	E := &FakeExecutor{nil, nil, nil}
	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	_ = fw.CreateChains([]int{0, 1, 2})

	// expect
	expect := strings.Join([]string{"/sbin/iptables -N ROMANA-T0S0-INPUT",
		"/sbin/iptables -N ROMANA-T0S0-OUTPUT",
		"/sbin/iptables -N ROMANA-T0S0-FORWARD"}, "\n")

	if *E.Commands != expect {
		t.Errorf("Unexpected input from TestCreateChains, expect\n%s, got\n%s", expect, *E.Commands)
	}
}

// TestDivertTraffic is checking that DivertTrafficToRomanaIptablesChain generates correct commands for
// firewall to divert traffic into Romana chains.
func TestDivertTraffic(t *testing.T) {
	agent := mockAgent()
	// when

	// We need to simulate failure on response from os.exec
	// so isRuleExist would fail and trigger ensureIptablesRule.
	// But because ensureIptablesRule will use same object as
	// response from os.exec we will see error in test logs,
	// it's ok as long as function generates expected set of commands.
	E := &FakeExecutor{nil, errors.New("Rule not found"), nil}
	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	// 0 is a first standard chain - INPUT
	fw.DivertTrafficToRomanaIptablesChain(0)

	// expect
	expect := "/sbin/iptables -C INPUT -i eth0 -j ROMANA-T0S0-INPUT\n/sbin/iptables -A INPUT -i eth0 -j ROMANA-T0S0-INPUT"

	if *E.Commands != expect {
		t.Errorf("Unexpected input from TestDivertTraffic, expect\n%s, got\n%s", expect, *E.Commands)
	}
	t.Log("All good here, don't be afraid if 'Diverting traffic failed' message")
}

// TestCreateRules is checking that CreateRules generates correct commands to create
// firewall rules.
func TestCreateRules(t *testing.T) {
	agent := mockAgent()
	// when

	// we only care for recorded commands, no need for fake output or errors
	E := &FakeExecutor{nil, nil, nil}
	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	// 0 is a first standard chain - INPUT
	fw.CreateRules(0)

	// expect
	expect := strings.Join([]string{
		"/sbin/iptables -A ROMANA-T0S0-INPUT -d 172.17.0.1/32 -p icmp -m icmp --icmp-type 0 -m state --state RELATED,ESTABLISHED -j ACCEPT",
		"/sbin/iptables -A ROMANA-T0S0-INPUT -d 172.17.0.1/32 -p tcp -m tcp --sport 22 -j ACCEPT",
		"/sbin/iptables -A ROMANA-T0S0-INPUT -d 255.255.255.255/32 -p udp -m udp --sport 68 --dport 67 -j ACCEPT",
	}, "\n")

	if *E.Commands != expect {
		t.Errorf("Unexpected input from TestCreateRules, expect\n%s, got\n%s", expect, *E.Commands)
	}
}

// TestCreateU32Rule is checking that CreateU32Rules generates correct commands to
// create firewall rules.
func TestCreateU32Rules(t *testing.T) {
	agent := mockAgent()
	// when

	// we only care for recorded commands, no need for fake output or errors
	E := &FakeExecutor{nil, nil, nil}

	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	// 0 is a first standard chain - INPUT
	fw.CreateU32Rules(0)

	// expect
	expect := strings.Join([]string{"/sbin/iptables -A ROMANA-T0S0-INPUT -m u32 --u32 12&0xFF00FF00=0x7F000000 && 16&0xFF00FF00=0x7F000000 -j ACCEPT"}, "\n")

	if *E.Commands != expect {
		t.Errorf("Unexpected input from TestCreateU32Rules, expect\n%s, got\n%s", expect, *E.Commands)
	}
}
<commit_msg>Removing some comments<commit_after>// Copyright (c) 2016 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// firewall_test.go contains test cases for firewall.go
package agent

// Some comments on use of mocking framework in helpers_test.go.

import (
	"errors"
	"net"
	"strings"
	"testing"
)

// TestNewChains is checking that detectMissingChains correctly detects which
// Romana chains must be created for given NetIf.
func TestNewChains(t *testing.T) {
	agent := mockAgent()

	// detectMissingChains calls isChainExist which is reading FakeExecutor
	// isChainExist doesn't care for output but must receive not nil error
	// otherwise it would decide that chain exist already and skip
	E := &FakeExecutor{nil, errors.New("bla"), nil}
	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	newChains := fw.detectMissingChains()

	if len(newChains) != 3 {
		t.Error("TestNewChains failed")
	}

	// TODO test case with some chains already exist requires support for
	// stack of output in FakeExecutor
}

// TestCreateChains is checking that CreateChains generates correct OS commands
// for iptables to create firewall chains.
func TestCreateChains(t *testing.T) {
	agent := mockAgent()

	// CreateChains doesn't care for output and we don't any errors
	// we only want to analize which command generated by the function
	E := &FakeExecutor{nil, nil, nil}
	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	_ = fw.CreateChains([]int{0, 1, 2})

	expect := strings.Join([]string{"/sbin/iptables -N ROMANA-T0S0-INPUT",
		"/sbin/iptables -N ROMANA-T0S0-OUTPUT",
		"/sbin/iptables -N ROMANA-T0S0-FORWARD"}, "\n")

	if *E.Commands != expect {
		t.Errorf("Unexpected input from TestCreateChains, expect\n%s, got\n%s", expect, *E.Commands)
	}
}

// TestDivertTraffic is checking that DivertTrafficToRomanaIptablesChain generates correct commands for
// firewall to divert traffic into Romana chains.
func TestDivertTraffic(t *testing.T) {
	agent := mockAgent()

	// We need to simulate failure on response from os.exec
	// so isRuleExist would fail and trigger ensureIptablesRule.
	// But because ensureIptablesRule will use same object as
	// response from os.exec we will see error in test logs,
	// it's ok as long as function generates expected set of commands.
	E := &FakeExecutor{nil, errors.New("Rule not found"), nil}
	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	// 0 is a first standard chain - INPUT
	fw.DivertTrafficToRomanaIptablesChain(0)

	expect := "/sbin/iptables -C INPUT -i eth0 -j ROMANA-T0S0-INPUT\n/sbin/iptables -A INPUT -i eth0 -j ROMANA-T0S0-INPUT"

	if *E.Commands != expect {
		t.Errorf("Unexpected input from TestDivertTraffic, expect\n%s, got\n%s", expect, *E.Commands)
	}
	t.Log("All good here, don't be afraid if 'Diverting traffic failed' message")
}

// TestCreateRules is checking that CreateRules generates correct commands to create
// firewall rules.
func TestCreateRules(t *testing.T) {
	agent := mockAgent()

	// we only care for recorded commands, no need for fake output or errors
	E := &FakeExecutor{nil, nil, nil}
	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	// 0 is a first standard chain - INPUT
	fw.CreateRules(0)

	expect := strings.Join([]string{
		"/sbin/iptables -A ROMANA-T0S0-INPUT -d 172.17.0.1/32 -p icmp -m icmp --icmp-type 0 -m state --state RELATED,ESTABLISHED -j ACCEPT",
		"/sbin/iptables -A ROMANA-T0S0-INPUT -d 172.17.0.1/32 -p tcp -m tcp --sport 22 -j ACCEPT",
		"/sbin/iptables -A ROMANA-T0S0-INPUT -d 255.255.255.255/32 -p udp -m udp --sport 68 --dport 67 -j ACCEPT",
	}, "\n")

	if *E.Commands != expect {
		t.Errorf("Unexpected input from TestCreateRules, expect\n%s, got\n%s", expect, *E.Commands)
	}
}

// TestCreateU32Rule is checking that CreateU32Rules generates correct commands to
// create firewall rules.
func TestCreateU32Rules(t *testing.T) {
	agent := mockAgent()

	// we only care for recorded commands, no need for fake output or errors
	E := &FakeExecutor{nil, nil, nil}

	agent.Helper.Executor = E
	ip := net.ParseIP("127.0.0.1")
	fw, _ := NewFirewall(NetIf{"eth0", "A", ip}, &agent)

	// 0 is a first standard chain - INPUT
	fw.CreateU32Rules(0)

	expect := strings.Join([]string{"/sbin/iptables -A ROMANA-T0S0-INPUT -m u32 --u32 12&0xFF00FF00=0x7F000000 && 16&0xFF00FF00=0x7F000000 -j ACCEPT"}, "\n")

	if *E.Commands != expect {
		t.Errorf("Unexpected input from TestCreateU32Rules, expect\n%s, got\n%s", expect, *E.Commands)
	}
}
<|endoftext|>


<commit_before>// Package internal contains common API functions and structures shared between lexer packages.
package internal

import (
	"path/filepath"
	"sort"
	"strings"

	"github.com/danwakefield/fnmatch"

	"github.com/alecthomas/chroma"
)

// Registry of Lexers.
var Registry = struct {
	Lexers  chroma.Lexers
	byName  map[string]chroma.Lexer
	byAlias map[string]chroma.Lexer
}{
	byName:  map[string]chroma.Lexer{},
	byAlias: map[string]chroma.Lexer{},
}

// Names of all lexers, optionally including aliases.
func Names(withAliases bool) []string {
	out := []string{}
	for _, lexer := range Registry.Lexers {
		config := lexer.Config()
		out = append(out, config.Name)
		if withAliases {
			out = append(out, config.Aliases...)
		}
	}
	sort.Strings(out)
	return out
}

// Get a Lexer by name, alias or file extension.
func Get(name string) chroma.Lexer {
	if lexer := Registry.byName[name]; lexer != nil {
		return lexer
	}
	if lexer := Registry.byAlias[name]; lexer != nil {
		return lexer
	}
	if lexer := Registry.byName[strings.ToLower(name)]; lexer != nil {
		return lexer
	}
	if lexer := Registry.byAlias[strings.ToLower(name)]; lexer != nil {
		return lexer
	}

	candidates := chroma.PrioritisedLexers{}
	// Try file extension.
	if lexer := Match("filename." + name); lexer != nil {
		candidates = append(candidates, lexer)
	}
	// Try exact filename.
	if lexer := Match(name); lexer != nil {
		candidates = append(candidates, lexer)
	}
	if len(candidates) == 0 {
		return nil
	}
	sort.Sort(candidates)
	return candidates[0]
}

// MatchMimeType attempts to find a lexer for the given MIME type.
func MatchMimeType(mimeType string) chroma.Lexer {
	matched := chroma.PrioritisedLexers{}
	for _, l := range Registry.Lexers {
		for _, lmt := range l.Config().MimeTypes {
			if mimeType == lmt {
				matched = append(matched, l)
			}
		}
	}
	if len(matched) != 0 {
		sort.Sort(matched)
		return matched[0]
	}
	return nil
}

// Match returns the first lexer matching filename.
func Match(filename string) chroma.Lexer {
	filename = filepath.Base(filename)
	matched := chroma.PrioritisedLexers{}
	// First, try primary filename matches.
	for _, lexer := range Registry.Lexers {
		config := lexer.Config()
		for _, glob := range config.Filenames {
			if fnmatch.Match(glob, filename, 0) {
				matched = append(matched, lexer)
			}
		}
	}
	if len(matched) > 0 {
		sort.Sort(matched)
		return matched[0]
	}
	matched = nil
	// Next, try filename aliases.
	for _, lexer := range Registry.Lexers {
		config := lexer.Config()
		for _, glob := range config.AliasFilenames {
			if fnmatch.Match(glob, filename, 0) {
				matched = append(matched, lexer)
			}
		}
	}
	if len(matched) > 0 {
		sort.Sort(matched)
		return matched[0]
	}
	return nil
}

// Analyse text content and return the "best" lexer..
func Analyse(text string) chroma.Lexer {
	var picked chroma.Lexer
	highest := float32(0.0)
	for _, lexer := range Registry.Lexers {
		if analyser, ok := lexer.(chroma.Analyser); ok {
			weight := analyser.AnalyseText(text)
			if weight > highest {
				picked = lexer
				highest = weight
			}
		}
	}
	return picked
}

// Register a Lexer with the global registry.
func Register(lexer chroma.Lexer) chroma.Lexer {
	config := lexer.Config()
	Registry.byName[config.Name] = lexer
	Registry.byName[strings.ToLower(config.Name)] = lexer
	for _, alias := range config.Aliases {
		Registry.byAlias[alias] = lexer
		Registry.byAlias[strings.ToLower(alias)] = lexer
	}
	Registry.Lexers = append(Registry.Lexers, lexer)
	return lexer
}

// PlaintextRules is used for the fallback lexer as well as the explicit
// plaintext lexer.
func PlaintextRules() chroma.Rules {
	return chroma.Rules{
		"root": []chroma.Rule{
			{`.+`, chroma.Text, nil},
			{`\n`, chroma.Text, nil},
		},
	}
}

// Fallback lexer if no other is found.
var Fallback chroma.Lexer = chroma.MustNewLazyLexer(&chroma.Config{
	Name:      "fallback",
	Filenames: []string{"*"},
}, PlaintextRules)
<commit_msg>lexers: match by known backup filename suffixes<commit_after>// Package internal contains common API functions and structures shared between lexer packages.
package internal

import (
	"path/filepath"
	"sort"
	"strings"

	"github.com/danwakefield/fnmatch"

	"github.com/alecthomas/chroma"
)

var (
	backupSuffixes = [...]string{
		// Editor backups
		"~", ".bak", ".old", ".orig",
		// Debian and derivatives apt/dpkg
		".dpkg-dist", ".dpkg-old",
		// Red Hat and derivatives rpm
		".rpmnew", ".rpmorig", ".rpmsave",
	}
)

// Registry of Lexers.
var Registry = struct {
	Lexers  chroma.Lexers
	byName  map[string]chroma.Lexer
	byAlias map[string]chroma.Lexer
}{
	byName:  map[string]chroma.Lexer{},
	byAlias: map[string]chroma.Lexer{},
}

// Names of all lexers, optionally including aliases.
func Names(withAliases bool) []string {
	out := []string{}
	for _, lexer := range Registry.Lexers {
		config := lexer.Config()
		out = append(out, config.Name)
		if withAliases {
			out = append(out, config.Aliases...)
		}
	}
	sort.Strings(out)
	return out
}

// Get a Lexer by name, alias or file extension.
func Get(name string) chroma.Lexer {
	if lexer := Registry.byName[name]; lexer != nil {
		return lexer
	}
	if lexer := Registry.byAlias[name]; lexer != nil {
		return lexer
	}
	if lexer := Registry.byName[strings.ToLower(name)]; lexer != nil {
		return lexer
	}
	if lexer := Registry.byAlias[strings.ToLower(name)]; lexer != nil {
		return lexer
	}

	candidates := chroma.PrioritisedLexers{}
	// Try file extension.
	if lexer := Match("filename." + name); lexer != nil {
		candidates = append(candidates, lexer)
	}
	// Try exact filename.
	if lexer := Match(name); lexer != nil {
		candidates = append(candidates, lexer)
	}
	if len(candidates) == 0 {
		return nil
	}
	sort.Sort(candidates)
	return candidates[0]
}

// MatchMimeType attempts to find a lexer for the given MIME type.
func MatchMimeType(mimeType string) chroma.Lexer {
	matched := chroma.PrioritisedLexers{}
	for _, l := range Registry.Lexers {
		for _, lmt := range l.Config().MimeTypes {
			if mimeType == lmt {
				matched = append(matched, l)
			}
		}
	}
	if len(matched) != 0 {
		sort.Sort(matched)
		return matched[0]
	}
	return nil
}

// Match returns the first lexer matching filename.
func Match(filename string) chroma.Lexer {
	filename = filepath.Base(filename)
	matched := chroma.PrioritisedLexers{}
	// First, try primary filename matches.
	for _, lexer := range Registry.Lexers {
		config := lexer.Config()
		for _, glob := range config.Filenames {
			if fnmatch.Match(glob, filename, 0) {
				matched = append(matched, lexer)
			} else {
				for _, suf := range &backupSuffixes {
					if fnmatch.Match(glob+suf, filename, 0) {
						matched = append(matched, lexer)
						break
					}
				}
			}
		}
	}
	if len(matched) > 0 {
		sort.Sort(matched)
		return matched[0]
	}
	matched = nil
	// Next, try filename aliases.
	for _, lexer := range Registry.Lexers {
		config := lexer.Config()
		for _, glob := range config.AliasFilenames {
			if fnmatch.Match(glob, filename, 0) {
				matched = append(matched, lexer)
			} else {
				for _, suf := range &backupSuffixes {
					if fnmatch.Match(glob+suf, filename, 0) {
						matched = append(matched, lexer)
						break
					}
				}
			}
		}
	}
	if len(matched) > 0 {
		sort.Sort(matched)
		return matched[0]
	}
	return nil
}

// Analyse text content and return the "best" lexer..
func Analyse(text string) chroma.Lexer {
	var picked chroma.Lexer
	highest := float32(0.0)
	for _, lexer := range Registry.Lexers {
		if analyser, ok := lexer.(chroma.Analyser); ok {
			weight := analyser.AnalyseText(text)
			if weight > highest {
				picked = lexer
				highest = weight
			}
		}
	}
	return picked
}

// Register a Lexer with the global registry.
func Register(lexer chroma.Lexer) chroma.Lexer {
	config := lexer.Config()
	Registry.byName[config.Name] = lexer
	Registry.byName[strings.ToLower(config.Name)] = lexer
	for _, alias := range config.Aliases {
		Registry.byAlias[alias] = lexer
		Registry.byAlias[strings.ToLower(alias)] = lexer
	}
	Registry.Lexers = append(Registry.Lexers, lexer)
	return lexer
}

// PlaintextRules is used for the fallback lexer as well as the explicit
// plaintext lexer.
func PlaintextRules() chroma.Rules {
	return chroma.Rules{
		"root": []chroma.Rule{
			{`.+`, chroma.Text, nil},
			{`\n`, chroma.Text, nil},
		},
	}
}

// Fallback lexer if no other is found.
var Fallback chroma.Lexer = chroma.MustNewLazyLexer(&chroma.Config{
	Name:      "fallback",
	Filenames: []string{"*"},
}, PlaintextRules)
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2013 IBM Corp.
 *
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *    Seth Hoenig
 *    Allan Stockdill-Mander
 *    Mike Robertson
 */

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"time"

	"github.com/eclipse/paho.mqtt.golang"
)

var f mqtt.MessageHandler = func(client mqtt.Client, msg mqtt.Message) {
	//	fmt.Printf("TOPIC: %s\n", msg.Topic())
	//	fmt.Printf("MSG: %s\n", msg.Payload())
}

var clients []mqtt.Client
var newClientChan = make(chan mqtt.Client)
var closeAll = make(chan bool)

func push(host string, port int, i int) {
	//	mqtt.DEBUG = log.New(os.Stdout, "", 0)
	mqtt.ERROR = log.New(os.Stdout, "", 0)
	opts := mqtt.NewClientOptions().AddBroker("tcp://" + host + fmt.Sprintf(":%d", port)).SetClientID(fmt.Sprintf("client%d", i))
	opts.SetKeepAlive(20 * time.Second)
	opts.SetDefaultPublishHandler(f)
	opts.SetPingTimeout(10 * time.Second)

	c := mqtt.NewClient(opts)
	if token := c.Connect(); token.Wait() && token.Error() != nil {
		panic(token.Error())
	}

	if token := c.Subscribe("go-mqtt/sample", 0, nil); token.Wait() && token.Error() != nil {
		fmt.Println(token.Error())
		os.Exit(1)
	}

	for i := 0; true; i++ {
		text := fmt.Sprintf("this is msg #%d!", i)
		token := c.Publish("go-mqtt/sample", 0, false, text)
		token.Wait()
		time.Sleep(time.Second)
	}
}
func subscribe(host string, port int, i int, resultCh chan<- bool) {
	//	mqtt.DEBUG = log.New(os.Stdout, "", 0)
	mqtt.ERROR = log.New(os.Stdout, "", 0)
	opts := mqtt.NewClientOptions().AddBroker("tcp://" + host + fmt.Sprintf(":%d", port)).SetClientID(fmt.Sprintf("client%d", time.Now().Nanosecond()))
	opts.SetConnectTimeout(100 * time.Second)

	opts.SetKeepAlive(200 * time.Second)
	opts.SetDefaultPublishHandler(f)
	opts.SetPingTimeout(100 * time.Second)

	defer func() {
		recover()
		close(resultCh)
	}()
	c := mqtt.NewClient(opts)
	if token := c.Connect(); token.Wait() && token.Error() != nil {
		panic(token.Error())
	}

	if token := c.Subscribe("go-mqtt/sample", 0, nil); token.Wait() && token.Error() != nil {
		fmt.Println(token.Error())
		os.Exit(1)
	}

}
func main() {
	host := flag.String("host", "localhost", "host")
	port := flag.Int("port", 1883, "server port")
	p := flag.Int("p", 1, "count process")
	flag.Parse()
	var i int
	defer func() {
		recover()
		mqtt.DEBUG.Printf("connected :%d", i)
	}()
	go func() {
		for {
			select {
			case c := <-newClientChan:
				clients = append(clients, c)
			case <-closeAll:
				for _, c := range clients {
					c.Disconnect(250)
				}
				close(closeAll)
				return
			}
		}
	}()
	//	go push(*host,*port, 0)
	for i = 1; i <= *p; i++ {
		ch := make(chan bool)
		go subscribe(*host, *port, i, ch)
		<-ch
	}
	signals := make(chan os.Signal, 1)

	signal.Notify(signals, os.Interrupt)
	select {
	case <-signals:
		//关闭服务器的端口监听，以退出
		closeAll <- true
	}
	<-closeAll
}
<commit_msg>1.修复clientId可以在服务端重复的问题<commit_after>/*
 * Copyright (c) 2013 IBM Corp.
 *
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *    Seth Hoenig
 *    Allan Stockdill-Mander
 *    Mike Robertson
 */

package main

import (
	"crypto/md5"
	"encoding/hex"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"os/signal"
	"time"

	"github.com/eclipse/paho.mqtt.golang"
)

var f mqtt.MessageHandler = func(client mqtt.Client, msg mqtt.Message) {
	//	fmt.Printf("TOPIC: %s\n", msg.Topic())
	//	fmt.Printf("MSG: %s\n", msg.Payload())
}

var clients []mqtt.Client
var newClientChan = make(chan mqtt.Client)
var closeAll = make(chan bool)
var localId string

func newClientId() string {
	sum := md5.Sum([]byte(fmt.Sprintf("%s%d", localId, time.Now().Nanosecond())))
	return hex.EncodeToString(sum[:])
}
func push(host string, port int, i int) {
	//	mqtt.DEBUG = log.New(os.Stdout, "", 0)
	mqtt.ERROR = log.New(os.Stdout, "", 0)
	opts := mqtt.NewClientOptions().AddBroker("tcp://" + host + fmt.Sprintf(":%d", port)).SetClientID(newClientId())
	opts.SetKeepAlive(20 * time.Second)
	opts.SetDefaultPublishHandler(f)
	opts.SetPingTimeout(10 * time.Second)

	c := mqtt.NewClient(opts)
	if token := c.Connect(); token.Wait() && token.Error() != nil {
		panic(token.Error())
	}

	if token := c.Subscribe("go-mqtt/sample", 0, nil); token.Wait() && token.Error() != nil {
		fmt.Println(token.Error())
		os.Exit(1)
	}

	for i := 0; true; i++ {
		text := fmt.Sprintf("this is msg #%d!", i)
		token := c.Publish("go-mqtt/sample", 0, false, text)
		token.Wait()
		time.Sleep(time.Second)
	}
}
func subscribe(host string, port int, i int, resultCh chan<- bool) {
	//	mqtt.DEBUG = log.New(os.Stdout, "", 0)
	mqtt.ERROR = log.New(os.Stdout, "", 0)
	opts := mqtt.NewClientOptions().AddBroker("tcp://" + host + fmt.Sprintf(":%d", port)).SetClientID(newClientId())
	opts.SetConnectTimeout(100 * time.Second)

	opts.SetKeepAlive(200 * time.Second)
	opts.SetDefaultPublishHandler(f)
	opts.SetPingTimeout(100 * time.Second)

	defer func() {
		recover()
		close(resultCh)
	}()
	c := mqtt.NewClient(opts)
	if token := c.Connect(); token.Wait() && token.Error() != nil {
		panic(token.Error())
	}

	if token := c.Subscribe("go-mqtt/sample", 0, nil); token.Wait() && token.Error() != nil {
		fmt.Println(token.Error())
		os.Exit(1)
	}

}
func main() {
	host := flag.String("host", "localhost", "host")
	port := flag.Int("port", 1883, "server port")
	p := flag.Int("p", 1, "count process")
	flag.Parse()

	netifs, err := net.Interfaces()
	md5hash := md5.New()
	if err != nil {
		panic(err)
	}
	for _, nif := range netifs {
		mac := nif.HardwareAddr.String()
		log.Printf("%s\n", mac)
		md5hash.Write([]byte(mac))
	}
	localId = hex.EncodeToString(md5hash.Sum(nil))
	log.Printf("localId:%s\n", localId)

	var i int
	defer func() {
		recover()
		mqtt.DEBUG.Printf("connected :%d", i)
	}()
	go func() {
		for {
			select {
			case c := <-newClientChan:
				clients = append(clients, c)
			case <-closeAll:
				for _, c := range clients {
					c.Disconnect(250)
				}
				close(closeAll)
				return
			}
		}
	}()
	//	go push(*host,*port, 0)
	for i = 1; i <= *p; i++ {
		ch := make(chan bool)
		go subscribe(*host, *port, i, ch)
		<-ch
	}
	signals := make(chan os.Signal, 1)

	signal.Notify(signals, os.Interrupt)
	select {
	case <-signals:
		//关闭服务器的端口监听，以退出
		closeAll <- true
	}
	<-closeAll
}
<|endoftext|>