<commit_before>// Contains utility functions for helping combine .jar files.
package java

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"zip"

	"gopkg.in/op/go-logging.v1"
)

var log = logging.MustGetLogger("zip_writer")

// AddZipFile copies the contents of a zip file into an existing zip writer.
func AddZipFile(w *zip.Writer, filepath string, include, exclude []string, stripPrefix string, strict bool, renameDirs map[string]string) error {
	r, err := zip.OpenReader(filepath)
	if err != nil {
		return err
	}
	defer r.Close()

	// Reopen file to get a directly readable version without decompression.
	r2, err := os.Open(filepath)
	if err != nil {
		return err
	}
	defer r2.Close()

outer:
	for _, f := range r.File {
		log.Debug("Found file %s (from %s)", f.Name, filepath)
		// This directory is very awkward. We need to merge the contents by concatenating them,
		// we can't replace them or leave them out.
		if strings.HasPrefix(f.Name, "META-INF/services/") ||
			strings.HasPrefix(f.Name, "META-INF/spring") ||
			f.Name == "META-INF/please_sourcemap" {
			if err := concatenateFile(w, f); err != nil {
				return err
			}
			continue
		}
		if !shouldInclude(f.Name, include, exclude) {
			continue outer
		}
		hasTrailingSlash := strings.HasSuffix(f.Name, "/")
		isDir := hasTrailingSlash || f.FileInfo().IsDir()
		if isDir && !hasTrailingSlash {
			f.Name = f.Name + "/"
		}
		if existing, present := getExistingFile(w, f.Name); present {
			// Allow duplicates of directories. Seemingly the best way to identify them is that
			// they end in a trailing slash.
			if isDir {
				continue
			}
			// Allow skipping existing files that are exactly the same as the added ones.
			// It's unnecessarily awkward to insist on not ever doubling up on a dependency.
			if existing.CRC32 == f.CRC32 {
				log.Info("Skipping %s / %s: already added (from %s)", filepath, f.Name, existing.ZipFile)
				continue
			}
			if strict {
				log.Error("Duplicate file %s (from %s, already added from %s); crc %d / %d", f.Name, filepath, existing.ZipFile, f.CRC32, existing.CRC32)
				return fmt.Errorf("File %s already added to destination zip file (from %s)", f.Name, existing.ZipFile)
			}
			continue
		}
		for before, after := range renameDirs {
			if strings.HasPrefix(f.Name, before) {
				f.Name = path.Join(after, strings.TrimPrefix(f.Name, before))
				if isDir {
					f.Name = f.Name + "/"
				}
				break
			}
		}
		f.Name = strings.TrimPrefix(f.Name, stripPrefix)
		// Java tools don't seem to like writing a data descriptor for stored items.
		// Unsure if this is a limitation of the format or a problem of those tools.
		f.Flags = 0
		addExistingFile(w, f.Name, filepath, f.CompressedSize64, f.UncompressedSize64, f.CRC32)

		start, err := f.DataOffset()
		if err != nil {
			return err
		}
		if _, err := r2.Seek(start, 0); err != nil {
			return err
		}
		// Make these deterministic.
		f.FileHeader.ModifiedDate = 0
		f.FileHeader.ModifiedTime = 0
		if err := addFile(w, &f.FileHeader, r2, f.CRC32); err != nil {
			return err
		}
	}
	return nil
}

func shouldInclude(name string, include, exclude []string) bool {
	for _, excl := range exclude {
		if matched, _ := filepath.Match(excl, name); matched {
			log.Debug("Skipping %s (excluded by %s)", name, excl)
			return false
		} else if matched, _ := filepath.Match(excl, filepath.Base(name)); matched {
			log.Debug("Skipping %s (excluded by %s)", name, excl)
			return false
		}
	}
	if len(include) == 0 {
		return true
	}
	for _, incl := range include {
		if matched, _ := filepath.Match(incl, name); matched || strings.HasPrefix(name, incl) {
			return true
		}
	}
	log.Debug("Skipping %s (didn't match any includes)", name)
	return false
}

// AddInitPyFiles adds an __init__.py file to every directory in the zip file that doesn't already have one.
func AddInitPyFiles(w *zip.Writer) error {
	m := files[w]
	s := make([]string, 0, len(m))
	for p := range m {
		s = append(s, p)
	}
	sort.Strings(s)
	for _, p := range s {
		d := filepath.Dir(p)
		if filepath.Base(d) == "__pycache__" {
			continue // Don't need to add an __init__.py here.
		}
		initPyPath := path.Join(d, "__init__.py")
		if _, present := m[initPyPath]; !present {
			// If we already have a pyc / pyo we don't need the __init__.py as well.
			if _, present := m[initPyPath+"c"]; present {
				continue
			} else if _, present := m[initPyPath+"o"]; present {
				continue
			}
			// Don't write one at the root, it's not necessary.
			if initPyPath != "__init__.py" {
				log.Debug("Adding %s", initPyPath)
				m[initPyPath] = fileRecord{}
				if err := WriteFile(w, initPyPath, []byte{}); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

// AddManifest adds a manifest to the given zip writer with a Main-Class entry (and a couple of others)
func AddManifest(w *zip.Writer, mainClass string) error {
	manifest := fmt.Sprintf("Manifest-Version: 1.0\nMain-Class: %s\n", mainClass)
	return WriteFile(w, "META-INF/MANIFEST.MF", []byte(manifest))
}

// Records some information about a file that we use to check if they're exact duplicates.
type fileRecord struct {
	ZipFile            string
	CompressedSize64   uint64
	UncompressedSize64 uint64
	CRC32              uint32
}

var files = map[*zip.Writer]map[string]fileRecord{}
var concatenatedFiles = map[string]string{}

func getExistingFile(w *zip.Writer, name string) (fileRecord, bool) {
	if m := files[w]; m != nil {
		record, present := m[name]
		return record, present
	}
	return fileRecord{}, false
}

// HasExistingFile returns true if the writer has already written the given file.
func HasExistingFile(w *zip.Writer, name string) bool {
	_, b := getExistingFile(w, name)
	return b
}

func addExistingFile(w *zip.Writer, name, file string, c, u uint64, crc uint32) {
	m := files[w]
	if m == nil {
		m = map[string]fileRecord{}
		files[w] = m
	}
	m[name] = fileRecord{file, c, u, crc}
}

// Add a file to the zip which is concatenated with any existing content with the same name.
// Writing is deferred since we obviously can't append to it later.
func concatenateFile(w *zip.Writer, f *zip.File) error {
	r, err := f.Open()
	if err != nil {
		return err
	}
	defer r.Close()
	buf := new(bytes.Buffer)
	if _, err := io.Copy(buf, r); err != nil {
		return err
	}
	contents := buf.String()
	if !strings.HasSuffix(contents, "\n") {
		contents += "\n"
	}
	concatenatedFiles[f.Name] += contents
	return nil
}

// HandleConcatenatedFiles appends concatenated files to the archive's directory for writing.
func HandleConcatenatedFiles(w *zip.Writer) error {
	// Must do it in a deterministic order
	files := make([]string, 0, len(concatenatedFiles))
	for name := range concatenatedFiles {
		files = append(files, name)
	}
	sort.Strings(files)
	for _, name := range files {
		if err := WriteFile(w, name, []byte(concatenatedFiles[name])); err != nil {
			return err
		}
	}
	return nil
}

// Writes a file to the new writer.
func addFile(w *zip.Writer, fh *zip.FileHeader, r io.Reader, crc uint32) error {
	fh.Flags = 0 // we're not writing a data descriptor after the file
	comp := func(w io.Writer) (io.WriteCloser, error) { return nopCloser{w}, nil }
	fw, err := w.CreateHeaderWithCompressor(fh, comp, fixedCrc32{value: crc})
	if err == nil {
		_, err = io.CopyN(fw, r, int64(fh.CompressedSize64))
	}
	return err
}

// WriteFile writes a complete file to the writer.
func WriteFile(w *zip.Writer, filename string, data []byte) error {
	if fw, err := w.Create(filename); err != nil {
		return err
	} else if _, err := fw.Write(data); err != nil {
		return err
	}
	return nil
}

// WriteDir writes a directory entry to the writer.
func WriteDir(w *zip.Writer, filename string) error {
	filename += "/" // Must have trailing slash to tell it it's a directory.
	fh := zip.FileHeader{
		Name:   filename,
		Method: zip.Store,
	}
	if _, err := w.CreateHeader(&fh); err != nil {
		return err
	}
	addExistingFile(w, filename, filename, 0, 0, 0)
	return nil
}

type nopCloser struct {
	io.Writer
}

func (w nopCloser) Close() error {
	return nil
}

// fixedCrc32 implements a Hash32 interface that just writes out a predetermined value.
// this is really cheating of course but serves our purposes here.
type fixedCrc32 struct {
	value uint32
}

func (crc fixedCrc32) Write(p []byte) (n int, err error) {
	return len(p), nil
}

func (crc fixedCrc32) Sum(b []byte) []byte {
	buf := make([]byte, 4)
	binary.LittleEndian.PutUint32(buf, crc.value)
	return b
}

func (crc fixedCrc32) Sum32() uint32 {
	return crc.value
}

func (crc fixedCrc32) Reset() {
}

func (crc fixedCrc32) Size() int {
	return 32
}

func (crc fixedCrc32) BlockSize() int {
	return 32
}
<commit_msg>Mark files outside a .zip as added so we don't duplicate __init__.py files in some cases.<commit_after>// Contains utility functions for helping combine .jar files.
package java

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"zip"

	"gopkg.in/op/go-logging.v1"
)

var log = logging.MustGetLogger("zip_writer")

// AddZipFile copies the contents of a zip file into an existing zip writer.
func AddZipFile(w *zip.Writer, filepath string, include, exclude []string, stripPrefix string, strict bool, renameDirs map[string]string) error {
	r, err := zip.OpenReader(filepath)
	if err != nil {
		return err
	}
	defer r.Close()

	// Reopen file to get a directly readable version without decompression.
	r2, err := os.Open(filepath)
	if err != nil {
		return err
	}
	defer r2.Close()

outer:
	for _, f := range r.File {
		log.Debug("Found file %s (from %s)", f.Name, filepath)
		// This directory is very awkward. We need to merge the contents by concatenating them,
		// we can't replace them or leave them out.
		if strings.HasPrefix(f.Name, "META-INF/services/") ||
			strings.HasPrefix(f.Name, "META-INF/spring") ||
			f.Name == "META-INF/please_sourcemap" {
			if err := concatenateFile(w, f); err != nil {
				return err
			}
			continue
		}
		if !shouldInclude(f.Name, include, exclude) {
			continue outer
		}
		hasTrailingSlash := strings.HasSuffix(f.Name, "/")
		isDir := hasTrailingSlash || f.FileInfo().IsDir()
		if isDir && !hasTrailingSlash {
			f.Name = f.Name + "/"
		}
		if existing, present := getExistingFile(w, f.Name); present {
			// Allow duplicates of directories. Seemingly the best way to identify them is that
			// they end in a trailing slash.
			if isDir {
				continue
			}
			// Allow skipping existing files that are exactly the same as the added ones.
			// It's unnecessarily awkward to insist on not ever doubling up on a dependency.
			if existing.CRC32 == f.CRC32 {
				log.Info("Skipping %s / %s: already added (from %s)", filepath, f.Name, existing.ZipFile)
				continue
			}
			if strict {
				log.Error("Duplicate file %s (from %s, already added from %s); crc %d / %d", f.Name, filepath, existing.ZipFile, f.CRC32, existing.CRC32)
				return fmt.Errorf("File %s already added to destination zip file (from %s)", f.Name, existing.ZipFile)
			}
			continue
		}
		for before, after := range renameDirs {
			if strings.HasPrefix(f.Name, before) {
				f.Name = path.Join(after, strings.TrimPrefix(f.Name, before))
				if isDir {
					f.Name = f.Name + "/"
				}
				break
			}
		}
		f.Name = strings.TrimPrefix(f.Name, stripPrefix)
		// Java tools don't seem to like writing a data descriptor for stored items.
		// Unsure if this is a limitation of the format or a problem of those tools.
		f.Flags = 0
		addExistingFile(w, f.Name, filepath, f.CompressedSize64, f.UncompressedSize64, f.CRC32)

		start, err := f.DataOffset()
		if err != nil {
			return err
		}
		if _, err := r2.Seek(start, 0); err != nil {
			return err
		}
		// Make these deterministic.
		f.FileHeader.ModifiedDate = 0
		f.FileHeader.ModifiedTime = 0
		if err := addFile(w, &f.FileHeader, r2, f.CRC32); err != nil {
			return err
		}
	}
	return nil
}

func shouldInclude(name string, include, exclude []string) bool {
	for _, excl := range exclude {
		if matched, _ := filepath.Match(excl, name); matched {
			log.Debug("Skipping %s (excluded by %s)", name, excl)
			return false
		} else if matched, _ := filepath.Match(excl, filepath.Base(name)); matched {
			log.Debug("Skipping %s (excluded by %s)", name, excl)
			return false
		}
	}
	if len(include) == 0 {
		return true
	}
	for _, incl := range include {
		if matched, _ := filepath.Match(incl, name); matched || strings.HasPrefix(name, incl) {
			return true
		}
	}
	log.Debug("Skipping %s (didn't match any includes)", name)
	return false
}

// AddInitPyFiles adds an __init__.py file to every directory in the zip file that doesn't already have one.
func AddInitPyFiles(w *zip.Writer) error {
	m := files[w]
	s := make([]string, 0, len(m))
	for p := range m {
		s = append(s, p)
	}
	sort.Strings(s)
	for _, p := range s {
		d := filepath.Dir(p)
		if filepath.Base(d) == "__pycache__" {
			continue // Don't need to add an __init__.py here.
		}
		initPyPath := path.Join(d, "__init__.py")
		if _, present := m[initPyPath]; !present {
			// If we already have a pyc / pyo we don't need the __init__.py as well.
			if _, present := m[initPyPath+"c"]; present {
				continue
			} else if _, present := m[initPyPath+"o"]; present {
				continue
			}
			// Don't write one at the root, it's not necessary.
			if initPyPath != "__init__.py" {
				log.Debug("Adding %s", initPyPath)
				m[initPyPath] = fileRecord{}
				if err := WriteFile(w, initPyPath, []byte{}); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

// AddManifest adds a manifest to the given zip writer with a Main-Class entry (and a couple of others)
func AddManifest(w *zip.Writer, mainClass string) error {
	manifest := fmt.Sprintf("Manifest-Version: 1.0\nMain-Class: %s\n", mainClass)
	return WriteFile(w, "META-INF/MANIFEST.MF", []byte(manifest))
}

// Records some information about a file that we use to check if they're exact duplicates.
type fileRecord struct {
	ZipFile            string
	CompressedSize64   uint64
	UncompressedSize64 uint64
	CRC32              uint32
}

var files = map[*zip.Writer]map[string]fileRecord{}
var concatenatedFiles = map[string]string{}

func getExistingFile(w *zip.Writer, name string) (fileRecord, bool) {
	if m := files[w]; m != nil {
		record, present := m[name]
		return record, present
	}
	return fileRecord{}, false
}

// HasExistingFile returns true if the writer has already written the given file.
func HasExistingFile(w *zip.Writer, name string) bool {
	_, b := getExistingFile(w, name)
	return b
}

func addExistingFile(w *zip.Writer, name, file string, c, u uint64, crc uint32) {
	m := files[w]
	if m == nil {
		m = map[string]fileRecord{}
		files[w] = m
	}
	m[name] = fileRecord{file, c, u, crc}
}

// Add a file to the zip which is concatenated with any existing content with the same name.
// Writing is deferred since we obviously can't append to it later.
func concatenateFile(w *zip.Writer, f *zip.File) error {
	r, err := f.Open()
	if err != nil {
		return err
	}
	defer r.Close()
	buf := new(bytes.Buffer)
	if _, err := io.Copy(buf, r); err != nil {
		return err
	}
	contents := buf.String()
	if !strings.HasSuffix(contents, "\n") {
		contents += "\n"
	}
	concatenatedFiles[f.Name] += contents
	return nil
}

// HandleConcatenatedFiles appends concatenated files to the archive's directory for writing.
func HandleConcatenatedFiles(w *zip.Writer) error {
	// Must do it in a deterministic order
	files := make([]string, 0, len(concatenatedFiles))
	for name := range concatenatedFiles {
		files = append(files, name)
	}
	sort.Strings(files)
	for _, name := range files {
		if err := WriteFile(w, name, []byte(concatenatedFiles[name])); err != nil {
			return err
		}
	}
	return nil
}

// Writes a file to the new writer.
func addFile(w *zip.Writer, fh *zip.FileHeader, r io.Reader, crc uint32) error {
	fh.Flags = 0 // we're not writing a data descriptor after the file
	comp := func(w io.Writer) (io.WriteCloser, error) { return nopCloser{w}, nil }
	fw, err := w.CreateHeaderWithCompressor(fh, comp, fixedCrc32{value: crc})
	if err == nil {
		_, err = io.CopyN(fw, r, int64(fh.CompressedSize64))
	}
	return err
}

// WriteFile writes a complete file to the writer.
func WriteFile(w *zip.Writer, filename string, data []byte) error {
	if fw, err := w.Create(filename); err != nil {
		return err
	} else if _, err := fw.Write(data); err != nil {
		return err
	}
	addExistingFile(w, filename, filename, 0, 0, 0)
	return nil
}

// WriteDir writes a directory entry to the writer.
func WriteDir(w *zip.Writer, filename string) error {
	filename += "/" // Must have trailing slash to tell it it's a directory.
	fh := zip.FileHeader{
		Name:   filename,
		Method: zip.Store,
	}
	if _, err := w.CreateHeader(&fh); err != nil {
		return err
	}
	addExistingFile(w, filename, filename, 0, 0, 0)
	return nil
}

type nopCloser struct {
	io.Writer
}

func (w nopCloser) Close() error {
	return nil
}

// fixedCrc32 implements a Hash32 interface that just writes out a predetermined value.
// this is really cheating of course but serves our purposes here.
type fixedCrc32 struct {
	value uint32
}

func (crc fixedCrc32) Write(p []byte) (n int, err error) {
	return len(p), nil
}

func (crc fixedCrc32) Sum(b []byte) []byte {
	buf := make([]byte, 4)
	binary.LittleEndian.PutUint32(buf, crc.value)
	return b
}

func (crc fixedCrc32) Sum32() uint32 {
	return crc.value
}

func (crc fixedCrc32) Reset() {
}

func (crc fixedCrc32) Size() int {
	return 32
}

func (crc fixedCrc32) BlockSize() int {
	return 32
}
<|endoftext|>


<commit_before>package utils

import (
	"database/sql"
	"fmt"
	"reflect"
	"strings"
	"sync"
	"time"
)

// SQLScan helper class for reading sql to Struct
// Columns in struct must be marked with a `sql:"col_name"` tag
// Ex: in sql a column name is col1, in struct the col tag must be `sql:"col1"`
type SQLScan struct {
	sync.RWMutex
	columnNames []string
}

// Clear - clears the columns array.
// Used to be able to reuse the scan helper for another SQL
func (s *SQLScan) Clear() {
	s.Lock()
	defer s.Unlock()

	s.columnNames = nil
}

// Scan - reads sql statement into a struct
func (s *SQLScan) Scan(u *DbUtils, rows *sql.Rows, dest interface{}) error {
	s.Lock()
	defer s.Unlock()

	if s.columnNames == nil || len(s.columnNames) == 0 {
		cols, err := rows.Columns()
		if err != nil {
			return err
		}

		s.columnNames = cols

		if u.dbType == Oci8 || u.dbType == Oracle || u.dbType == Oracle11g {
			for i, colName := range s.columnNames {
				if colName[0:1] != "\"" {
					s.columnNames[i] = strings.ToLower(colName)
				}
			}
		}
	}

	nrCols := len(s.columnNames)
	pointers := make([]interface{}, nrCols)
	fieldTypes := make([]reflect.Type, nrCols)

	structVal := reflect.ValueOf(dest).Elem()
	nFields := structVal.NumField()

	rnum := 0
	rnumPos := -1

	for i, colName := range s.columnNames {
		if u.dbType == Oracle && colName == "rnumignore" {
			rnumPos = i
			pointersAux := make([]interface{}, nrCols+1)

			for k := 0; k < i; k++ {
				pointersAux[k] = pointers[k]
			}

			pointersAux[i] = &rnum

			for k := i; k < nrCols; k++ {
				pointersAux[k+1] = pointers[k]
			}

			pointers = pointersAux

			continue
		}

		for j := 0; j < nFields; j++ {
			typeField := structVal.Type().Field(j)
			tag := typeField.Tag

			if tag.Get("sql") == colName {
				pointers[i] = structVal.Field(j).Addr().Interface()
				fieldTypes[i] = typeField.Type
				break
			}
		}
	}

	fmt.Println("pointers", pointers)

	err := rows.Scan(pointers...)
	if err != nil {
		return err
	}

	if u.dbType == Oci8 || u.dbType == Oracle || u.dbType == Oracle11g {
		// in oci, the timestamp is comming up as local time zone
		// even if you ask for the UTC
		dt := time.Now()
		dtnull := NullTime{}
		dtType := reflect.TypeOf(dt)
		dtnullType := reflect.TypeOf(dtnull)

		for i := 0; i < nrCols; i++ {
			if fieldTypes[i] == dtType {
				if rnumPos < 0 {
					dtval := pointers[i].(*time.Time)
					strdt := Date2string(*dtval, ISODateTimestamp)
					*dtval = String2dateNoErr(strdt, UTCDateTimestamp)
				} else if i >= rnumPos {
					dtval := pointers[i+1].(*time.Time)
					strdt := Date2string(*dtval, ISODateTimestamp)
					*dtval = String2dateNoErr(strdt, UTCDateTimestamp)
				}
			} else if fieldTypes[i] == dtnullType {
				if rnumPos < 0 {
					dtval := pointers[i].(*NullTime)
					if dtval.Valid {
						strdt := Date2string((*dtval).Time, ISODateTimestamp)
						(*dtval).Time = String2dateNoErr(strdt, UTCDateTimestamp)
					}
				} else if i >= rnumPos {
					dtval := pointers[i+1].(*NullTime)
					if dtval.Valid {
						strdt := Date2string((*dtval).Time, ISODateTimestamp)
						(*dtval).Time = String2dateNoErr(strdt, UTCDateTimestamp)
					}
				}
			}
		}
	}

	return nil
}
<commit_msg>Fix: limit and offset for oracle 11g<commit_after>package utils

import (
	"database/sql"
	"reflect"
	"strings"
	"sync"
	"time"
)

// SQLScan helper class for reading sql to Struct
// Columns in struct must be marked with a `sql:"col_name"` tag
// Ex: in sql a column name is col1, in struct the col tag must be `sql:"col1"`
type SQLScan struct {
	sync.RWMutex
	columnNames []string
}

// Clear - clears the columns array.
// Used to be able to reuse the scan helper for another SQL
func (s *SQLScan) Clear() {
	s.Lock()
	defer s.Unlock()

	s.columnNames = nil
}

// Scan - reads sql statement into a struct
func (s *SQLScan) Scan(u *DbUtils, rows *sql.Rows, dest interface{}) error {
	s.Lock()
	defer s.Unlock()

	if s.columnNames == nil || len(s.columnNames) == 0 {
		cols, err := rows.Columns()
		if err != nil {
			return err
		}

		s.columnNames = cols

		if u.dbType == Oci8 || u.dbType == Oracle || u.dbType == Oracle11g {
			for i, colName := range s.columnNames {
				if colName[0:1] != "\"" {
					s.columnNames[i] = strings.ToLower(colName)
				}
			}
		}
	}

	nrCols := len(s.columnNames)
	pointers := make([]interface{}, nrCols)
	fieldTypes := make([]reflect.Type, nrCols)

	structVal := reflect.ValueOf(dest).Elem()
	nFields := structVal.NumField()

	rnum := 0
	rnumPos := -1

	for i, colName := range s.columnNames {
		if u.dbType == Oracle11g && colName == "rnumignore" {
			rnumPos = i

			var pointersAux []interface{}
			pointersAux = append(pointersAux, pointers[:i]...)
			pointersAux = append(pointersAux, &rnum)
			pointersAux = append(pointersAux, pointers[i:]...)
			pointers = pointersAux

			continue
		}

		for j := 0; j < nFields; j++ {
			typeField := structVal.Type().Field(j)
			tag := typeField.Tag

			if tag.Get("sql") == colName {
				pointers[i] = structVal.Field(j).Addr().Interface()
				fieldTypes[i] = typeField.Type
				break
			}
		}
	}

	err := rows.Scan(pointers...)
	if err != nil {
		return err
	}

	if u.dbType == Oci8 || u.dbType == Oracle || u.dbType == Oracle11g {
		// in oci, the timestamp is comming up as local time zone
		// even if you ask for the UTC
		dt := time.Now()
		dtnull := NullTime{}
		dtType := reflect.TypeOf(dt)
		dtnullType := reflect.TypeOf(dtnull)

		for i := 0; i < nrCols; i++ {
			if fieldTypes[i] == dtType {
				if rnumPos < 0 {
					dtval := pointers[i].(*time.Time)
					strdt := Date2string(*dtval, ISODateTimestamp)
					*dtval = String2dateNoErr(strdt, UTCDateTimestamp)
				} else if i >= rnumPos {
					dtval := pointers[i+1].(*time.Time)
					strdt := Date2string(*dtval, ISODateTimestamp)
					*dtval = String2dateNoErr(strdt, UTCDateTimestamp)
				}
			} else if fieldTypes[i] == dtnullType {
				if rnumPos < 0 {
					dtval := pointers[i].(*NullTime)
					if dtval.Valid {
						strdt := Date2string((*dtval).Time, ISODateTimestamp)
						(*dtval).Time = String2dateNoErr(strdt, UTCDateTimestamp)
					}
				} else if i >= rnumPos {
					dtval := pointers[i+1].(*NullTime)
					if dtval.Valid {
						strdt := Date2string((*dtval).Time, ISODateTimestamp)
						(*dtval).Time = String2dateNoErr(strdt, UTCDateTimestamp)
					}
				}
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package utils

import (
	"net/url"
	"reflect"
	"time"
)

const (
	_TAG_FIELD = "Field"
)

func IsMap(obj interface{}) bool {
	objT := reflect.TypeOf(obj)
	if objT.Kind() == reflect.Map {
		return true
	}
	return false
}
func HasFieldofStruct(obj interface{}, fieldName string) bool {

	_, rv, err :=GetStructTypeValue(obj)
	if err != nil {
		return false
	}
	//	rv := reflect.ValueOf(obj)
	val := rv.FieldByName(fieldName)
	return val.IsValid()
}
func GetRealType(obj interface{}) reflect.Type {
	objT := reflect.TypeOf(obj)
	if objT.Kind() == reflect.Ptr {
		return objT.Elem()
	}
	return objT
}

func CreateObjByType(t reflect.Type) interface{} {
	return reflect.New(t).Interface()
}

func GetStructTypeValue(obj interface{}) (reflect.Type, reflect.Value, error) {
	objT := reflect.TypeOf(obj)
	objV := reflect.ValueOf(obj)
	switch {
	case objT.Kind() == reflect.Struct:
	case objT.Kind() == reflect.Ptr && objT.Elem().Kind() == reflect.Struct:
		objT = objT.Elem()
		objV = objV.Elem()
	default:
		return objT, objV, CreateError(500, "Must set a struct or a struct pointer!")

	}
	return objT, objV, nil
}

func FillStructByForm(values url.Values, target interface{}) error {
	objT, objV, err := GetStructTypeValue(target)
	if err != nil {
		return err
	}

	for i := 0; i < objT.NumField(); i++ {
		f := objT.Field(i)
		vf := objV.Field(i)
		if !vf.CanInterface() {
			continue
		}

		colName := getFormFieldName(f)
		v := values.Get(colName)
		if v != "" && vf.CanSet() {
			setFieldValue(vf, v)
		}
	}
	return nil
}

func FillStruct(values map[string]interface{}, target interface{}) error {
	objT, objV, err := GetStructTypeValue(target)
	if err != nil {
		return err
	}

	for i := 0; i < objT.NumField(); i++ {
		f := objT.Field(i)
		vf := objV.Field(i)
		if !vf.CanInterface() {
			continue
		}

		colName := GetColName(f)
		v := values[colName]
		if v != nil && vf.CanSet() {
			setFieldValue(vf, v)
		}
	}

	return nil

}

func getFormFieldName(field reflect.StructField) string {
	//	colName := strings.ToLower(field.Name)
	//	tagField := field.Tag.Get(_TAG_FIELD)
	//	if tagField != "" {
	//		colName = tagField
	//	}
	//	return colName
	return GetColName(field)
}

func GetColName(field reflect.StructField) string {
	colName := field.Tag.Get(_TAG_FIELD)
	if colName == "" {
		colName = BingoString(field.Name).SnakeString()
	}

	return colName
}

func setFieldValue(ind reflect.Value, value interface{}) {
	var val interface{}
	objT := reflect.TypeOf(value)
	if objT.Kind() == reflect.Ptr {

		val = reflect.ValueOf(value).Elem().Interface()
	} else {
		val = value
	}

	switch ind.Kind() {
	case reflect.Bool:
		setBool(ind, val)
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		setInt(ind, val)
	case reflect.String:
		setString(ind, val)
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		setUint(ind, val)
	case reflect.Float64, reflect.Float32:
		setFloat(ind, val)
	case reflect.Struct:
		if value == nil {
			ind.Set(reflect.Zero(ind.Type()))

		} else if _, ok := ind.Interface().(time.Time); ok {
			setTime(ind, val)
		}
	}

}
func setString(ind reflect.Value, value interface{}) {
	if value == nil {
		ind.SetString("")
	} else {
		ind.SetString(ToStr(value))
	}
}

func setBool(ind reflect.Value, value interface{}) {
	if value == nil {
		ind.SetBool(false)
	} else if v, ok := value.(bool); ok {
		ind.SetBool(v)
	} else {
		v, _ := BingoString(ToStr(value)).Bool()
		ind.SetBool(v)
	}
}

func setInt(ind reflect.Value, value interface{}) {
	if value == nil {
		ind.SetInt(0)
	} else {
		val := reflect.ValueOf(value)
		switch val.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			ind.SetInt(val.Int())
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			ind.SetInt(int64(val.Uint()))
		default:
			v, _ := BingoString(ToStr(value)).Int64()
			ind.SetInt(v)
		}
	}
}

func setUint(ind reflect.Value, value interface{}) {
	if value == nil {
		ind.SetUint(0)
	} else {
		val := reflect.ValueOf(value)
		switch val.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			ind.SetUint(uint64(val.Int()))
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			ind.SetUint(val.Uint())
		default:
			v, _ := BingoString(ToStr(value)).Uint64()
			ind.SetUint(v)
		}
	}
}

func setFloat(ind reflect.Value, value interface{}) {
	if value == nil {
		ind.SetFloat(0)
	} else {
		val := reflect.ValueOf(value)
		switch val.Kind() {
		case reflect.Float64:
			ind.SetFloat(val.Float())
		default:
			v, _ := BingoString(ToStr(value)).Float64()
			ind.SetFloat(v)
		}
	}
}

func setTime(ind reflect.Value, value interface{}) {

}
<commit_msg>修复未校验对象是否有效<commit_after>package utils

import (
	"net/url"
	"reflect"
	"time"
)

const (
	_TAG_FIELD = "Field"
)

func IsMap(obj interface{}) bool {
	objT := reflect.TypeOf(obj)
	if objT.Kind() == reflect.Map {
		return true
	}
	return false
}
func HasFieldofStruct(obj interface{}, fieldName string) bool {

	_, rv, err :=GetStructTypeValue(obj)
	if err != nil {
		return false
	}
	if rv.IsValid(){

	//	rv := reflect.ValueOf(obj)
	val := rv.FieldByName(fieldName)
	return val.IsValid()
	}
	return  false
}
func GetRealType(obj interface{}) reflect.Type {
	objT := reflect.TypeOf(obj)
	if objT.Kind() == reflect.Ptr {
		return objT.Elem()
	}
	return objT
}

func CreateObjByType(t reflect.Type) interface{} {
	return reflect.New(t).Interface()
}

func GetStructTypeValue(obj interface{}) (reflect.Type, reflect.Value, error) {
	objT := reflect.TypeOf(obj)
	objV := reflect.ValueOf(obj)
	switch {
	case objT.Kind() == reflect.Struct:
	case objT.Kind() == reflect.Ptr && objT.Elem().Kind() == reflect.Struct:
		objT = objT.Elem()
		objV = objV.Elem()
	default:
		return objT, objV, CreateError(500, "Must set a struct or a struct pointer!")

	}
	return objT, objV, nil
}

func FillStructByForm(values url.Values, target interface{}) error {
	objT, objV, err := GetStructTypeValue(target)
	if err != nil {
		return err
	}

	for i := 0; i < objT.NumField(); i++ {
		f := objT.Field(i)
		vf := objV.Field(i)
		if !vf.CanInterface() {
			continue
		}

		colName := getFormFieldName(f)
		v := values.Get(colName)
		if v != "" && vf.CanSet() {
			setFieldValue(vf, v)
		}
	}
	return nil
}

func FillStruct(values map[string]interface{}, target interface{}) error {
	objT, objV, err := GetStructTypeValue(target)
	if err != nil {
		return err
	}

	for i := 0; i < objT.NumField(); i++ {
		f := objT.Field(i)
		vf := objV.Field(i)
		if !vf.CanInterface() {
			continue
		}

		colName := GetColName(f)
		v := values[colName]
		if v != nil && vf.CanSet() {
			setFieldValue(vf, v)
		}
	}

	return nil

}

func getFormFieldName(field reflect.StructField) string {
	//	colName := strings.ToLower(field.Name)
	//	tagField := field.Tag.Get(_TAG_FIELD)
	//	if tagField != "" {
	//		colName = tagField
	//	}
	//	return colName
	return GetColName(field)
}

func GetColName(field reflect.StructField) string {
	colName := field.Tag.Get(_TAG_FIELD)
	if colName == "" {
		colName = BingoString(field.Name).SnakeString()
	}

	return colName
}

func setFieldValue(ind reflect.Value, value interface{}) {
	var val interface{}
	objT := reflect.TypeOf(value)
	if objT.Kind() == reflect.Ptr {

		val = reflect.ValueOf(value).Elem().Interface()
	} else {
		val = value
	}

	switch ind.Kind() {
	case reflect.Bool:
		setBool(ind, val)
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		setInt(ind, val)
	case reflect.String:
		setString(ind, val)
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		setUint(ind, val)
	case reflect.Float64, reflect.Float32:
		setFloat(ind, val)
	case reflect.Struct:
		if value == nil {
			ind.Set(reflect.Zero(ind.Type()))

		} else if _, ok := ind.Interface().(time.Time); ok {
			setTime(ind, val)
		}
	}

}
func setString(ind reflect.Value, value interface{}) {
	if value == nil {
		ind.SetString("")
	} else {
		ind.SetString(ToStr(value))
	}
}

func setBool(ind reflect.Value, value interface{}) {
	if value == nil {
		ind.SetBool(false)
	} else if v, ok := value.(bool); ok {
		ind.SetBool(v)
	} else {
		v, _ := BingoString(ToStr(value)).Bool()
		ind.SetBool(v)
	}
}

func setInt(ind reflect.Value, value interface{}) {
	if value == nil {
		ind.SetInt(0)
	} else {
		val := reflect.ValueOf(value)
		switch val.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			ind.SetInt(val.Int())
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			ind.SetInt(int64(val.Uint()))
		default:
			v, _ := BingoString(ToStr(value)).Int64()
			ind.SetInt(v)
		}
	}
}

func setUint(ind reflect.Value, value interface{}) {
	if value == nil {
		ind.SetUint(0)
	} else {
		val := reflect.ValueOf(value)
		switch val.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			ind.SetUint(uint64(val.Int()))
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			ind.SetUint(val.Uint())
		default:
			v, _ := BingoString(ToStr(value)).Uint64()
			ind.SetUint(v)
		}
	}
}

func setFloat(ind reflect.Value, value interface{}) {
	if value == nil {
		ind.SetFloat(0)
	} else {
		val := reflect.ValueOf(value)
		switch val.Kind() {
		case reflect.Float64:
			ind.SetFloat(val.Float())
		default:
			v, _ := BingoString(ToStr(value)).Float64()
			ind.SetFloat(v)
		}
	}
}

func setTime(ind reflect.Value, value interface{}) {

}
<|endoftext|>


<commit_before>package framework

import (
	"context"
	"fmt"
	"time"

	"github.com/hashicorp/vault/sdk/logical"
)

// LeaseExtend is left for backwards compatibility for plugins. This function
// now just passes back the data that was passed into it to be processed in core.
// DEPRECATED
func LeaseExtend(backendIncrement, backendMax time.Duration, systemView logical.SystemView) OperationFunc {
	return func(ctx context.Context, req *logical.Request, data *FieldData) (*logical.Response, error) {
		switch {
		case req.Auth != nil:
			req.Auth.TTL = backendIncrement
			req.Auth.MaxTTL = backendMax
			return &logical.Response{Auth: req.Auth}, nil
		case req.Secret != nil:
			req.Secret.TTL = backendIncrement
			req.Secret.MaxTTL = backendMax
			return &logical.Response{Secret: req.Secret}, nil
		}
		return nil, fmt.Errorf("no lease options for request")
	}
}

// CalculateTTL takes all the user-specified, backend, and system inputs and calculates
// a TTL for a lease
func CalculateTTL(sysView logical.SystemView, increment, backendTTL, period, backendMaxTTL, explicitMaxTTL time.Duration, startTime time.Time) (ttl time.Duration, warnings []string, errors error) {
	// Truncate all times to the second since that is the lowest precision for
	// TTLs
	now := time.Now().Truncate(time.Second)
	if startTime.IsZero() {
		startTime = now
	} else {
		startTime = startTime.Truncate(time.Second)
	}

	// Use the mount's configured max unless the backend specifies
	// something more restrictive (perhaps from a role configuration
	// parameter)
	maxTTL := sysView.MaxLeaseTTL()
	if backendMaxTTL > 0 && backendMaxTTL < maxTTL {
		maxTTL = backendMaxTTL
	}
	if explicitMaxTTL > 0 && explicitMaxTTL < maxTTL {
		maxTTL = explicitMaxTTL
	}

	// Should never happen, but guard anyways
	if maxTTL <= 0 {
		return 0, nil, fmt.Errorf("max TTL must be greater than zero")
	}

	var maxValidTime time.Time
	switch {
	case period > 0:
		// Cap the period value to the sys max_ttl value
		if period > maxTTL {
			warnings = append(warnings,
				fmt.Sprintf("period of %q exceeded the effective max_ttl of %q; period value is capped accordingly", period, maxTTL))
			period = maxTTL
		}
		ttl = period

		if explicitMaxTTL > 0 {
			maxValidTime = startTime.Add(explicitMaxTTL)
		}
	default:
		switch {
		case increment > 0:
			ttl = increment
		case backendTTL > 0:
			ttl = backendTTL
		default:
			ttl = sysView.DefaultLeaseTTL()
		}

		// We cannot go past this time
		maxValidTime = startTime.Add(maxTTL)
	}

	if !maxValidTime.IsZero() {
		// Determine the max valid TTL
		maxValidTTL := maxValidTime.Sub(now)

		// If we are past the max TTL, we shouldn't be in this function...but
		// fast path out if we are
		if maxValidTTL < 0 {
			return 0, nil, fmt.Errorf("past the max TTL, cannot renew")
		}

		// If the proposed expiration is after the maximum TTL of the lease,
		// cap the increment to whatever is left
		if maxValidTTL-ttl < 0 {
			warnings = append(warnings,
				fmt.Sprintf("TTL of %q exceeded the effective max_ttl of %q; TTL value is capped accordingly", ttl, maxValidTTL))
			ttl = maxValidTTL
		}
	}

	return ttl, warnings, nil
}
<commit_msg>Output human duration in TTL warnings (#7901)<commit_after>package framework

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/hashicorp/vault/sdk/logical"
)

// LeaseExtend is left for backwards compatibility for plugins. This function
// now just passes back the data that was passed into it to be processed in core.
// DEPRECATED
func LeaseExtend(backendIncrement, backendMax time.Duration, systemView logical.SystemView) OperationFunc {
	return func(ctx context.Context, req *logical.Request, data *FieldData) (*logical.Response, error) {
		switch {
		case req.Auth != nil:
			req.Auth.TTL = backendIncrement
			req.Auth.MaxTTL = backendMax
			return &logical.Response{Auth: req.Auth}, nil
		case req.Secret != nil:
			req.Secret.TTL = backendIncrement
			req.Secret.MaxTTL = backendMax
			return &logical.Response{Secret: req.Secret}, nil
		}
		return nil, fmt.Errorf("no lease options for request")
	}
}

// CalculateTTL takes all the user-specified, backend, and system inputs and calculates
// a TTL for a lease
func CalculateTTL(sysView logical.SystemView, increment, backendTTL, period, backendMaxTTL, explicitMaxTTL time.Duration, startTime time.Time) (ttl time.Duration, warnings []string, errors error) {
	// Truncate all times to the second since that is the lowest precision for
	// TTLs
	now := time.Now().Truncate(time.Second)
	if startTime.IsZero() {
		startTime = now
	} else {
		startTime = startTime.Truncate(time.Second)
	}

	// Use the mount's configured max unless the backend specifies
	// something more restrictive (perhaps from a role configuration
	// parameter)
	maxTTL := sysView.MaxLeaseTTL()
	if backendMaxTTL > 0 && backendMaxTTL < maxTTL {
		maxTTL = backendMaxTTL
	}
	if explicitMaxTTL > 0 && explicitMaxTTL < maxTTL {
		maxTTL = explicitMaxTTL
	}

	// Should never happen, but guard anyways
	if maxTTL <= 0 {
		return 0, nil, fmt.Errorf("max TTL must be greater than zero")
	}

	var maxValidTime time.Time
	switch {
	case period > 0:
		// Cap the period value to the sys max_ttl value
		if period > maxTTL {
			warnings = append(warnings,
				fmt.Sprintf("period of %q exceeded the effective max_ttl of %q; period value is capped accordingly",
					humanDuration(period), humanDuration(maxTTL)))
			period = maxTTL
		}
		ttl = period

		if explicitMaxTTL > 0 {
			maxValidTime = startTime.Add(explicitMaxTTL)
		}
	default:
		switch {
		case increment > 0:
			ttl = increment
		case backendTTL > 0:
			ttl = backendTTL
		default:
			ttl = sysView.DefaultLeaseTTL()
		}

		// We cannot go past this time
		maxValidTime = startTime.Add(maxTTL)
	}

	if !maxValidTime.IsZero() {
		// Determine the max valid TTL
		maxValidTTL := maxValidTime.Sub(now)

		// If we are past the max TTL, we shouldn't be in this function...but
		// fast path out if we are
		if maxValidTTL < 0 {
			return 0, nil, fmt.Errorf("past the max TTL, cannot renew")
		}

		// If the proposed expiration is after the maximum TTL of the lease,
		// cap the increment to whatever is left
		if maxValidTTL-ttl < 0 {
			warnings = append(warnings,
				fmt.Sprintf("TTL of %q exceeded the effective max_ttl of %q; TTL value is capped accordingly",
					humanDuration(ttl), humanDuration(maxValidTTL)))
			ttl = maxValidTTL
		}
	}

	return ttl, warnings, nil
}

// humanDuration prints the time duration without zero elements.
func humanDuration(d time.Duration) string {
	if d == 0 {
		return "0s"
	}

	s := d.String()
	if strings.HasSuffix(s, "m0s") {
		s = s[:len(s)-2]
	}
	if idx := strings.Index(s, "h0m"); idx > 0 {
		s = s[:idx+1] + s[idx+3:]
	}
	return s
}
<|endoftext|>


<commit_before>package logrus_syslog

import (
	"fmt"
	"github.com/Sirupsen/logrus"
	"log/syslog"
	"os"
)

// SyslogHook to send logs via syslog.
type SyslogHook struct {
	Writer        *syslog.Writer
	SyslogNetwork string
	SyslogRaddr   string
}

// Creates a hook to be added to an instance of logger. This is called with
// `hook, err := NewSyslogHook("udp", "localhost:514", syslog.LOG_DEBUG, "")`
// `if err == nil { log.Hooks.Add(hook) }`
func NewSyslogHook(network, raddr string, priority syslog.Priority, tag string) (*SyslogHook, error) {
	w, err := syslog.Dial(network, raddr, priority, tag)
	return &SyslogHook{w, network, raddr}, err
}

func (hook *SyslogHook) Fire(entry *logrus.Entry) error {
	line, err := entry.String()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to read entry, %v", err)
		return err
	}

	switch entry.Level {
	case logrus.PanicLevel:
		return hook.Writer.Crit(line)
	case logrus.FatalLevel:
		return hook.Writer.Crit(line)
	case logrus.ErrorLevel:
		return hook.Writer.Err(line)
	case logrus.WarnLevel:
		return hook.Writer.Warning(line)
	case logrus.InfoLevel:
		return hook.Writer.Info(line)
	case logrus.DebugLevel:
		return hook.Writer.Debug(line)
	default:
		return nil
	}
}

func (hook *SyslogHook) Levels() []logrus.Level {
	return []logrus.Level{
		logrus.PanicLevel,
		logrus.FatalLevel,
		logrus.ErrorLevel,
		logrus.WarnLevel,
		logrus.InfoLevel,
		logrus.DebugLevel,
	}
}
<commit_msg>Added Build Rule<commit_after>// +build !windows,!nacl,!plan9

package logrus_syslog

import (
	"fmt"
	"github.com/Sirupsen/logrus"
	"log/syslog"
	"os"
)

// SyslogHook to send logs via syslog.
type SyslogHook struct {
	Writer        *syslog.Writer
	SyslogNetwork string
	SyslogRaddr   string
}

// Creates a hook to be added to an instance of logger. This is called with
// `hook, err := NewSyslogHook("udp", "localhost:514", syslog.LOG_DEBUG, "")`
// `if err == nil { log.Hooks.Add(hook) }`
func NewSyslogHook(network, raddr string, priority syslog.Priority, tag string) (*SyslogHook, error) {
	w, err := syslog.Dial(network, raddr, priority, tag)
	return &SyslogHook{w, network, raddr}, err
}

func (hook *SyslogHook) Fire(entry *logrus.Entry) error {
	line, err := entry.String()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to read entry, %v", err)
		return err
	}

	switch entry.Level {
	case logrus.PanicLevel:
		return hook.Writer.Crit(line)
	case logrus.FatalLevel:
		return hook.Writer.Crit(line)
	case logrus.ErrorLevel:
		return hook.Writer.Err(line)
	case logrus.WarnLevel:
		return hook.Writer.Warning(line)
	case logrus.InfoLevel:
		return hook.Writer.Info(line)
	case logrus.DebugLevel:
		return hook.Writer.Debug(line)
	default:
		return nil
	}
}

func (hook *SyslogHook) Levels() []logrus.Level {
	return []logrus.Level{
		logrus.PanicLevel,
		logrus.FatalLevel,
		logrus.ErrorLevel,
		logrus.WarnLevel,
		logrus.InfoLevel,
		logrus.DebugLevel,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"sort"
	"strings"

	"github.com/spf13/cobra"

	"github.com/weaveworks/flux/api/v6"
	"github.com/weaveworks/flux/policy"
)

type workloadListOpts struct {
	*rootOpts
	namespace     string
	allNamespaces bool
}

func newWorkloadList(parent *rootOpts) *workloadListOpts {
	return &workloadListOpts{rootOpts: parent}
}

func (opts *workloadListOpts) Command() *cobra.Command {
	cmd := &cobra.Command{
		Use:     "list-workloads",
		Aliases: []string{"list-controllers"}, // Transient backwards compatibility after replacing controller by workload
		Short:   "List workloads currently running in the cluster.",
		Example: makeExample("fluxctl list-workloads"),
		RunE:    opts.RunE,
	}
	cmd.Flags().StringVarP(&opts.namespace, "namespace", "n", "default", "Confine query to namespace")
	cmd.Flags().BoolVarP(&opts.allNamespaces, "all-namespaces", "a", false, "Query across all namespaces")
	return cmd
}

func (opts *workloadListOpts) RunE(cmd *cobra.Command, args []string) error {
	if len(args) != 0 {
		return errorWantedNoArgs
	}

	if opts.allNamespaces {
		opts.namespace = ""
	}

	ctx := context.Background()

	workloads, err := opts.API.ListServices(ctx, opts.namespace)
	if err != nil {
		return err
	}

	sort.Sort(workloadStatusByName(workloads))

	w := newTabwriter()
	fmt.Fprintf(w, "WORKLOAD\tCONTAINER\tIMAGE\tRELEASE\tPOLICY\n")
	for _, workload := range workloads {
		if len(workload.Containers) > 0 {
			c := workload.Containers[0]
			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", workload.ID, c.Name, c.Current.ID, workload.Status, policies(workload))
			for _, c := range workload.Containers[1:] {
				fmt.Fprintf(w, "\t%s\t%s\t\t\n", c.Name, c.Current.ID)
			}
		} else {
			fmt.Fprintf(w, "%s\t\t\t\t\n", workload.ID)
		}
	}
	w.Flush()
	return nil
}

type workloadStatusByName []v6.ControllerStatus

func (s workloadStatusByName) Len() int {
	return len(s)
}

func (s workloadStatusByName) Less(a, b int) bool {
	return s[a].ID.String() < s[b].ID.String()
}

func (s workloadStatusByName) Swap(a, b int) {
	s[a], s[b] = s[b], s[a]
}

func policies(s v6.ControllerStatus) string {
	var ps []string
	if s.Automated {
		ps = append(ps, string(policy.Automated))
	}
	if s.Locked {
		ps = append(ps, string(policy.Locked))
	}
	if s.Ignore {
		ps = append(ps, string(policy.Ignore))
	}
	sort.Strings(ps)
	return strings.Join(ps, ",")
}
<commit_msg>Always list the status of a workload in `fluxctl` (#2035)<commit_after>package main

import (
	"context"
	"fmt"
	"sort"
	"strings"

	"github.com/spf13/cobra"

	"github.com/weaveworks/flux/api/v6"
	"github.com/weaveworks/flux/policy"
)

type workloadListOpts struct {
	*rootOpts
	namespace     string
	allNamespaces bool
}

func newWorkloadList(parent *rootOpts) *workloadListOpts {
	return &workloadListOpts{rootOpts: parent}
}

func (opts *workloadListOpts) Command() *cobra.Command {
	cmd := &cobra.Command{
		Use:     "list-workloads",
		Aliases: []string{"list-controllers"}, // Transient backwards compatibility after replacing controller by workload
		Short:   "List workloads currently running in the cluster.",
		Example: makeExample("fluxctl list-workloads"),
		RunE:    opts.RunE,
	}
	cmd.Flags().StringVarP(&opts.namespace, "namespace", "n", "default", "Confine query to namespace")
	cmd.Flags().BoolVarP(&opts.allNamespaces, "all-namespaces", "a", false, "Query across all namespaces")
	return cmd
}

func (opts *workloadListOpts) RunE(cmd *cobra.Command, args []string) error {
	if len(args) != 0 {
		return errorWantedNoArgs
	}

	if opts.allNamespaces {
		opts.namespace = ""
	}

	ctx := context.Background()

	workloads, err := opts.API.ListServices(ctx, opts.namespace)
	if err != nil {
		return err
	}

	sort.Sort(workloadStatusByName(workloads))

	w := newTabwriter()
	fmt.Fprintf(w, "WORKLOAD\tCONTAINER\tIMAGE\tRELEASE\tPOLICY\n")
	for _, workload := range workloads {
		if len(workload.Containers) > 0 {
			c := workload.Containers[0]
			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", workload.ID, c.Name, c.Current.ID, workload.Status, policies(workload))
			for _, c := range workload.Containers[1:] {
				fmt.Fprintf(w, "\t%s\t%s\t\t\n", c.Name, c.Current.ID)
			}
		} else {
			fmt.Fprintf(w, "%s\t\t\t%s\t%s\n", workload.ID, workload.Status, policies(workload))
		}
	}
	w.Flush()
	return nil
}

type workloadStatusByName []v6.ControllerStatus

func (s workloadStatusByName) Len() int {
	return len(s)
}

func (s workloadStatusByName) Less(a, b int) bool {
	return s[a].ID.String() < s[b].ID.String()
}

func (s workloadStatusByName) Swap(a, b int) {
	s[a], s[b] = s[b], s[a]
}

func policies(s v6.ControllerStatus) string {
	var ps []string
	if s.Automated {
		ps = append(ps, string(policy.Automated))
	}
	if s.Locked {
		ps = append(ps, string(policy.Locked))
	}
	if s.Ignore {
		ps = append(ps, string(policy.Ignore))
	}
	sort.Strings(ps)
	return strings.Join(ps, ",")
}
<|endoftext|>


<commit_before>package kafka

import (
	"github.com/Shopify/sarama"
	"github.com/eapache/go-resiliency/breaker"
	"github.com/funkygao/gafka/cmd/kateway/store"
	log "github.com/funkygao/log4go"
)

func (this *pubStore) doSyncPub(allAck bool, cluster, topic string,
	key, msg []byte) (partition int32, offset int64, err error) {
	this.pubPoolsLock.RLock()
	pool, present := this.pubPools[cluster]
	this.pubPoolsLock.RUnlock()
	if !present {
		err = store.ErrInvalidCluster
		return
	}

	if pool.breaker.Open() {
		err = store.ErrCircuitOpen
		return
	}

	var (
		producer   *syncProducerClient
		keyEncoder sarama.Encoder = nil // will use random partitioner
	)
	if len(key) > 0 {
		keyEncoder = sarama.ByteEncoder(key) // will use hash partition
	}

	producerMsg := &sarama.ProducerMessage{
		Topic: topic,
		Key:   keyEncoder,
		Value: sarama.ByteEncoder(msg),
	}

	getProducer := pool.GetSyncProducer
	if allAck {
		getProducer = pool.GetSyncAllProducer
	}

	if this.dryRun {
		// ignore kafka I/O
		producer, err = getProducer()
		producer.Recycle()
		return
	}

	producer, err = getProducer()
	if err != nil {
		// e,g. during factory method, kafka breaks down
		pool.breaker.Fail()

		if producer != nil {
			// should never happen
			producer.CloseAndRecycle()
		}

		return
	}

	// sarama will retry Producer.Retry.Max(3) times on produce failure before return error
	// meanwhile, it will auto refresh meta
	partition, offset, err = producer.SendMessage(producerMsg)
	if err == nil {
		// send ok
		pool.breaker.Succeed()
		producer.Recycle()
		return
	}

	log.Warn("cluster[%s] topic:%s %v", cluster, topic, err)
	switch err {
	// read tcp 10.209.36.33:50607->10.209.18.16:11005: i/o timeout
	// dial tcp 10.209.18.65:11005: getsockopt: connection refused

	case sarama.ErrUnknownTopicOrPartition, sarama.ErrInvalidTopic:
		// this conn is still valid
		pool.breaker.Succeed()
		producer.Recycle()
		err = store.ErrInvalidTopic
		return

	case breaker.ErrBreakerOpen, sarama.ErrOutOfBrokers:
		// sarama is using breaker: 3 error/1 success/10s
		// will not retry FIXME breaker didn't work
		pool.breaker.Fail()
		producer.CloseAndRecycle()
		// err = store.ErrBusy TODO hide the underlying err
		return

	default:
		// e,g. sarama.ErrLeaderNotAvailable, sarama.ErrNotLeaderForPartition
		// will retry
		pool.breaker.Fail()
		producer.CloseAndRecycle()
		// err = store.ErrBusy TODO hide the underlying err
	}

	return
}

func (this *pubStore) IsSystemError(err error) bool {
	switch err {
	case store.ErrInvalidCluster:
		return false

	default:
		return true
	}
}

func (this *pubStore) SyncAllPub(cluster, topic string, key, msg []byte) (partition int32, offset int64, err error) {
	return this.doSyncPub(true, cluster, topic, key, msg)
}

func (this *pubStore) SyncPub(cluster, topic string, key, msg []byte) (partition int32, offset int64, err error) {
	return this.doSyncPub(false, cluster, topic, key, msg)
}

// FIXME not fully fault tolerant like SyncPub.
func (this *pubStore) AsyncPub(cluster string, topic string, key []byte,
	msg []byte) (partition int32, offset int64, err error) {
	this.pubPoolsLock.RLock()
	pool, present := this.pubPools[cluster]
	this.pubPoolsLock.RUnlock()
	if !present {
		err = store.ErrInvalidCluster
		return
	}

	producer, e := pool.GetAsyncProducer()
	if e != nil {
		if producer != nil {
			producer.Recycle()
		}

		err = e
		return
	}

	var keyEncoder sarama.Encoder = nil // will use random partitioner
	if len(key) > 0 {
		keyEncoder = sarama.ByteEncoder(key) // will use hash partition
	}

	// TODO can be pooled
	producer.Input() <- &sarama.ProducerMessage{
		Topic: topic,
		Key:   keyEncoder,
		Value: sarama.ByteEncoder(msg),
	}
	producer.Recycle()
	return
}
<commit_msg>tweak of log message<commit_after>package kafka

import (
	"github.com/Shopify/sarama"
	"github.com/eapache/go-resiliency/breaker"
	"github.com/funkygao/gafka/cmd/kateway/store"
	log "github.com/funkygao/log4go"
)

func (this *pubStore) doSyncPub(allAck bool, cluster, topic string,
	key, msg []byte) (partition int32, offset int64, err error) {
	this.pubPoolsLock.RLock()
	pool, present := this.pubPools[cluster]
	this.pubPoolsLock.RUnlock()
	if !present {
		err = store.ErrInvalidCluster
		return
	}

	if pool.breaker.Open() {
		err = store.ErrCircuitOpen
		return
	}

	var (
		producer   *syncProducerClient
		keyEncoder sarama.Encoder = nil // will use random partitioner
	)
	if len(key) > 0 {
		keyEncoder = sarama.ByteEncoder(key) // will use hash partition
	}

	producerMsg := &sarama.ProducerMessage{
		Topic: topic,
		Key:   keyEncoder,
		Value: sarama.ByteEncoder(msg),
	}

	getProducer := pool.GetSyncProducer
	if allAck {
		getProducer = pool.GetSyncAllProducer
	}

	if this.dryRun {
		// ignore kafka I/O
		producer, err = getProducer()
		producer.Recycle()
		return
	}

	producer, err = getProducer()
	if err != nil {
		// e,g. during factory method, kafka breaks down
		pool.breaker.Fail()

		if producer != nil {
			// should never happen
			producer.CloseAndRecycle()
		}

		return
	}

	// sarama will retry Producer.Retry.Max(3) times on produce failure before return error
	// meanwhile, it will auto refresh meta
	partition, offset, err = producer.SendMessage(producerMsg)
	if err == nil {
		// send ok
		pool.breaker.Succeed()
		producer.Recycle()
		return
	}

	log.Error("cluster[%s] topic:%s %v", cluster, topic, err)
	switch err {
	// read tcp 10.209.36.33:50607->10.209.18.16:11005: i/o timeout
	// dial tcp 10.209.18.65:11005: getsockopt: connection refused

	case sarama.ErrUnknownTopicOrPartition, sarama.ErrInvalidTopic:
		// this conn is still valid
		pool.breaker.Succeed()
		producer.Recycle()
		err = store.ErrInvalidTopic
		return

	case breaker.ErrBreakerOpen, sarama.ErrOutOfBrokers:
		// sarama is using breaker: 3 error/1 success/10s
		// will not retry FIXME breaker didn't work
		pool.breaker.Fail()
		producer.CloseAndRecycle()
		// err = store.ErrBusy TODO hide the underlying err
		return

	default:
		// e,g. sarama.ErrLeaderNotAvailable, sarama.ErrNotLeaderForPartition
		// will retry
		pool.breaker.Fail()
		producer.CloseAndRecycle()
		// err = store.ErrBusy TODO hide the underlying err
	}

	return
}

func (this *pubStore) IsSystemError(err error) bool {
	switch err {
	case store.ErrInvalidCluster:
		return false

	default:
		return true
	}
}

func (this *pubStore) SyncAllPub(cluster, topic string, key, msg []byte) (partition int32, offset int64, err error) {
	return this.doSyncPub(true, cluster, topic, key, msg)
}

func (this *pubStore) SyncPub(cluster, topic string, key, msg []byte) (partition int32, offset int64, err error) {
	return this.doSyncPub(false, cluster, topic, key, msg)
}

// FIXME not fully fault tolerant like SyncPub.
func (this *pubStore) AsyncPub(cluster string, topic string, key []byte,
	msg []byte) (partition int32, offset int64, err error) {
	this.pubPoolsLock.RLock()
	pool, present := this.pubPools[cluster]
	this.pubPoolsLock.RUnlock()
	if !present {
		err = store.ErrInvalidCluster
		return
	}

	producer, e := pool.GetAsyncProducer()
	if e != nil {
		if producer != nil {
			producer.Recycle()
		}

		err = e
		return
	}

	var keyEncoder sarama.Encoder = nil // will use random partitioner
	if len(key) > 0 {
		keyEncoder = sarama.ByteEncoder(key) // will use hash partition
	}

	// TODO can be pooled
	producer.Input() <- &sarama.ProducerMessage{
		Topic: topic,
		Key:   keyEncoder,
		Value: sarama.ByteEncoder(msg),
	}
	producer.Recycle()
	return
}
<|endoftext|>


<commit_before>// Package collect provides functions for sending data to OpenTSDB.
//
// The "collect" namespace is used (i.e., <metric_root>.collect) to collect
// program and queue metrics.
package collect

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/StackExchange/scollector/opentsdb"
)

var (
	// Freq is how often metrics are sent to OpenTSDB.
	Freq = time.Second * 15

	// MaxQueueLen is the maximum size of the queue, above which incoming data will
	// be discarded. Defaults to about 150MB.
	MaxQueueLen = 200000

	// BatchSize is the maximum length of data points sent at once to OpenTSDB.
	BatchSize = 50

	// Debug enables debug logging.
	Debug = false

	// Dropped is the number of dropped data points due to a full queue.
	dropped int64

	// Sent is the number of sent data points.
	sent int64

	tchan               chan *opentsdb.DataPoint
	tsdbURL             string
	osHostname          string
	metricRoot          string
	queue               opentsdb.MultiDataPoint
	qlock, mlock, slock sync.Mutex   // Locks for queues, maps, stats.
	counters                         = make(map[string]*addMetric)
	sets                             = make(map[string]*setMetric)
	puts                             = make(map[string]*putMetric)
	client              *http.Client = &http.Client{
		Transport: &timeoutTransport{Transport: new(http.Transport)},
		Timeout:   time.Minute,
	}
)

type timeoutTransport struct {
	*http.Transport
	Timeout time.Time
}

func (t *timeoutTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	if time.Now().After(t.Timeout) {
		t.Transport.CloseIdleConnections()
		t.Timeout = time.Now().Add(time.Minute * 5)
	}
	return t.Transport.RoundTrip(r)
}

// InitChan is similar to Init, but uses the given channel instead of creating a
// new one.
func InitChan(tsdbhost *url.URL, metric_root string, ch chan *opentsdb.DataPoint) error {
	if tchan != nil {
		return fmt.Errorf("cannot init twice")
	}
	if err := checkClean(metric_root, "metric root"); err != nil {
		return err
	}
	u, err := tsdbhost.Parse("/api/put")
	if err != nil {
		return err
	}
	if strings.HasPrefix(u.Host, ":") {
		u.Host = "localhost" + u.Host
	}
	tsdbURL = u.String()
	metricRoot = metric_root + "."
	tchan = ch
	go func() {
		for dp := range tchan {
			qlock.Lock()
			for {
				if len(queue) > MaxQueueLen {
					slock.Lock()
					dropped++
					slock.Unlock()
					break
				}
				queue = append(queue, dp)
				select {
				case dp = <-tchan:
					continue
				default:
				}
				break
			}
			qlock.Unlock()
		}
	}()
	go send()

	go collect()
	Set("collect.dropped", nil, func() (i interface{}) {
		slock.Lock()
		i = dropped
		slock.Unlock()
		return
	})
	Set("collect.sent", nil, func() (i interface{}) {
		slock.Lock()
		i = sent
		slock.Unlock()
		return
	})
	Set("collect.alloc", nil, func() interface{} {
		var ms runtime.MemStats
		runtime.ReadMemStats(&ms)
		return ms.Alloc
	})
	Set("collect.goroutines", nil, func() interface{} {
		return runtime.NumGoroutine()
	})
	return nil
}

// Init sets up the channels and the queue for sending data to OpenTSDB. It also
// sets up the basename for all metrics.
func Init(tsdbhost *url.URL, metric_root string) error {
	return InitChan(tsdbhost, metric_root, make(chan *opentsdb.DataPoint))
}

func setHostName() error {
	h, err := os.Hostname()
	if err != nil {
		return err
	}
	osHostname = strings.ToLower(strings.SplitN(h, ".", 2)[0])
	if err := checkClean(osHostname, "host tag"); err != nil {
		return err
	}
	return nil
}

type setMetric struct {
	metric string
	ts     opentsdb.TagSet
	f      func() interface{}
}

func Set(metric string, ts opentsdb.TagSet, f func() interface{}) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	sets[tss] = &setMetric{metric, ts.Copy(), f}
	mlock.Unlock()
	return nil
}

type addMetric struct {
	metric string
	ts     opentsdb.TagSet
	value  int64
}

// Add takes a metric and increments a counter for that metric. The metric name
// is appended to the basename specified in the Init function.
func Add(metric string, ts opentsdb.TagSet, inc int64) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	if counters[tss] == nil {
		counters[tss] = &addMetric{
			metric: metric,
			ts:     ts.Copy(),
		}
	}
	counters[tss].value += inc
	mlock.Unlock()
	return nil
}

type putMetric struct {
	metric string
	ts     opentsdb.TagSet
	value  interface{}
}

// Put is useful for capturing "events" that have a gauge value. Subsequent
// calls between the sending interval will overwrite previous calls.
func Put(metric string, ts opentsdb.TagSet, v interface{}) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	puts[tss] = &putMetric{metric, ts.Copy(), v}
	mlock.Unlock()
	return nil
}

func check(metric string, ts *opentsdb.TagSet) error {
	if err := checkClean(metric, "metric"); err != nil {
		return err
	}
	for k, v := range *ts {
		if err := checkClean(k, "tagk"); err != nil {
			return err
		}
		if err := checkClean(v, "tagv"); err != nil {
			return err
		}
	}
	if osHostname == "" {
		if err := setHostName(); err != nil {
			return err
		}
	}
	if *ts == nil {
		*ts = make(opentsdb.TagSet)
	}
	if (*ts)["host"] == "" {
		(*ts)["host"] = osHostname
	}
	return nil
}

func checkClean(s, t string) error {
	if sc, err := opentsdb.Clean(s); s != sc || err != nil {
		if err != nil {
			return err
		}
		return fmt.Errorf("%s %s may only contain a to z, A to Z, 0 to 9, -, _, ., / or Unicode letters and may not be empty", t, s)
	}
	return nil
}

func collect() {
	for {
		mlock.Lock()
		now := time.Now().Unix()
		for _, c := range counters {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + c.metric,
				Timestamp: now,
				Value:     c.value,
				Tags:      c.ts,
			}
			tchan <- dp
		}
		for _, s := range sets {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + s.metric,
				Timestamp: now,
				Value:     s.f(),
				Tags:      s.ts,
			}
			tchan <- dp
		}
		for _, s := range puts {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + s.metric,
				Timestamp: now,
				Value:     s.value,
				Tags:      s.ts,
			}
			tchan <- dp
		}
		puts = make(map[string]*putMetric)
		mlock.Unlock()
		time.Sleep(Freq)
	}
}
<commit_msg>cmd/scollector: Record queue size<commit_after>// Package collect provides functions for sending data to OpenTSDB.
//
// The "collect" namespace is used (i.e., <metric_root>.collect) to collect
// program and queue metrics.
package collect

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/StackExchange/scollector/opentsdb"
)

var (
	// Freq is how often metrics are sent to OpenTSDB.
	Freq = time.Second * 15

	// MaxQueueLen is the maximum size of the queue, above which incoming data will
	// be discarded. Defaults to about 150MB.
	MaxQueueLen = 200000

	// BatchSize is the maximum length of data points sent at once to OpenTSDB.
	BatchSize = 50

	// Debug enables debug logging.
	Debug = false

	// Dropped is the number of dropped data points due to a full queue.
	dropped int64

	// Sent is the number of sent data points.
	sent int64

	tchan               chan *opentsdb.DataPoint
	tsdbURL             string
	osHostname          string
	metricRoot          string
	queue               opentsdb.MultiDataPoint
	qlock, mlock, slock sync.Mutex   // Locks for queues, maps, stats.
	counters                         = make(map[string]*addMetric)
	sets                             = make(map[string]*setMetric)
	puts                             = make(map[string]*putMetric)
	client              *http.Client = &http.Client{
		Transport: &timeoutTransport{Transport: new(http.Transport)},
		Timeout:   time.Minute,
	}
)

type timeoutTransport struct {
	*http.Transport
	Timeout time.Time
}

func (t *timeoutTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	if time.Now().After(t.Timeout) {
		t.Transport.CloseIdleConnections()
		t.Timeout = time.Now().Add(time.Minute * 5)
	}
	return t.Transport.RoundTrip(r)
}

// InitChan is similar to Init, but uses the given channel instead of creating a
// new one.
func InitChan(tsdbhost *url.URL, metric_root string, ch chan *opentsdb.DataPoint) error {
	if tchan != nil {
		return fmt.Errorf("cannot init twice")
	}
	if err := checkClean(metric_root, "metric root"); err != nil {
		return err
	}
	u, err := tsdbhost.Parse("/api/put")
	if err != nil {
		return err
	}
	if strings.HasPrefix(u.Host, ":") {
		u.Host = "localhost" + u.Host
	}
	tsdbURL = u.String()
	metricRoot = metric_root + "."
	tchan = ch
	go func() {
		for dp := range tchan {
			qlock.Lock()
			for {
				if len(queue) > MaxQueueLen {
					slock.Lock()
					dropped++
					slock.Unlock()
					break
				}
				queue = append(queue, dp)
				select {
				case dp = <-tchan:
					continue
				default:
				}
				break
			}
			qlock.Unlock()
		}
	}()
	go send()

	go collect()
	Set("collect.dropped", nil, func() (i interface{}) {
		slock.Lock()
		i = dropped
		slock.Unlock()
		return
	})
	Set("collect.sent", nil, func() (i interface{}) {
		slock.Lock()
		i = sent
		slock.Unlock()
		return
	})
	Set("collect.queued", nil, func() (i interface{}) {
		qlock.Lock()
		i = len(queue)
		qlock.Unlock()
		return
	})
	Set("collect.alloc", nil, func() interface{} {
		var ms runtime.MemStats
		runtime.ReadMemStats(&ms)
		return ms.Alloc
	})
	Set("collect.goroutines", nil, func() interface{} {
		return runtime.NumGoroutine()
	})
	return nil
}

// Init sets up the channels and the queue for sending data to OpenTSDB. It also
// sets up the basename for all metrics.
func Init(tsdbhost *url.URL, metric_root string) error {
	return InitChan(tsdbhost, metric_root, make(chan *opentsdb.DataPoint))
}

func setHostName() error {
	h, err := os.Hostname()
	if err != nil {
		return err
	}
	osHostname = strings.ToLower(strings.SplitN(h, ".", 2)[0])
	if err := checkClean(osHostname, "host tag"); err != nil {
		return err
	}
	return nil
}

type setMetric struct {
	metric string
	ts     opentsdb.TagSet
	f      func() interface{}
}

func Set(metric string, ts opentsdb.TagSet, f func() interface{}) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	sets[tss] = &setMetric{metric, ts.Copy(), f}
	mlock.Unlock()
	return nil
}

type addMetric struct {
	metric string
	ts     opentsdb.TagSet
	value  int64
}

// Add takes a metric and increments a counter for that metric. The metric name
// is appended to the basename specified in the Init function.
func Add(metric string, ts opentsdb.TagSet, inc int64) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	if counters[tss] == nil {
		counters[tss] = &addMetric{
			metric: metric,
			ts:     ts.Copy(),
		}
	}
	counters[tss].value += inc
	mlock.Unlock()
	return nil
}

type putMetric struct {
	metric string
	ts     opentsdb.TagSet
	value  interface{}
}

// Put is useful for capturing "events" that have a gauge value. Subsequent
// calls between the sending interval will overwrite previous calls.
func Put(metric string, ts opentsdb.TagSet, v interface{}) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	puts[tss] = &putMetric{metric, ts.Copy(), v}
	mlock.Unlock()
	return nil
}

func check(metric string, ts *opentsdb.TagSet) error {
	if err := checkClean(metric, "metric"); err != nil {
		return err
	}
	for k, v := range *ts {
		if err := checkClean(k, "tagk"); err != nil {
			return err
		}
		if err := checkClean(v, "tagv"); err != nil {
			return err
		}
	}
	if osHostname == "" {
		if err := setHostName(); err != nil {
			return err
		}
	}
	if *ts == nil {
		*ts = make(opentsdb.TagSet)
	}
	if (*ts)["host"] == "" {
		(*ts)["host"] = osHostname
	}
	return nil
}

func checkClean(s, t string) error {
	if sc, err := opentsdb.Clean(s); s != sc || err != nil {
		if err != nil {
			return err
		}
		return fmt.Errorf("%s %s may only contain a to z, A to Z, 0 to 9, -, _, ., / or Unicode letters and may not be empty", t, s)
	}
	return nil
}

func collect() {
	for {
		mlock.Lock()
		now := time.Now().Unix()
		for _, c := range counters {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + c.metric,
				Timestamp: now,
				Value:     c.value,
				Tags:      c.ts,
			}
			tchan <- dp
		}
		for _, s := range sets {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + s.metric,
				Timestamp: now,
				Value:     s.f(),
				Tags:      s.ts,
			}
			tchan <- dp
		}
		for _, s := range puts {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + s.metric,
				Timestamp: now,
				Value:     s.value,
				Tags:      s.ts,
			}
			tchan <- dp
		}
		puts = make(map[string]*putMetric)
		mlock.Unlock()
		time.Sleep(Freq)
	}
}
<|endoftext|>


<commit_before>package ca

import (
	"crypto"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/hex"
	"fmt"
	"log"
	"math"
	"math/big"
	"time"

	"github.com/letsencrypt/pebble/acme"
	"github.com/letsencrypt/pebble/core"
	"github.com/letsencrypt/pebble/db"
)

const (
	rootCAPrefix         = "Pebble Root CA "
	intermediateCAPrefix = "Pebble Intermediate CA "
)

type CAImpl struct {
	log *log.Logger
	db  *db.MemoryStore

	root         *issuer
	intermediate *issuer
}

type issuer struct {
	key  crypto.Signer
	cert *core.Certificate
}

func makeSerial() *big.Int {
	serial, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
	if err != nil {
		panic(fmt.Sprintf("unable to create random serial number: %s", err.Error()))
	}
	return serial
}

// makeKey and makeRootCert are adapted from MiniCA:
// https://github.com/jsha/minica/blob/3a621c05b61fa1c24bcb42fbde4b261db504a74f/main.go

// makeKey creates a new 2048 bit RSA private key
func makeKey() (*rsa.PrivateKey, error) {
	key, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, err
	}
	return key, nil
}

func (ca *CAImpl) makeRootCert(
	subjectKey crypto.Signer,
	subjCNPrefix string,
	signer *issuer) (*core.Certificate, error) {

	serial := makeSerial()
	template := &x509.Certificate{
		Subject: pkix.Name{
			CommonName: subjCNPrefix + hex.EncodeToString(serial.Bytes()[:3]),
		},
		SerialNumber: serial,
		NotBefore:    time.Now(),
		NotAfter:     time.Now().AddDate(30, 0, 0),

		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
		BasicConstraintsValid: true,
		IsCA: true,
	}

	var signerKey crypto.Signer
	if signer != nil && signer.key != nil {
		signerKey = signer.key
	} else {
		signerKey = subjectKey
	}

	der, err := x509.CreateCertificate(rand.Reader, template, template, subjectKey.Public(), signerKey)
	if err != nil {
		return nil, err
	}

	cert, err := x509.ParseCertificate(der)
	if err != nil {
		return nil, err
	}

	hexSerial := hex.EncodeToString(cert.SerialNumber.Bytes())
	newCert := &core.Certificate{
		ID:   hexSerial,
		Cert: cert,
		DER:  der,
	}
	if signer != nil && signer.cert != nil {
		newCert.Issuer = signer.cert
	}
	_, err = ca.db.AddCertificate(newCert)
	if err != nil {
		return nil, err
	}
	return newCert, nil
}

func (ca *CAImpl) newRootIssuer() error {
	// Make a root private key
	rk, err := makeKey()
	if err != nil {
		return err
	}
	// Make a self-signed root certificate
	rc, err := ca.makeRootCert(rk, rootCAPrefix, nil)
	if err != nil {
		return err
	}

	ca.root = &issuer{
		key:  rk,
		cert: rc,
	}
	ca.log.Printf("Generated new root issuer with serial %s\n", rc.ID)
	return nil
}

func (ca *CAImpl) newIntermediateIssuer() error {
	if ca.root == nil {
		return fmt.Errorf("newIntermediateIssuer() called before newRootIssuer()")
	}

	// Make an intermediate private key
	ik, err := makeKey()
	if err != nil {
		return err
	}

	// Make an intermediate certificate with the root issuer
	ic, err := ca.makeRootCert(ik, intermediateCAPrefix, ca.root)
	if err != nil {
		return err
	}
	ca.intermediate = &issuer{
		key:  ik,
		cert: ic,
	}
	ca.log.Printf("Generated new intermediate issuer with serial %s\n", ic.ID)
	return nil
}

func (ca *CAImpl) newCertificate(domains []string, key crypto.PublicKey, accountID string) (*core.Certificate, error) {
	var cn string
	if len(domains) > 0 {
		cn = domains[0]
	} else {
		return nil, fmt.Errorf("must specify at least one domain name")
	}

	issuer := ca.intermediate
	if issuer == nil || issuer.cert == nil {
		return nil, fmt.Errorf("cannot sign certificate - nil issuer")
	}

	serial := makeSerial()
	template := &x509.Certificate{
		DNSNames: domains,
		Subject: pkix.Name{
			CommonName: cn,
		},
		SerialNumber: serial,
		NotBefore:    time.Now(),
		NotAfter:     time.Now().AddDate(5, 0, 0),

		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
		BasicConstraintsValid: true,
		IsCA: false,
	}
	der, err := x509.CreateCertificate(rand.Reader, template, issuer.cert.Cert, key, issuer.key)
	if err != nil {
		return nil, err
	}
	cert, err := x509.ParseCertificate(der)
	if err != nil {
		return nil, err
	}

	hexSerial := hex.EncodeToString(cert.SerialNumber.Bytes())
	newCert := &core.Certificate{
		ID:        hexSerial,
		AccountID: accountID,
		Cert:      cert,
		DER:       der,
		Issuer:    issuer.cert,
	}
	_, err = ca.db.AddCertificate(newCert)
	if err != nil {
		return nil, err
	}
	return newCert, nil
}

func New(log *log.Logger, db *db.MemoryStore) *CAImpl {
	ca := &CAImpl{
		log: log,
		db:  db,
	}
	err := ca.newRootIssuer()
	if err != nil {
		panic(fmt.Sprintf("Error creating new root issuer: %s", err.Error()))
	}
	err = ca.newIntermediateIssuer()
	if err != nil {
		panic(fmt.Sprintf("Error creating new intermediate issuer: %s", err.Error()))
	}
	return ca
}

func (ca *CAImpl) CompleteOrder(order *core.Order) {
	// Lock the order for reading
	order.RLock()
	// If the order isn't set as beganProcessing produce an error and immediately unlock
	if !order.BeganProcessing {
		ca.log.Printf("Error: Asked to complete order %s which had false beganProcessing.",
			order.ID)
		order.RUnlock()
		return
	}
	// Unlock the order again
	order.RUnlock()

	// Check the authorizations - this is done by the VA before calling
	// CompleteOrder but we do it again for robustness sake.
	for _, authz := range order.AuthorizationObjects {
		// Lock the authorization for reading
		authz.RLock()
		if authz.Status != acme.StatusValid {
			return
		}
		authz.RUnlock()
	}

	// issue a certificate for the csr
	csr := order.ParsedCSR
	cert, err := ca.newCertificate(csr.DNSNames, csr.PublicKey, order.AccountID)
	if err != nil {
		ca.log.Printf("Error: unable to issue order: %s", err.Error())
		return
	}
	ca.log.Printf("Issued certificate serial %s for order %s\n", cert.ID, order.ID)

	// Lock and update the order to store the issued certificate
	order.Lock()
	order.CertificateObject = cert
	order.Unlock()
}
<commit_msg>Fix issue where intermediate CA was self signed (#148)<commit_after>package ca

import (
	"crypto"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/hex"
	"fmt"
	"log"
	"math"
	"math/big"
	"time"

	"github.com/letsencrypt/pebble/acme"
	"github.com/letsencrypt/pebble/core"
	"github.com/letsencrypt/pebble/db"
)

const (
	rootCAPrefix         = "Pebble Root CA "
	intermediateCAPrefix = "Pebble Intermediate CA "
)

type CAImpl struct {
	log *log.Logger
	db  *db.MemoryStore

	root         *issuer
	intermediate *issuer
}

type issuer struct {
	key  crypto.Signer
	cert *core.Certificate
}

func makeSerial() *big.Int {
	serial, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
	if err != nil {
		panic(fmt.Sprintf("unable to create random serial number: %s", err.Error()))
	}
	return serial
}

// makeKey and makeRootCert are adapted from MiniCA:
// https://github.com/jsha/minica/blob/3a621c05b61fa1c24bcb42fbde4b261db504a74f/main.go

// makeKey creates a new 2048 bit RSA private key
func makeKey() (*rsa.PrivateKey, error) {
	key, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, err
	}
	return key, nil
}

func (ca *CAImpl) makeRootCert(
	subjectKey crypto.Signer,
	subjCNPrefix string,
	signer *issuer) (*core.Certificate, error) {

	serial := makeSerial()
	template := &x509.Certificate{
		Subject: pkix.Name{
			CommonName: subjCNPrefix + hex.EncodeToString(serial.Bytes()[:3]),
		},
		SerialNumber: serial,
		NotBefore:    time.Now(),
		NotAfter:     time.Now().AddDate(30, 0, 0),

		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
		BasicConstraintsValid: true,
		IsCA: true,
	}

	var signerKey crypto.Signer
	var parent *x509.Certificate
	if signer != nil && signer.key != nil && signer.cert != nil && signer.cert.Cert != nil {
		signerKey = signer.key
		parent = signer.cert.Cert
	} else {
		signerKey = subjectKey
		parent = template
	}

	der, err := x509.CreateCertificate(rand.Reader, template, parent, subjectKey.Public(), signerKey)
	if err != nil {
		return nil, err
	}

	cert, err := x509.ParseCertificate(der)
	if err != nil {
		return nil, err
	}

	hexSerial := hex.EncodeToString(cert.SerialNumber.Bytes())
	newCert := &core.Certificate{
		ID:   hexSerial,
		Cert: cert,
		DER:  der,
	}
	if signer != nil && signer.cert != nil {
		newCert.Issuer = signer.cert
	}
	_, err = ca.db.AddCertificate(newCert)
	if err != nil {
		return nil, err
	}
	return newCert, nil
}

func (ca *CAImpl) newRootIssuer() error {
	// Make a root private key
	rk, err := makeKey()
	if err != nil {
		return err
	}
	// Make a self-signed root certificate
	rc, err := ca.makeRootCert(rk, rootCAPrefix, nil)
	if err != nil {
		return err
	}

	ca.root = &issuer{
		key:  rk,
		cert: rc,
	}
	ca.log.Printf("Generated new root issuer with serial %s\n", rc.ID)
	return nil
}

func (ca *CAImpl) newIntermediateIssuer() error {
	if ca.root == nil {
		return fmt.Errorf("newIntermediateIssuer() called before newRootIssuer()")
	}

	// Make an intermediate private key
	ik, err := makeKey()
	if err != nil {
		return err
	}

	// Make an intermediate certificate with the root issuer
	ic, err := ca.makeRootCert(ik, intermediateCAPrefix, ca.root)
	if err != nil {
		return err
	}
	ca.intermediate = &issuer{
		key:  ik,
		cert: ic,
	}
	ca.log.Printf("Generated new intermediate issuer with serial %s\n", ic.ID)
	return nil
}

func (ca *CAImpl) newCertificate(domains []string, key crypto.PublicKey, accountID string) (*core.Certificate, error) {
	var cn string
	if len(domains) > 0 {
		cn = domains[0]
	} else {
		return nil, fmt.Errorf("must specify at least one domain name")
	}

	issuer := ca.intermediate
	if issuer == nil || issuer.cert == nil {
		return nil, fmt.Errorf("cannot sign certificate - nil issuer")
	}

	serial := makeSerial()
	template := &x509.Certificate{
		DNSNames: domains,
		Subject: pkix.Name{
			CommonName: cn,
		},
		SerialNumber: serial,
		NotBefore:    time.Now(),
		NotAfter:     time.Now().AddDate(5, 0, 0),

		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
		BasicConstraintsValid: true,
		IsCA: false,
	}
	der, err := x509.CreateCertificate(rand.Reader, template, issuer.cert.Cert, key, issuer.key)
	if err != nil {
		return nil, err
	}
	cert, err := x509.ParseCertificate(der)
	if err != nil {
		return nil, err
	}

	hexSerial := hex.EncodeToString(cert.SerialNumber.Bytes())
	newCert := &core.Certificate{
		ID:        hexSerial,
		AccountID: accountID,
		Cert:      cert,
		DER:       der,
		Issuer:    issuer.cert,
	}
	_, err = ca.db.AddCertificate(newCert)
	if err != nil {
		return nil, err
	}
	return newCert, nil
}

func New(log *log.Logger, db *db.MemoryStore) *CAImpl {
	ca := &CAImpl{
		log: log,
		db:  db,
	}
	err := ca.newRootIssuer()
	if err != nil {
		panic(fmt.Sprintf("Error creating new root issuer: %s", err.Error()))
	}
	err = ca.newIntermediateIssuer()
	if err != nil {
		panic(fmt.Sprintf("Error creating new intermediate issuer: %s", err.Error()))
	}
	return ca
}

func (ca *CAImpl) CompleteOrder(order *core.Order) {
	// Lock the order for reading
	order.RLock()
	// If the order isn't set as beganProcessing produce an error and immediately unlock
	if !order.BeganProcessing {
		ca.log.Printf("Error: Asked to complete order %s which had false beganProcessing.",
			order.ID)
		order.RUnlock()
		return
	}
	// Unlock the order again
	order.RUnlock()

	// Check the authorizations - this is done by the VA before calling
	// CompleteOrder but we do it again for robustness sake.
	for _, authz := range order.AuthorizationObjects {
		// Lock the authorization for reading
		authz.RLock()
		if authz.Status != acme.StatusValid {
			return
		}
		authz.RUnlock()
	}

	// issue a certificate for the csr
	csr := order.ParsedCSR
	cert, err := ca.newCertificate(csr.DNSNames, csr.PublicKey, order.AccountID)
	if err != nil {
		ca.log.Printf("Error: unable to issue order: %s", err.Error())
		return
	}
	ca.log.Printf("Issued certificate serial %s for order %s\n", cert.ID, order.ID)

	// Lock and update the order to store the issued certificate
	order.Lock()
	order.CertificateObject = cert
	order.Unlock()
}
<|endoftext|>


<commit_before>package luar

import (
	"container/list"
	"reflect"

	"github.com/yuin/gopher-lua"
)

func addMethods(L *lua.LState, c *Config, vtype reflect.Type, tbl *lua.LTable, ptrReceiver bool) {
	for i := 0; i < vtype.NumMethod(); i++ {
		method := vtype.Method(i)
		if method.PkgPath != "" {
			continue
		}
		namesFn := c.MethodNames
		if namesFn == nil {
			namesFn = defaultMethodNames
		}
		fn := funcWrapper(L, method.Func, ptrReceiver)
		for _, name := range namesFn(vtype, method) {
			tbl.RawSetString(name, fn)
		}
	}
}

func addFields(L *lua.LState, c *Config, vtype reflect.Type, tbl *lua.LTable) {
	type element struct {
		Type  reflect.Type
		Index []int
	}

	queue := list.New()
	queue.PushFront(element{
		Type: vtype,
	})

	namesFn := c.FieldNames
	if namesFn == nil {
		namesFn = defaultFieldNames
	}

	for queue.Len() > 0 {
		e := queue.Back()
		elem := e.Value.(element)
		vtype := elem.Type
	fields:
		for i := 0; i < vtype.NumField(); i++ {
			field := vtype.Field(i)
			if field.PkgPath != "" && !field.Anonymous {
				continue
			}
			names := namesFn(vtype, field)
			for _, key := range names {
				if tbl.RawGetString(key) != lua.LNil {
					continue fields
				}
			}
			index := make([]int, len(elem.Index)+1)
			copy(index, elem.Index)
			index[len(elem.Index)] = i

			ud := L.NewUserData()
			ud.Value = index
			for _, key := range names {
				tbl.RawSetString(key, ud)
			}
			if field.Anonymous {
				t := field.Type
				if field.Type.Kind() != reflect.Struct {
					if field.Type.Kind() != reflect.Ptr || field.Type.Elem().Kind() != reflect.Struct {
						continue
					}
					t = field.Type.Elem()
				}
				index := make([]int, len(elem.Index)+1)
				copy(index, elem.Index)
				index[len(elem.Index)] = i
				queue.PushFront(element{
					Type:  t,
					Index: index,
				})
			}
		}

		queue.Remove(e)
	}
}

func getMetatable(L *lua.LState, vtype reflect.Type) *lua.LTable {
	config := GetConfig(L)

	if v := config.regular[vtype]; v != nil {
		return v
	}

	var (
		mt      *lua.LTable
		methods = L.CreateTable(0, vtype.NumMethod())
	)

	switch vtype.Kind() {
	case reflect.Array:
		mt = L.CreateTable(0, 7)

		mt.RawSetString("__index", L.NewFunction(arrayIndex))
		mt.RawSetString("__len", L.NewFunction(arrayLen))
		mt.RawSetString("__call", L.NewFunction(arrayCall))
		mt.RawSetString("__eq", L.NewFunction(arrayEq))

		addMethods(L, config, vtype, methods, false)
	case reflect.Chan:
		mt = L.CreateTable(0, 8)

		mt.RawSetString("__index", L.NewFunction(chanIndex))
		mt.RawSetString("__len", L.NewFunction(chanLen))
		mt.RawSetString("__eq", L.NewFunction(chanEq))
		mt.RawSetString("__call", L.NewFunction(chanCall))
		mt.RawSetString("__unm", L.NewFunction(chanUnm))

		addMethods(L, config, vtype, methods, false)
	case reflect.Map:
		mt = L.CreateTable(0, 7)

		mt.RawSetString("__index", L.NewFunction(mapIndex))
		mt.RawSetString("__newindex", L.NewFunction(mapNewIndex))
		mt.RawSetString("__len", L.NewFunction(mapLen))
		mt.RawSetString("__call", L.NewFunction(mapCall))

		addMethods(L, config, vtype, methods, false)
	case reflect.Slice:
		mt = L.CreateTable(0, 8)

		mt.RawSetString("__index", L.NewFunction(sliceIndex))
		mt.RawSetString("__newindex", L.NewFunction(sliceNewIndex))
		mt.RawSetString("__len", L.NewFunction(sliceLen))
		mt.RawSetString("__call", L.NewFunction(sliceCall))
		mt.RawSetString("__add", L.NewFunction(sliceAdd))

		addMethods(L, config, vtype, methods, false)
	case reflect.Struct:
		mt = L.CreateTable(0, 6)

		fields := L.CreateTable(0, vtype.NumField())
		addFields(L, config, vtype, fields)
		mt.RawSetString("fields", fields)

		mt.RawSetString("__index", L.NewFunction(structIndex))
		mt.RawSetString("__eq", L.NewFunction(structEq))

		addMethods(L, config, vtype, methods, false)
	case reflect.Ptr:
		switch vtype.Elem().Kind() {
		case reflect.Array:
			mt = L.CreateTable(0, 10)

			mt.RawSetString("__index", L.NewFunction(arrayPtrIndex))
			mt.RawSetString("__newindex", L.NewFunction(arrayPtrNewIndex))
			mt.RawSetString("__call", L.NewFunction(arrayCall)) // same as non-pointer
			mt.RawSetString("__len", L.NewFunction(arrayLen))   // same as non-pointer
		case reflect.Struct:
			mt = L.CreateTable(0, 8)

			mt.RawSetString("__index", L.NewFunction(structPtrIndex))
			mt.RawSetString("__newindex", L.NewFunction(structPtrNewIndex))
		default:
			mt = L.CreateTable(0, 7)

			mt.RawSetString("__index", L.NewFunction(ptrIndex))
		}

		mt.RawSetString("__eq", L.NewFunction(ptrEq))
		mt.RawSetString("__pow", L.NewFunction(ptrPow))
		mt.RawSetString("__unm", L.NewFunction(ptrUnm))

		addMethods(L, config, vtype, methods, true)
	default:
		panic("unexpected kind " + vtype.Kind().String())
	}

	mt.RawSetString("__tostring", L.NewFunction(tostring))
	mt.RawSetString("__metatable", lua.LString("gopher-luar"))
	mt.RawSetString("methods", methods)

	config.regular[vtype] = mt
	return mt
}

func getMetatableFromValue(L *lua.LState, value reflect.Value) *lua.LTable {
	vtype := value.Type()
	return getMetatable(L, vtype)
}

func getTypeMetatable(L *lua.LState, t reflect.Type) *lua.LTable {
	config := GetConfig(L)

	if v := config.types; v != nil {
		return v
	}

	mt := L.CreateTable(0, 2)
	mt.RawSetString("__call", L.NewFunction(typeCall))
	mt.RawSetString("__eq", L.NewFunction(typeEq))

	config.types = mt
	return mt
}
<commit_msg>simplify index creation in addFields<commit_after>package luar

import (
	"container/list"
	"reflect"

	"github.com/yuin/gopher-lua"
)

func addMethods(L *lua.LState, c *Config, vtype reflect.Type, tbl *lua.LTable, ptrReceiver bool) {
	for i := 0; i < vtype.NumMethod(); i++ {
		method := vtype.Method(i)
		if method.PkgPath != "" {
			continue
		}
		namesFn := c.MethodNames
		if namesFn == nil {
			namesFn = defaultMethodNames
		}
		fn := funcWrapper(L, method.Func, ptrReceiver)
		for _, name := range namesFn(vtype, method) {
			tbl.RawSetString(name, fn)
		}
	}
}

func addFields(L *lua.LState, c *Config, vtype reflect.Type, tbl *lua.LTable) {
	type element struct {
		Type  reflect.Type
		Index []int
	}

	queue := list.New()
	queue.PushFront(element{
		Type: vtype,
	})

	namesFn := c.FieldNames
	if namesFn == nil {
		namesFn = defaultFieldNames
	}

	for queue.Len() > 0 {
		e := queue.Back()
		elem := e.Value.(element)
		vtype := elem.Type
	fields:
		for i := 0; i < vtype.NumField(); i++ {
			field := vtype.Field(i)
			if field.PkgPath != "" && !field.Anonymous {
				continue
			}
			names := namesFn(vtype, field)
			for _, key := range names {
				if tbl.RawGetString(key) != lua.LNil {
					continue fields
				}
			}

			ud := L.NewUserData()
			ud.Value = append(elem.Index[:len(elem.Index):len(elem.Index)], i)
			for _, key := range names {
				tbl.RawSetString(key, ud)
			}
			if field.Anonymous {
				t := field.Type
				if field.Type.Kind() != reflect.Struct {
					if field.Type.Kind() != reflect.Ptr || field.Type.Elem().Kind() != reflect.Struct {
						continue
					}
					t = field.Type.Elem()
				}
				queue.PushFront(element{
					Type:  t,
					Index: append(elem.Index[:len(elem.Index):len(elem.Index)], i),
				})
			}
		}

		queue.Remove(e)
	}
}

func getMetatable(L *lua.LState, vtype reflect.Type) *lua.LTable {
	config := GetConfig(L)

	if v := config.regular[vtype]; v != nil {
		return v
	}

	var (
		mt      *lua.LTable
		methods = L.CreateTable(0, vtype.NumMethod())
	)

	switch vtype.Kind() {
	case reflect.Array:
		mt = L.CreateTable(0, 7)

		mt.RawSetString("__index", L.NewFunction(arrayIndex))
		mt.RawSetString("__len", L.NewFunction(arrayLen))
		mt.RawSetString("__call", L.NewFunction(arrayCall))
		mt.RawSetString("__eq", L.NewFunction(arrayEq))

		addMethods(L, config, vtype, methods, false)
	case reflect.Chan:
		mt = L.CreateTable(0, 8)

		mt.RawSetString("__index", L.NewFunction(chanIndex))
		mt.RawSetString("__len", L.NewFunction(chanLen))
		mt.RawSetString("__eq", L.NewFunction(chanEq))
		mt.RawSetString("__call", L.NewFunction(chanCall))
		mt.RawSetString("__unm", L.NewFunction(chanUnm))

		addMethods(L, config, vtype, methods, false)
	case reflect.Map:
		mt = L.CreateTable(0, 7)

		mt.RawSetString("__index", L.NewFunction(mapIndex))
		mt.RawSetString("__newindex", L.NewFunction(mapNewIndex))
		mt.RawSetString("__len", L.NewFunction(mapLen))
		mt.RawSetString("__call", L.NewFunction(mapCall))

		addMethods(L, config, vtype, methods, false)
	case reflect.Slice:
		mt = L.CreateTable(0, 8)

		mt.RawSetString("__index", L.NewFunction(sliceIndex))
		mt.RawSetString("__newindex", L.NewFunction(sliceNewIndex))
		mt.RawSetString("__len", L.NewFunction(sliceLen))
		mt.RawSetString("__call", L.NewFunction(sliceCall))
		mt.RawSetString("__add", L.NewFunction(sliceAdd))

		addMethods(L, config, vtype, methods, false)
	case reflect.Struct:
		mt = L.CreateTable(0, 6)

		fields := L.CreateTable(0, vtype.NumField())
		addFields(L, config, vtype, fields)
		mt.RawSetString("fields", fields)

		mt.RawSetString("__index", L.NewFunction(structIndex))
		mt.RawSetString("__eq", L.NewFunction(structEq))

		addMethods(L, config, vtype, methods, false)
	case reflect.Ptr:
		switch vtype.Elem().Kind() {
		case reflect.Array:
			mt = L.CreateTable(0, 10)

			mt.RawSetString("__index", L.NewFunction(arrayPtrIndex))
			mt.RawSetString("__newindex", L.NewFunction(arrayPtrNewIndex))
			mt.RawSetString("__call", L.NewFunction(arrayCall)) // same as non-pointer
			mt.RawSetString("__len", L.NewFunction(arrayLen))   // same as non-pointer
		case reflect.Struct:
			mt = L.CreateTable(0, 8)

			mt.RawSetString("__index", L.NewFunction(structPtrIndex))
			mt.RawSetString("__newindex", L.NewFunction(structPtrNewIndex))
		default:
			mt = L.CreateTable(0, 7)

			mt.RawSetString("__index", L.NewFunction(ptrIndex))
		}

		mt.RawSetString("__eq", L.NewFunction(ptrEq))
		mt.RawSetString("__pow", L.NewFunction(ptrPow))
		mt.RawSetString("__unm", L.NewFunction(ptrUnm))

		addMethods(L, config, vtype, methods, true)
	default:
		panic("unexpected kind " + vtype.Kind().String())
	}

	mt.RawSetString("__tostring", L.NewFunction(tostring))
	mt.RawSetString("__metatable", lua.LString("gopher-luar"))
	mt.RawSetString("methods", methods)

	config.regular[vtype] = mt
	return mt
}

func getMetatableFromValue(L *lua.LState, value reflect.Value) *lua.LTable {
	vtype := value.Type()
	return getMetatable(L, vtype)
}

func getTypeMetatable(L *lua.LState, t reflect.Type) *lua.LTable {
	config := GetConfig(L)

	if v := config.types; v != nil {
		return v
	}

	mt := L.CreateTable(0, 2)
	mt.RawSetString("__call", L.NewFunction(typeCall))
	mt.RawSetString("__eq", L.NewFunction(typeEq))

	config.types = mt
	return mt
}
<|endoftext|>


<commit_before>package stored

import (
	"github.com/jenchik/stored/api"
	"sync"
	"sync/atomic"
	"time"
)

type item struct {
	val interface{}
	ttl int64
}

type Cache struct {
	sm        api.StoredMap
	maxItems  int
	ttl       int64
	cntGcCall uint64 // счётчик вызовов мусорщика
	cntGcHit  uint64 // счётчик успешной чистки каждого элемента
	cntHit    uint64 // общий счётчик общих обращений
	cntFail   uint64 // общий счётчик "тухлых" элементов
	cntInsert uint64 // общий счётчик вставок
	cntMissed uint64 // общий счётчик промашек
	gcTimeout time.Duration
	gcm       sync.RWMutex
}

// each instance may be creates two streams
func NewCache(opt Options, sm api.StoredMap) *Cache {
	c := &Cache{
		sm:        sm,
		maxItems:  saneMaxItems(opt.MaxItems),
		ttl:       int64(saneTTL(opt.TTL)),
		gcTimeout: time.Duration(saneTimeoutGC(opt.TimeoutGC)),
	}
	if !opt.DisabledGC {
		go gc(c)
	}
	return c
}

func (c *Cache) GetItem(key string, fback api.GetterFunc) (interface{}, error) {
	var valItem *item
	var err error
	c.sm.AtomicWait(func(m api.Mapper) {
		atomic.AddUint64(&c.cntHit, 1)
		m.SetKey(key)
		if value := m.Value(); value != nil {
			valItem = value.(*item)
			if time.Now().Unix() <= valItem.ttl {
				return
			}
			c.cntFail++
			valItem = nil
		}
		c.cntMissed++
		value, e := fback(key)
		if e != nil {
			err = e
			return
		}
		valItem = &item{val: value, ttl: time.Now().Unix() + c.ttl}
		m.Update(valItem)
		c.cntInsert++
	})
	if err != nil {
		return nil, err
	}
	return valItem.val, nil
}

func (c *Cache) ResetItem(key string) {
	c.sm.Delete(key)
}

func (c *Cache) Stats() map[string]uint64 {
	var s map[string]uint64
	c.gcm.RLock()
	defer c.gcm.RUnlock()
	c.sm.AtomicWait(func(m api.Mapper) {
		s = map[string]uint64{
			"gcCall": c.cntGcCall,
			"gcHit":  c.cntGcHit,
			"fail":   c.cntFail,
			"hit":    c.cntHit,
			"insert": c.cntInsert,
			"missed": c.cntMissed,
			"len":    uint64(m.Len()),
		}
	})
	return s
}

func (c *Cache) timeoutGC() time.Duration {
	return c.gcTimeout
}

func (c *Cache) GC() {
	c.gcm.Lock()
	defer c.gcm.Unlock()
	c.cntGcCall++
	index := 0
	c.sm.Each(func(m api.Mapper) {
		if m.Len() <= c.maxItems {
			m.Stop()
			return
		}
		if index > c.maxItems {
			c.cntGcHit++
			m.Delete()
			return
		}
		v := m.Value().(item)
		if time.Now().Unix() > v.ttl {
			c.cntGcHit++
			m.Delete()
		} else {
			index++
		}
		return
	})
}
<commit_msg>[!] gc type assertion fix<commit_after>package stored

import (
	"github.com/jenchik/stored/api"
	"sync"
	"sync/atomic"
	"time"
)

type item struct {
	val interface{}
	ttl int64
}

type Cache struct {
	sm        api.StoredMap
	maxItems  int
	ttl       int64
	cntGcCall uint64 // счётчик вызовов мусорщика
	cntGcHit  uint64 // счётчик успешной чистки каждого элемента
	cntHit    uint64 // общий счётчик общих обращений
	cntFail   uint64 // общий счётчик "тухлых" элементов
	cntInsert uint64 // общий счётчик вставок
	cntMissed uint64 // общий счётчик промашек
	gcTimeout time.Duration
	gcm       sync.RWMutex
}

// each instance may be creates two streams
func NewCache(opt Options, sm api.StoredMap) *Cache {
	c := &Cache{
		sm:        sm,
		maxItems:  saneMaxItems(opt.MaxItems),
		ttl:       int64(saneTTL(opt.TTL)),
		gcTimeout: time.Duration(saneTimeoutGC(opt.TimeoutGC)),
	}
	if !opt.DisabledGC {
		go gc(c)
	}
	return c
}

func (c *Cache) GetItem(key string, fback api.GetterFunc) (interface{}, error) {
	var valItem *item
	var err error
	c.sm.AtomicWait(func(m api.Mapper) {
		atomic.AddUint64(&c.cntHit, 1)
		m.SetKey(key)
		if value := m.Value(); value != nil {
			valItem = value.(*item)
			if time.Now().Unix() <= valItem.ttl {
				return
			}
			c.cntFail++
			valItem = nil
		}
		c.cntMissed++
		value, e := fback(key)
		if e != nil {
			err = e
			return
		}
		valItem = &item{val: value, ttl: time.Now().Unix() + c.ttl}
		m.Update(valItem)
		c.cntInsert++
	})
	if err != nil {
		return nil, err
	}
	return valItem.val, nil
}

func (c *Cache) ResetItem(key string) {
	c.sm.Delete(key)
}

func (c *Cache) Stats() map[string]uint64 {
	var s map[string]uint64
	c.gcm.RLock()
	defer c.gcm.RUnlock()
	c.sm.AtomicWait(func(m api.Mapper) {
		s = map[string]uint64{
			"gcCall": c.cntGcCall,
			"gcHit":  c.cntGcHit,
			"fail":   c.cntFail,
			"hit":    c.cntHit,
			"insert": c.cntInsert,
			"missed": c.cntMissed,
			"len":    uint64(m.Len()),
		}
	})
	return s
}

func (c *Cache) timeoutGC() time.Duration {
	return c.gcTimeout
}

func (c *Cache) GC() {
	c.gcm.Lock()
	defer c.gcm.Unlock()
	c.cntGcCall++
	index := 0
	c.sm.Each(func(m api.Mapper) {
		if m.Len() <= c.maxItems {
			m.Stop()
			return
		}
		if index > c.maxItems {
			c.cntGcHit++
			m.Delete()
			return
		}
		v := m.Value().(*item)
		if time.Now().Unix() > v.ttl {
			c.cntGcHit++
			m.Delete()
		} else {
			index++
		}
		return
	})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math"

	"github.com/fogleman/gg"
)

func main() {
	im, err := gg.LoadPNG("examples/lenna.png")
	if err != nil {
		panic(err)
	}
	var x0, y0, x1, y1 float64 = 5, 5, 15, 15
	a := math.Atan2(y1-y0, x1-x0)
	d := a * 180 / math.Pi
	fmt.Println(a, d)
	a = (2*math.Pi - a)
	d = a * 180 / math.Pi
	fmt.Println(a, d)
	m := gg.Translate(-x0, -y0).Rotate(a)
	tx, ty := m.TransformPoint(5, 5)
	fmt.Println(tx, ty)
	pattern := gg.NewSurfacePattern(im, gg.RepeatBoth)
	dc := gg.NewContext(600, 600)
	dc.MoveTo(20, 20)
	dc.LineTo(590, 20)
	dc.LineTo(590, 590)
	dc.LineTo(20, 590)
	dc.ClosePath()
	dc.SetFillStyle(pattern)
	dc.Fill()
	dc.SavePNG("out.png")
}
<commit_msg>cleanup again<commit_after>package main

import "github.com/fogleman/gg"

func main() {
	im, err := gg.LoadPNG("examples/lenna.png")
	if err != nil {
		panic(err)
	}
	pattern := gg.NewSurfacePattern(im, gg.RepeatBoth)
	dc := gg.NewContext(600, 600)
	dc.MoveTo(20, 20)
	dc.LineTo(590, 20)
	dc.LineTo(590, 590)
	dc.LineTo(20, 590)
	dc.ClosePath()
	dc.SetFillStyle(pattern)
	dc.Fill()
	dc.SavePNG("out.png")
}
<|endoftext|>