<commit_before>package pdb

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/TuftsBCB/seq"
	"github.com/TuftsBCB/structure"
)

type pdbParser struct {
	entry    *Entry
	curModel int
	line     []byte
	modified map[string]string
	seqres   map[byte][]string

	missing map[byte][]missingResidue
	r465    bool

	processed map[seen]bool
	lastSeen  seen
}

type missingResidue struct {
	residue string
	seqNum  int
	insCode byte
}

type seen struct {
	chain byte
	model int
}

func ReadPDB(fp string) (*Entry, error) {
	var reader io.Reader
	var err error

	f, err := os.Open(fp)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	reader = f

	// If the file is gzipped, use the gzip decompressor.
	if path.Ext(fp) == ".gz" {
		reader, err = gzip.NewReader(reader)
		if err != nil {
			return nil, err
		}
	}

	return Read(reader, fp)
}

func Read(reader io.Reader, fpath string) (*Entry, error) {
	entry := &Entry{
		Path:   fpath,
		Chains: make([]*Chain, 0),
	}

	// Now traverse each line, and process it according to the record name.
	// Note that it is imperative that we preserve the order of ATOM records
	// as we read them. We are currently trying to replicate Fragbag, and this
	// is what Fragbag does. (A more stable approach would require more
	// information from the PDB file; like differentiating models, since
	// sorting on ATOM serial number isn't good enough.)
	breader := bufio.NewReaderSize(reader, 1000)
	parser := pdbParser{
		entry:     entry,
		curModel:  1,
		line:      nil,
		modified:  make(map[string]string),
		seqres:    make(map[byte][]string),
		missing:   make(map[byte][]missingResidue),
		r465:      false,
		processed: make(map[seen]bool),
		lastSeen:  seen{0, 0},
	}
	for {
		// We ignore 'isPrefix' here, since we never care about lines longer
		// than 1000 characters, which is the size of our buffer.
		line, _, err := breader.ReadLine()
		if err == io.EOF && len(line) == 0 {
			break
		} else if err != io.EOF && err != nil {
			return nil, err
		}
		parser.line = line
		if err := parser.parseLine(); err != nil {
			return nil, err
		}
	}

	// We've got to back and translate SEQRES residues to their single-letter
	// abbreviations.
	// We didn't do this before, because the MODRES records typically come
	// afterwards. Ug.
	for _, chain := range parser.entry.Chains {
		for _, r := range parser.seqres[chain.Ident] {
			newr, err := parser.residueAbbrev(r)
			if err != nil {
				return nil, fmt.Errorf("PDB (SEQRES) '%s': %s",
					parser.entry.Path, err)
			}
			chain.Sequence = append(chain.Sequence, newr)
		}
	}

	// We also need to translate the missing residues.
	for _, chain := range parser.entry.Chains {
		for _, r := range parser.missing[chain.Ident] {
			abbrev, err := parser.residueAbbrev(r.residue)
			if err != nil {
				return nil, fmt.Errorf("PDB (REMARK 465) '%s': %s",
					parser.entry.Path, err)
			}
			newr := &Residue{
				Name:          abbrev,
				SequenceNum:   r.seqNum,
				InsertionCode: r.insCode,
				Atoms:         nil,
			}
			chain.Missing = append(chain.Missing, newr)
		}
	}

	// If we didn't pick up any chains, this probably isn't a valid PDB file.
	if len(entry.Chains) == 0 {
		return nil, fmt.Errorf("The file '%s' does not appear to be a valid "+
			"PDB file.", entry.Path)
	}

	// If we couldn't find an Id code, inspect the base name of the file path.
	if len(entry.IdCode) == 0 {
		name := path.Base(entry.Path)
		switch {
		case len(name) >= 7 && name[0:3] == "pdb":
			entry.IdCode = name[3:7]
		case len(name) >= 6: // SCOP
			entry.Scop = stripExt(name)
			entry.IdCode = name[1:5]
		case len(name) == 7: // cath
			entry.Cath = stripExt(name)
			entry.IdCode = name[0:4]
		}
	}

	return entry, nil
}

func stripExt(s string) string {
	i := strings.Index(s, ".")
	if i == -1 {
		return s
	}
	return s[0:i]
}

func (p *pdbParser) parseLine() error {
	var err error

	switch p.cols(1, 6) {
	case "HEADER":
		p.entry.IdCode = p.cols(63, 66)
	case "MODEL":
		p.curModel, err = p.atoi(11, 14)
		if err != nil {
			return err
		}
	case "SEQRES":
		p.parseSeqres()
	case "MODRES":
		residue := p.cols(25, 27)
		if len(residue) == 0 {
			residue = "UNK"
		}
		p.modified[p.cols(13, 15)] = residue
	case "HET":
		residue := p.cols(8, 10)

		// Only add this if there isn't a MODRES for this residue.
		if _, ok := p.modified[residue]; !ok {
			p.modified[residue] = "UNK"
		}
	case "ATOM":
		fallthrough
	case "HETATM":
		if err := p.parseAtom(); err != nil {
			return err
		}
	case "TER":
		// Whatever ATOM record we saw last will have its {chain, model}
		// added to the 'processed' map. This effectively disallows any further
		// additions to this particular {chain, model}.
		// This cuts off HETATM's after "TER", but I'm fine with that for now.
		// fmt.Printf("%c %d\n", p.lastSeen.chain, p.lastSeen.model)
		p.processed[p.lastSeen] = true
	case "REMARK":
		num, err := p.atoi(8, 10)
		if err != nil {
			// Some people like to put in their own remarks and not conform
			// to the SPEC. Why............
			return nil
		}
		switch num {
		case 465:
			if err := p.parseRemark465(); err != nil {
				return err
			}
		}
	}
	return nil
}

func (p pdbParser) parseSeqres() {
	chain := p.getChain(p.at(12))
	if _, ok := p.seqres[chain.Ident]; !ok {
		p.seqres[chain.Ident] = make([]string, 0, 25)
	}
	for c := 20; c <= 68; c += 4 {
		res := p.cols(c, c+2)
		if len(res) == 0 {
			break
		}
		if chain.SeqType == -1 {
			// Label the sequence type if we haven't yet.
			// This basically just looks at the length of the unabbreviated
			// residue. len of 3 = protein, len of 2 = deoxy, len of 1 = ribo.
			chain.SeqType = getAbbrevType(res)
		}
		p.seqres[chain.Ident] = append(p.seqres[chain.Ident], res)
	}
}

func (p *pdbParser) parseAtom() error {
	ident := p.at(22)

	// If we've already processed this {chain, model}, then don't look
	// for more ATOMs.
	if p.processed[seen{ident, p.curModel}] {
		return nil
	}

	res := p.cols(18, 20)
	if res == "HOH" { // ignore water HETATMs
		return nil
	}

	insCode := p.at(27)
	if insCode == ' ' {
		insCode = 0
	}

	seqNum, err := p.atoi(23, 26)
	if err != nil {
		return err
	}

	// If this is a HETATM, we often see weird residues, so we'll suppress an
	// error if we don't recognize the residue. The residue will be set to 'X'.
	flexible := p.cols(1, 6) == "HETATM"
	residue, err := p.getResidue(ident, res, seqNum, insCode, flexible)
	if err != nil {
		return err
	}
	atom := Atom{
		Name:   p.cols(13, 16),
		Het:    p.cols(1, 6) == "HETATM",
		Coords: structure.Coords{},
	}

	atom.X, err = p.atof(31, 38)
	if err != nil {
		return err
	}
	atom.Y, err = p.atof(39, 46)
	if err != nil {
		return err
	}
	atom.Z, err = p.atof(47, 54)
	if err != nil {
		return err
	}

	p.lastSeen = seen{ident, p.curModel}
	residue.Atoms = append(residue.Atoms, atom)
	return nil
}

func (p *pdbParser) parseRemark465() error {
	residue := p.cols(16, 18)

	if residue == "RES" && p.at(20) == 'C' && p.cols(22, 27) == "SSSEQI" {
		p.r465 = true
		return nil
	}
	if !p.r465 {
		return nil
	}
	if p.r465 && (len(residue) == 0 || len(residue) > 3) {
		p.r465 = false
		return nil
	}

	chainIdent := p.at(20)

	seqNum, err := p.atoi(22, 26)
	if err != nil {
		return err
	}

	insCode := p.at(27)
	if insCode == ' ' {
		insCode = 0
	}

	if _, ok := p.missing[chainIdent]; !ok {
		p.missing[chainIdent] = make([]missingResidue, 0)
	}
	missing := missingResidue{residue, seqNum, insCode}
	p.missing[chainIdent] = append(p.missing[chainIdent], missing)
	return nil
}

func (p *pdbParser) getChain(ident byte) *Chain {
	for i, chain := range p.entry.Chains {
		if chain.Ident == ident {
			return p.entry.Chains[i]
		}
	}
	chain := &Chain{
		Entry:    p.entry,
		Ident:    ident,
		SeqType:  -1,
		Sequence: make([]seq.Residue, 0, 25),
		Models:   make([]*Model, 0),
		Missing:  make([]*Residue, 0),
	}
	p.entry.Chains = append(p.entry.Chains, chain)
	return chain
}

func (p pdbParser) getModel(ident byte) *Model {
	chain := p.getChain(ident)
	for i, model := range chain.Models {
		if model.Num == p.curModel {
			return chain.Models[i]
		}
	}

	model := &Model{
		Entry:    p.entry,
		Chain:    chain,
		Num:      p.curModel,
		Residues: make([]*Residue, 0, 25),
	}
	chain.Models = append(chain.Models, model)
	return model
}

func (p pdbParser) getResidue(ident byte,
	res string, seqNum int, insCode byte, flexible bool) (*Residue, error) {

	model := p.getModel(ident)
	for i := range model.Residues {
		if model.Residues[i].SequenceNum == seqNum &&
			model.Residues[i].InsertionCode == insCode {

			return model.Residues[i], nil
		}
	}
	abbrev, err := p.residueAbbrev(res)
	if !flexible && err != nil {
		return nil, fmt.Errorf("PDB (ATOM) '%s': %s", p.entry.Path, err)
	}

	residue := &Residue{
		Name:          abbrev,
		SequenceNum:   seqNum,
		InsertionCode: insCode,
		Atoms:         make([]Atom, 0, 2),
	}
	model.Residues = append(model.Residues, residue)
	return residue, nil
}

func (p pdbParser) atoi(start, end int) (int, error) {
	return strconv.Atoi(p.cols(start, end))
}

func (p pdbParser) atof(start, end int) (float64, error) {
	return strconv.ParseFloat(p.cols(start, end), 64)
}

func (p pdbParser) cols(start, end int) string {
	rs, re := start-1, end
	if rs >= len(p.line) || rs < 0 {
		return ""
	}
	if re > len(p.line) || re < 0 || re < rs {
		return ""
	}
	return string(bytes.TrimSpace(p.line[rs:re]))
}

func (p pdbParser) at(column int) byte {
	i := column - 1
	if i < 0 || i >= len(p.line) {
		return 0
	}
	return p.line[i]
}

func (p pdbParser) residueAbbrev(long string) (seq.Residue, error) {
	if res, ok := p.modified[long]; ok {
		return getAbbrev(res)
	}
	abbrev, err := getAbbrev(long)

	// If we couldn't find a residue, check the missing residues. If this
	// residue shows up there, let's just ignore it and return 'X'.
	// If it isn't in the missing residues, then quit with an error.
	if err != nil {
		for _, missing := range p.missing {
			for _, residue := range missing {
				if residue.residue == long {
					return 'X', nil
				}
			}
		}
		return 'X', err
	}
	return abbrev, nil
}
<commit_msg>Dammit SCOP. WTF are you doing?<commit_after>package pdb

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/TuftsBCB/seq"
	"github.com/TuftsBCB/structure"
)

type pdbParser struct {
	entry    *Entry
	curModel int
	line     []byte
	modified map[string]string
	seqres   map[byte][]string

	missing map[byte][]missingResidue
	r465    bool

	processed map[seen]bool
	lastSeen  seen
}

type missingResidue struct {
	residue string
	seqNum  int
	insCode byte
}

type seen struct {
	chain byte
	model int
}

func ReadPDB(fp string) (*Entry, error) {
	var reader io.Reader
	var err error

	f, err := os.Open(fp)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	reader = f

	// If the file is gzipped, use the gzip decompressor.
	if path.Ext(fp) == ".gz" {
		reader, err = gzip.NewReader(reader)
		if err != nil {
			return nil, err
		}
	}

	return Read(reader, fp)
}

func Read(reader io.Reader, fpath string) (*Entry, error) {
	entry := &Entry{
		Path:   fpath,
		Chains: make([]*Chain, 0),
	}

	// Now traverse each line, and process it according to the record name.
	// Note that it is imperative that we preserve the order of ATOM records
	// as we read them. We are currently trying to replicate Fragbag, and this
	// is what Fragbag does. (A more stable approach would require more
	// information from the PDB file; like differentiating models, since
	// sorting on ATOM serial number isn't good enough.)
	breader := bufio.NewReaderSize(reader, 1000)
	parser := pdbParser{
		entry:     entry,
		curModel:  1,
		line:      nil,
		modified:  make(map[string]string),
		seqres:    make(map[byte][]string),
		missing:   make(map[byte][]missingResidue),
		r465:      false,
		processed: make(map[seen]bool),
		lastSeen:  seen{0, 0},
	}
	for {
		// We ignore 'isPrefix' here, since we never care about lines longer
		// than 1000 characters, which is the size of our buffer.
		line, _, err := breader.ReadLine()
		if err == io.EOF && len(line) == 0 {
			break
		} else if err != io.EOF && err != nil {
			return nil, err
		}
		parser.line = line
		if err := parser.parseLine(); err != nil {
			return nil, err
		}
	}

	// We've got to back and translate SEQRES residues to their single-letter
	// abbreviations.
	// We didn't do this before, because the MODRES records typically come
	// afterwards. Ug.
	for _, chain := range parser.entry.Chains {
		for _, r := range parser.seqres[chain.Ident] {
			newr, err := parser.residueAbbrev(r)
			if err != nil {
				return nil, fmt.Errorf("PDB (SEQRES) '%s': %s",
					parser.entry.Path, err)
			}
			chain.Sequence = append(chain.Sequence, newr)
		}
	}

	// We also need to translate the missing residues.
	for _, chain := range parser.entry.Chains {
		for _, r := range parser.missing[chain.Ident] {
			abbrev, err := parser.residueAbbrev(r.residue)
			if err != nil {
				return nil, fmt.Errorf("PDB (REMARK 465) '%s': %s",
					parser.entry.Path, err)
			}
			newr := &Residue{
				Name:          abbrev,
				SequenceNum:   r.seqNum,
				InsertionCode: r.insCode,
				Atoms:         nil,
			}
			chain.Missing = append(chain.Missing, newr)
		}
	}

	// If we didn't pick up any chains, this probably isn't a valid PDB file.
	if len(entry.Chains) == 0 {
		return nil, fmt.Errorf("The file '%s' does not appear to be a valid "+
			"PDB file.", entry.Path)
	}

	// If we couldn't find an Id code, inspect the base name of the file path.
	if len(entry.IdCode) == 0 || entry.IdCode == "0000" {
		name := path.Base(entry.Path)
		switch {
		case len(name) >= 7 && name[0:3] == "pdb":
			entry.IdCode = name[3:7]
		case len(name) >= 6: // SCOP
			entry.Scop = stripExt(name)
			entry.IdCode = name[1:5]
		case len(name) == 7: // cath
			entry.Cath = stripExt(name)
			entry.IdCode = name[0:4]
		}
	}

	return entry, nil
}

func stripExt(s string) string {
	i := strings.Index(s, ".")
	if i == -1 {
		return s
	}
	return s[0:i]
}

func (p *pdbParser) parseLine() error {
	var err error

	switch p.cols(1, 6) {
	case "HEADER":
		p.entry.IdCode = p.cols(63, 66)
	case "MODEL":
		p.curModel, err = p.atoi(11, 14)
		if err != nil {
			return err
		}
	case "SEQRES":
		p.parseSeqres()
	case "MODRES":
		residue := p.cols(25, 27)
		if len(residue) == 0 {
			residue = "UNK"
		}
		p.modified[p.cols(13, 15)] = residue
	case "HET":
		residue := p.cols(8, 10)

		// Only add this if there isn't a MODRES for this residue.
		if _, ok := p.modified[residue]; !ok {
			p.modified[residue] = "UNK"
		}
	case "ATOM":
		fallthrough
	case "HETATM":
		if err := p.parseAtom(); err != nil {
			return err
		}
	case "TER":
		// Whatever ATOM record we saw last will have its {chain, model}
		// added to the 'processed' map. This effectively disallows any further
		// additions to this particular {chain, model}.
		// This cuts off HETATM's after "TER", but I'm fine with that for now.
		// fmt.Printf("%c %d\n", p.lastSeen.chain, p.lastSeen.model)
		p.processed[p.lastSeen] = true
	case "REMARK":
		num, err := p.atoi(8, 10)
		if err != nil {
			// Some people like to put in their own remarks and not conform
			// to the SPEC. Why............
			return nil
		}
		switch num {
		case 465:
			if err := p.parseRemark465(); err != nil {
				return err
			}
		}
	}
	return nil
}

func (p pdbParser) parseSeqres() {
	chain := p.getChain(p.at(12))
	if _, ok := p.seqres[chain.Ident]; !ok {
		p.seqres[chain.Ident] = make([]string, 0, 25)
	}
	for c := 20; c <= 68; c += 4 {
		res := p.cols(c, c+2)
		if len(res) == 0 {
			break
		}
		if chain.SeqType == -1 {
			// Label the sequence type if we haven't yet.
			// This basically just looks at the length of the unabbreviated
			// residue. len of 3 = protein, len of 2 = deoxy, len of 1 = ribo.
			chain.SeqType = getAbbrevType(res)
		}
		p.seqres[chain.Ident] = append(p.seqres[chain.Ident], res)
	}
}

func (p *pdbParser) parseAtom() error {
	ident := p.at(22)

	// If we've already processed this {chain, model}, then don't look
	// for more ATOMs.
	if p.processed[seen{ident, p.curModel}] {
		return nil
	}

	res := p.cols(18, 20)
	if res == "HOH" { // ignore water HETATMs
		return nil
	}

	insCode := p.at(27)
	if insCode == ' ' {
		insCode = 0
	}

	seqNum, err := p.atoi(23, 26)
	if err != nil {
		return err
	}

	// If this is a HETATM, we often see weird residues, so we'll suppress an
	// error if we don't recognize the residue. The residue will be set to 'X'.
	flexible := p.cols(1, 6) == "HETATM"
	residue, err := p.getResidue(ident, res, seqNum, insCode, flexible)
	if err != nil {
		return err
	}
	atom := Atom{
		Name:   p.cols(13, 16),
		Het:    p.cols(1, 6) == "HETATM",
		Coords: structure.Coords{},
	}

	atom.X, err = p.atof(31, 38)
	if err != nil {
		return err
	}
	atom.Y, err = p.atof(39, 46)
	if err != nil {
		return err
	}
	atom.Z, err = p.atof(47, 54)
	if err != nil {
		return err
	}

	p.lastSeen = seen{ident, p.curModel}
	residue.Atoms = append(residue.Atoms, atom)
	return nil
}

func (p *pdbParser) parseRemark465() error {
	residue := p.cols(16, 18)

	if residue == "RES" && p.at(20) == 'C' && p.cols(22, 27) == "SSSEQI" {
		p.r465 = true
		return nil
	}
	if !p.r465 {
		return nil
	}
	if p.r465 && (len(residue) == 0 || len(residue) > 3) {
		p.r465 = false
		return nil
	}

	chainIdent := p.at(20)

	seqNum, err := p.atoi(22, 26)
	if err != nil {
		return err
	}

	insCode := p.at(27)
	if insCode == ' ' {
		insCode = 0
	}

	if _, ok := p.missing[chainIdent]; !ok {
		p.missing[chainIdent] = make([]missingResidue, 0)
	}
	missing := missingResidue{residue, seqNum, insCode}
	p.missing[chainIdent] = append(p.missing[chainIdent], missing)
	return nil
}

func (p *pdbParser) getChain(ident byte) *Chain {
	for i, chain := range p.entry.Chains {
		if chain.Ident == ident {
			return p.entry.Chains[i]
		}
	}
	chain := &Chain{
		Entry:    p.entry,
		Ident:    ident,
		SeqType:  -1,
		Sequence: make([]seq.Residue, 0, 25),
		Models:   make([]*Model, 0),
		Missing:  make([]*Residue, 0),
	}
	p.entry.Chains = append(p.entry.Chains, chain)
	return chain
}

func (p pdbParser) getModel(ident byte) *Model {
	chain := p.getChain(ident)
	for i, model := range chain.Models {
		if model.Num == p.curModel {
			return chain.Models[i]
		}
	}

	model := &Model{
		Entry:    p.entry,
		Chain:    chain,
		Num:      p.curModel,
		Residues: make([]*Residue, 0, 25),
	}
	chain.Models = append(chain.Models, model)
	return model
}

func (p pdbParser) getResidue(ident byte,
	res string, seqNum int, insCode byte, flexible bool) (*Residue, error) {

	model := p.getModel(ident)
	for i := range model.Residues {
		if model.Residues[i].SequenceNum == seqNum &&
			model.Residues[i].InsertionCode == insCode {

			return model.Residues[i], nil
		}
	}
	abbrev, err := p.residueAbbrev(res)
	if !flexible && err != nil {
		return nil, fmt.Errorf("PDB (ATOM) '%s': %s", p.entry.Path, err)
	}

	residue := &Residue{
		Name:          abbrev,
		SequenceNum:   seqNum,
		InsertionCode: insCode,
		Atoms:         make([]Atom, 0, 2),
	}
	model.Residues = append(model.Residues, residue)
	return residue, nil
}

func (p pdbParser) atoi(start, end int) (int, error) {
	return strconv.Atoi(p.cols(start, end))
}

func (p pdbParser) atof(start, end int) (float64, error) {
	return strconv.ParseFloat(p.cols(start, end), 64)
}

func (p pdbParser) cols(start, end int) string {
	rs, re := start-1, end
	if rs >= len(p.line) || rs < 0 {
		return ""
	}
	if re > len(p.line) || re < 0 || re < rs {
		return ""
	}
	return string(bytes.TrimSpace(p.line[rs:re]))
}

func (p pdbParser) at(column int) byte {
	i := column - 1
	if i < 0 || i >= len(p.line) {
		return 0
	}
	return p.line[i]
}

func (p pdbParser) residueAbbrev(long string) (seq.Residue, error) {
	if res, ok := p.modified[long]; ok {
		return getAbbrev(res)
	}
	abbrev, err := getAbbrev(long)

	// If we couldn't find a residue, check the missing residues. If this
	// residue shows up there, let's just ignore it and return 'X'.
	// If it isn't in the missing residues, then quit with an error.
	if err != nil {
		for _, missing := range p.missing {
			for _, residue := range missing {
				if residue.residue == long {
					return 'X', nil
				}
			}
		}
		return 'X', err
	}
	return abbrev, nil
}
<|endoftext|>


<commit_before>// Package permute provides a function that generates permutations of any collection that satisfies sort.Interface.
package permute

import "sort"

// NextPermutation generates the next permutation of the
// sortable collection x in lexical order.  It returns false
// if the permutations are exhausted.
//
// Knuth, Donald (2011), "Section 7.2.1.2: Generating All Permutations",
// The Art of Computer Programming, volume 4A.
//
// The test is on http://play.golang.org/p/ljft9xhOEn
func Next(x sort.Interface) bool {
	n := x.Len() - 1
	if n < 1 {
		return false
	}
	j := n - 1
	for ; !x.Less(j, j+1); j-- {
		if j == 0 {
			return false
		}
	}
	l := n
	for !x.Less(j, l) {
		l--
	}
	x.Swap(j, l)
	for k, l := j+1, n; k < l; {
		x.Swap(k, l)
		k++
		l--
	}
	return true
}
<commit_msg>Fixed comment.<commit_after>// Package permute provides a function that generates permutations of any collection that satisfies sort.Interface.
package permute

import "sort"

// Next generates the next permutation of the
// sortable collection x in lexical order.  It returns false
// if the permutations are exhausted.
//
// Knuth, Donald (2011), "Section 7.2.1.2: Generating All Permutations",
// The Art of Computer Programming, volume 4A.
//
// The test is on http://play.golang.org/p/ljft9xhOEn
func Next(x sort.Interface) bool {
	n := x.Len() - 1
	if n < 1 {
		return false
	}
	j := n - 1
	for ; !x.Less(j, j+1); j-- {
		if j == 0 {
			return false
		}
	}
	l := n
	for !x.Less(j, l) {
		l--
	}
	x.Swap(j, l)
	for k, l := j+1, n; k < l; {
		x.Swap(k, l)
		k++
		l--
	}
	return true
}
<|endoftext|>


<commit_before>package storage

import (
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"reflect"

	"github.com/jonog/redalert/backoffs"
	"github.com/jonog/redalert/checks"
	"github.com/jonog/redalert/notifiers"

	_ "github.com/lib/pq"
	"gopkg.in/gorp.v1"
)

type ConfigDB struct {
	DB *gorp.DbMap
}

type CheckRecord struct {
	Id         int64
	Name       string           `db:"name"`
	Type       string           `db:"type"`
	SendAlerts []string         `db:"send_alerts"`
	Backoff    backoffs.Config  `db:"backoff"`
	Config     json.RawMessage  `db:"config"`
	Triggers   []checks.Trigger `db:"triggers"`
}

func (c *ConfigDB) CreateCheckRecord(check checks.Config) (*CheckRecord, error) {
	record := &CheckRecord{
		Name:       check.Name,
		Type:       check.Type,
		SendAlerts: check.SendAlerts,
		Backoff:    check.Backoff,
		Config:     check.Config,
		Triggers:   check.Triggers,
	}
	err := c.DB.Insert(record)
	if err != nil {
		return nil, err
	}
	return record, nil
}

func (c *ConfigDB) getAllCheckRecords() (checks []CheckRecord, err error) {
	_, err = c.DB.Select(&checks, "select id, name, type, send_alerts, backoff, config, triggers from checks")
	return checks, err
}

type NotificationRecord struct {
	Id     int64             `db:"id"`
	Name   string            `db:"name"`
	Type   string            `db:"type"`
	Config map[string]string `db:"config"`
}

func (c *ConfigDB) CreateNotificationRecord(notifier notifiers.Config) (*NotificationRecord, error) {
	record := &NotificationRecord{
		Name:   notifier.Name,
		Type:   notifier.Type,
		Config: notifier.Config,
	}
	err := c.DB.Insert(record)
	if err != nil {
		return nil, err
	}
	return record, nil
}

func (c *ConfigDB) getAllNotificationRecords() (notifications []NotificationRecord, err error) {
	_, err = c.DB.Select(&notifications, "select id, name, type, config from notifications")
	return notifications, err
}

func NewConfigDB(connectionURL string) (*ConfigDB, error) {
	db, err := sql.Open("postgres", connectionURL)
	if err != nil {
		return nil, err
	}
	gorpDB := &gorp.DbMap{Db: db, Dialect: gorp.PostgresDialect{}}
	gorpDB.AddTableWithName(CheckRecord{}, "checks").SetKeys(true, "Id")
	gorpDB.AddTableWithName(NotificationRecord{}, "notifications").SetKeys(true, "Id")
	gorpDB.TypeConverter = TypeConverter{}
	return &ConfigDB{DB: gorpDB}, nil
}

func (c *ConfigDB) Notifications() ([]notifiers.Config, error) {
	notificationConfigs := make([]notifiers.Config, 0)
	records, err := c.getAllNotificationRecords()
	if err != nil {
		return notificationConfigs, err
	}
	for _, record := range records {
		notificationConfigs = append(notificationConfigs, notifiers.Config{
			Name:   record.Name,
			Type:   record.Type,
			Config: record.Config,
		})
	}
	return notificationConfigs, nil
}

func (c *ConfigDB) Checks() ([]checks.Config, error) {
	checkConfigs := make([]checks.Config, 0)
	records, err := c.getAllCheckRecords()
	if err != nil {
		return checkConfigs, err
	}
	for _, record := range records {
		checkConfigs = append(checkConfigs, checks.Config{
			Name:       record.Name,
			Type:       record.Type,
			SendAlerts: record.SendAlerts,
			Backoff:    record.Backoff,
			Config:     record.Config,
			Triggers:   record.Triggers,
		})
	}
	return checkConfigs, nil
}

type TypeConverter struct{}

func (t TypeConverter) ToDb(val interface{}) (interface{}, error) {
	switch t := val.(type) {
	case map[string]string, []string, backoffs.Config, []checks.Trigger:
		b, err := json.Marshal(t)
		if err != nil {
			return "", err
		}
		return string(b), nil
	case json.RawMessage:
		return string(t), nil
	}
	return val, nil
}

func (t TypeConverter) FromDb(target interface{}) (gorp.CustomScanner, bool) {
	switch target.(type) {
	case *map[string]string, *[]string, *backoffs.Config, *[]checks.Trigger:
		binder := func(holder, target interface{}) error {
			s, ok := holder.(*string)
			if !ok {
				return errors.New("Unable to convert to *string")
			}
			b := []byte(*s)
			return json.Unmarshal(b, target)
		}
		return gorp.CustomScanner{new(string), target, binder}, true
	case *json.RawMessage:
		binder := func(holder, target interface{}) error {
			s, ok := holder.(*string)
			if !ok {
				return errors.New("Unable to convert to *string")
			}
			b := []byte(*s)
			st, ok := target.(*json.RawMessage)
			if !ok {
				return errors.New(fmt.Sprint("FromDb: Unable to convert target to *CustomStringType: ", reflect.TypeOf(target)))
			}
			*st = json.RawMessage(b)
			return nil
		}
		return gorp.CustomScanner{new(string), target, binder}, true
	}
	return gorp.CustomScanner{}, false
}
<commit_msg>Refine err msg<commit_after>package storage

import (
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"reflect"

	"github.com/jonog/redalert/backoffs"
	"github.com/jonog/redalert/checks"
	"github.com/jonog/redalert/notifiers"

	_ "github.com/lib/pq"
	"gopkg.in/gorp.v1"
)

type ConfigDB struct {
	DB *gorp.DbMap
}

type CheckRecord struct {
	Id         int64
	Name       string           `db:"name"`
	Type       string           `db:"type"`
	SendAlerts []string         `db:"send_alerts"`
	Backoff    backoffs.Config  `db:"backoff"`
	Config     json.RawMessage  `db:"config"`
	Triggers   []checks.Trigger `db:"triggers"`
}

func (c *ConfigDB) CreateCheckRecord(check checks.Config) (*CheckRecord, error) {
	record := &CheckRecord{
		Name:       check.Name,
		Type:       check.Type,
		SendAlerts: check.SendAlerts,
		Backoff:    check.Backoff,
		Config:     check.Config,
		Triggers:   check.Triggers,
	}
	err := c.DB.Insert(record)
	if err != nil {
		return nil, err
	}
	return record, nil
}

func (c *ConfigDB) getAllCheckRecords() (checks []CheckRecord, err error) {
	_, err = c.DB.Select(&checks, "select id, name, type, send_alerts, backoff, config, triggers from checks")
	return checks, err
}

type NotificationRecord struct {
	Id     int64             `db:"id"`
	Name   string            `db:"name"`
	Type   string            `db:"type"`
	Config map[string]string `db:"config"`
}

func (c *ConfigDB) CreateNotificationRecord(notifier notifiers.Config) (*NotificationRecord, error) {
	record := &NotificationRecord{
		Name:   notifier.Name,
		Type:   notifier.Type,
		Config: notifier.Config,
	}
	err := c.DB.Insert(record)
	if err != nil {
		return nil, err
	}
	return record, nil
}

func (c *ConfigDB) getAllNotificationRecords() (notifications []NotificationRecord, err error) {
	_, err = c.DB.Select(&notifications, "select id, name, type, config from notifications")
	return notifications, err
}

func NewConfigDB(connectionURL string) (*ConfigDB, error) {
	db, err := sql.Open("postgres", connectionURL)
	if err != nil {
		return nil, err
	}
	gorpDB := &gorp.DbMap{Db: db, Dialect: gorp.PostgresDialect{}}
	gorpDB.AddTableWithName(CheckRecord{}, "checks").SetKeys(true, "Id")
	gorpDB.AddTableWithName(NotificationRecord{}, "notifications").SetKeys(true, "Id")
	gorpDB.TypeConverter = TypeConverter{}
	return &ConfigDB{DB: gorpDB}, nil
}

func (c *ConfigDB) Notifications() ([]notifiers.Config, error) {
	notificationConfigs := make([]notifiers.Config, 0)
	records, err := c.getAllNotificationRecords()
	if err != nil {
		return notificationConfigs, err
	}
	for _, record := range records {
		notificationConfigs = append(notificationConfigs, notifiers.Config{
			Name:   record.Name,
			Type:   record.Type,
			Config: record.Config,
		})
	}
	return notificationConfigs, nil
}

func (c *ConfigDB) Checks() ([]checks.Config, error) {
	checkConfigs := make([]checks.Config, 0)
	records, err := c.getAllCheckRecords()
	if err != nil {
		return checkConfigs, err
	}
	for _, record := range records {
		checkConfigs = append(checkConfigs, checks.Config{
			Name:       record.Name,
			Type:       record.Type,
			SendAlerts: record.SendAlerts,
			Backoff:    record.Backoff,
			Config:     record.Config,
			Triggers:   record.Triggers,
		})
	}
	return checkConfigs, nil
}

type TypeConverter struct{}

func (t TypeConverter) ToDb(val interface{}) (interface{}, error) {
	switch t := val.(type) {
	case map[string]string, []string, backoffs.Config, []checks.Trigger:
		b, err := json.Marshal(t)
		if err != nil {
			return "", err
		}
		return string(b), nil
	case json.RawMessage:
		return string(t), nil
	}
	return val, nil
}

func (t TypeConverter) FromDb(target interface{}) (gorp.CustomScanner, bool) {
	switch target.(type) {
	case *map[string]string, *[]string, *backoffs.Config, *[]checks.Trigger:
		binder := func(holder, target interface{}) error {
			s, ok := holder.(*string)
			if !ok {
				return errors.New("Unable to convert to *string")
			}
			b := []byte(*s)
			return json.Unmarshal(b, target)
		}
		return gorp.CustomScanner{new(string), target, binder}, true
	case *json.RawMessage:
		binder := func(holder, target interface{}) error {
			s, ok := holder.(*string)
			if !ok {
				return errors.New("Unable to convert to *string")
			}
			b := []byte(*s)
			st, ok := target.(*json.RawMessage)
			if !ok {
				return errors.New(fmt.Sprint("FromDb: Unable to convert target to *json.RawMessage: ", reflect.TypeOf(target)))
			}
			*st = json.RawMessage(b)
			return nil
		}
		return gorp.CustomScanner{new(string), target, binder}, true
	}
	return gorp.CustomScanner{}, false
}
<|endoftext|>


<commit_before>package storage

import (
	"calf/config"
	cdb "calf/db"
	"calf/image"
	"database/sql"
	_ "database/sql/driver"
	_ "github.com/lib/pq"
	"log"
	// "strings"
)

type MetaWrapper interface {
	Browse(limit int, offset int) (*sql.Rows, error)
	Store(entry *Entry) error
	Get(id EntryId) (*Entry, error)
	Delete(id EntryId) error
}

type MetaWrap struct {
	dsn string
	// section string
	// prefix  string

	table string
}

func newMetaWrap(section string) *MetaWrap {
	if section == "" {
		section = "common"
	}
	dsn := config.GetValue(section, "meta_dsn")
	table := config.GetValue(section, "meta_table")
	mw := &MetaWrap{dsn: dsn, table: table}

	return mw
}

func NewMetaWrapper(section string) (mw MetaWrapper) {
	mw = newMetaWrap(section)
	return mw
}

func (mw *MetaWrap) Browse(limit int, offset int) (rows *sql.Rows, err error) {
	if limit < 1 {
		limit = 1
	}
	if offset < 0 {
		offset = 0
	}

	db := mw.getDb()

	table := mw.table

	rows, err = db.Query("SELECT * FROM "+table+" LIMIT $1 OFFSET $2", limit, offset)

	log.Print(rows)
	return rows, err
}

func (mw *MetaWrap) Get(id EntryId) (*Entry, error) {
	db := mw.getDb()
	defer db.Close()

	table := mw.table
	sql := "SELECT name, path, size, mime, meta, ids, hashes FROM " + table + " WHERE id = $1 LIMIT 1"
	entry := Entry{Id: &id}
	row := db.QueryRow(sql, id.String())

	var meta cdb.Hstore
	err := row.Scan(&entry.Name, &entry.Path, &entry.Size, &entry.Mime, &meta, &entry.Hashes, &entry.Ids)

	if err != nil {
		log.Println(err)
		return &entry, err
	}
	// entry.Meta = image.
	log.Println("meta:", meta)
	var ia image.ImageAttr
	err = meta.ToStruct(&ia)
	if err != nil {
		log.Println(err)
		return &entry, err
	}
	log.Println(ia.Width)

	entry.Meta = &ia

	log.Printf("name: %s, path: %s, size: %d, mime: %s\n", entry.Name, entry.Path, entry.Size, entry.Mime)

	return &entry, nil
}

func (mw *MetaWrap) Store(entry *Entry) error {
	db := mw.getDb()
	defer db.Close()

	table := mw.table
	log.Println("table:", table)
	log.Printf("hashes: %s\n", entry.Hashes)
	log.Printf("ids: %s\n", entry.Ids)
	// hashes := "{" + strings.Join(entry.Hashes, ",") + "}"
	// ids := "{" + strings.Join(entry.Ids, ",") + "}"
	meta := entry.Meta.Hstore()
	log.Println(meta)
	tx, err := db.Begin()
	if err != nil {
		log.Println(err)
		// tx.Rollback()
		return err
	}

	sql := "INSERT INTO " + table + "(id, name, hashes, ids, meta, path, size, mime) VALUES($1, $2, $3, $4, $5, $6, $7, $8)"
	result, err := tx.Exec(sql, entry.Id.String(), entry.Name, entry.Hashes, entry.Ids, meta, entry.Path, entry.Size, entry.Mime)

	if err != nil {
		log.Fatal(err)
	}

	var ra int64
	ra, err = result.RowsAffected()

	if err != nil {
		log.Fatal(err)
	}

	log.Println("RowsAffected ", ra)

	tx.Commit()

	return nil
}

func (mw *MetaWrap) Delete(id EntryId) error {
	return nil
}

func (mw *MetaWrap) getDb() *sql.DB {
	dsn := mw.dsn
	db, err := sql.Open("postgres", dsn)

	if err != nil {
		log.Fatal(err)
	}
	return db
}
<commit_msg>fix metaWrap.Get()<commit_after>package storage

import (
	"calf/config"
	cdb "calf/db"
	"calf/image"
	"database/sql"
	_ "database/sql/driver"
	_ "github.com/lib/pq"
	"log"
	// "strings"
)

type MetaWrapper interface {
	Browse(limit int, offset int) (*sql.Rows, error)
	Store(entry *Entry) error
	Get(id EntryId) (*Entry, error)
	Delete(id EntryId) error
}

type MetaWrap struct {
	dsn string
	// section string
	// prefix  string

	table string
}

func newMetaWrap(section string) *MetaWrap {
	if section == "" {
		section = "common"
	}
	dsn := config.GetValue(section, "meta_dsn")
	table := config.GetValue(section, "meta_table")
	mw := &MetaWrap{dsn: dsn, table: table}

	return mw
}

func NewMetaWrapper(section string) (mw MetaWrapper) {
	mw = newMetaWrap(section)
	return mw
}

func (mw *MetaWrap) Browse(limit int, offset int) (rows *sql.Rows, err error) {
	if limit < 1 {
		limit = 1
	}
	if offset < 0 {
		offset = 0
	}

	db := mw.getDb()

	table := mw.table

	rows, err = db.Query("SELECT * FROM "+table+" LIMIT $1 OFFSET $2", limit, offset)

	log.Print(rows)
	return rows, err
}

func (mw *MetaWrap) Get(id EntryId) (*Entry, error) {
	db := mw.getDb()
	defer db.Close()

	table := mw.table
	sql := "SELECT name, path, size, mime, meta, ids, hashes FROM " + table + " WHERE id = $1 LIMIT 1"
	entry := Entry{Id: &id}
	row := db.QueryRow(sql, id.String())

	var meta cdb.Hstore
	err := row.Scan(&entry.Name, &entry.Path, &entry.Size, &entry.Mime, &meta, &entry.Ids, &entry.Hashes)

	if err != nil {
		log.Println(err)
		return &entry, err
	}

	log.Println("first id:", entry.Ids[0])
	// entry.Meta = image.
	log.Println("meta:", meta)
	var ia image.ImageAttr
	err = meta.ToStruct(&ia)
	if err != nil {
		log.Println(err)
		return &entry, err
	}
	log.Println(ia.Width)

	entry.Meta = &ia

	log.Printf("name: %s, path: %s, size: %d, mime: %s\n", entry.Name, entry.Path, entry.Size, entry.Mime)

	return &entry, nil
}

func (mw *MetaWrap) Store(entry *Entry) error {
	db := mw.getDb()
	defer db.Close()

	table := mw.table
	log.Println("table:", table)
	log.Printf("hashes: %s\n", entry.Hashes)
	log.Printf("ids: %s\n", entry.Ids)
	// hashes := "{" + strings.Join(entry.Hashes, ",") + "}"
	// ids := "{" + strings.Join(entry.Ids, ",") + "}"
	meta := entry.Meta.Hstore()
	log.Println(meta)
	tx, err := db.Begin()
	if err != nil {
		log.Println(err)
		// tx.Rollback()
		return err
	}

	sql := "INSERT INTO " + table + "(id, name, hashes, ids, meta, path, size, mime) VALUES($1, $2, $3, $4, $5, $6, $7, $8)"
	result, err := tx.Exec(sql, entry.Id.String(), entry.Name, entry.Hashes, entry.Ids, meta, entry.Path, entry.Size, entry.Mime)

	if err != nil {
		log.Fatal(err)
	}

	var ra int64
	ra, err = result.RowsAffected()

	if err != nil {
		log.Fatal(err)
	}

	log.Println("RowsAffected ", ra)

	tx.Commit()

	return nil
}

func (mw *MetaWrap) Delete(id EntryId) error {
	return nil
}

func (mw *MetaWrap) getDb() *sql.DB {
	dsn := mw.dsn
	db, err := sql.Open("postgres", dsn)

	if err != nil {
		log.Fatal(err)
	}
	return db
}
<|endoftext|>


<commit_before>/*-----------------------------------------------------------------------------
The MIT License (MIT)

 Copyright (C) Business Learning Incorporated (www.businesslearninginc.com)

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
-----------------------------------------------------------------------------*/

//
//  EsriASCIIParser - ESRI ASCII Raster Data File Parser
//
//  Validate and parse ESRI ASCII raster data file to various devices (i.e.,
//  console, file, database)
//
//  ESRI format specification reference:
//    http://resources.esri.com/help/9.3/ArcGISDesktop/com/Gp_ToolRef \
//      /Spatial_Analyst_Tools/esri_ascii_raster_format.htm
//

package main

import (
	"bufio"
	"flag"
	"fmt"
	"path/filepath"
	"strconv"
	"strings"

	"database/sql"
	_ "github.com/go-sql-driver/mysql"

	"errors"
	"os"
	"time"
)

// ================================================================================================
// local global constants/variables

// user-assigned values to override command-line interface flags (based on useCommandLineFlags)
const useCommandLineFlags bool = true

var outputToConsole = true
var outputToFile = true
var outputToDb = false

var inFilePath = "/tmp/cymuds00ag.asc"
var outFilePath = "/tmp/file.out"
var regionName = "esri_region_name"
var dbName = "root:root@/test_world"

// non-user-assigned constants
const appVersion = "0.10"

// esri ascii raster data format parameters
// http://resources.esri.com/help/9.3/ArcGISDesktop/com/Gp_ToolRef/Spatial_Analyst_Tools/esri_ascii_raster_format.htm
const (
	nCols = iota
	nRows
	xllCorner
	yllCorner
	cellSize
	noDataValue
)

type esriHeaderNameStruct struct {
	parameter string
	value     string
}

type esriHeaderDataStruct []struct {
	parameter string
	value     float64
}

type esriHeaderStruct struct {
	esriHeaderName esriHeaderNameStruct
	esriHeaderData esriHeaderDataStruct
}

// ================================================================================================
// functions

// checkErr is a simple wrapper to check err value passed to it
func checkErr(err error) {

	if err != nil {
		panic(err)
	}

}

// boolToInt converts a bool to an int (true = 1, else 0)
func boolToInt(b bool) int {

	if b {
		return 1
	}
	return 0
}

// showBanner displays the title banner to Stdout (configured for 80-columns)
func showBanner() {

	fmt.Fprintf(os.Stdout, "--------------------------------------------------------------------------------\n")
	fmt.Fprintf(os.Stdout, "  ESRI ASCII raster data file parser (v%s)\n", appVersion)
	fmt.Fprintf(os.Stdout, "  validate and parse file to console, file, or database\n")
	fmt.Fprintf(os.Stdout, "--------------------------------------------------------------------------------\n\n")

}

// parseFlags parses command-line flags and passes them on function return
//   flags used:
//     -dev=con, file, db (device output, can be cumulative)
//     -in=infile (fully qualified pathname to input esri ascii raster format file)
//     -out=outfile (relative path to output file to persist parse results)
//     -db=database (database name to persist parse results)
//	   -name=region name (when persisted to database)
func parseFlags() (inFile string, outputToConsole bool, outputToFile bool, outputToDb bool, outFile string, dbFile string, regionName string) {

	devFlag := flag.String("dev", "con", "output device(s): con, file, db, e.g,. -dev=con,file")
	inFileFlag := flag.String("in", "", "fully qualified pathname of esri ascii raster file to parse (required)")
	outFileFlag := flag.String("out", "/tmp/file.out", "file to persist parsed results (used when -dev=file)")
	dbFlag := flag.String("db", "", "database to persist parsed results (used when -dev=db)")
	nameFlag := flag.String("name", "region", "name to uniquely identify a region (used when -dev=db)")

	flag.Usage = func() {

		fmt.Fprintf(os.Stderr, "\nCurrent settings:\n")
		fmt.Fprintf(os.Stderr, "  -in=%s\n", *inFileFlag)
		fmt.Fprintf(os.Stderr, "  -out=%s\n", *outFileFlag)
		fmt.Fprintf(os.Stderr, "  -dev=%s\n", *devFlag)
		fmt.Fprintf(os.Stderr, "  -db=%s\n", *dbFlag)
		fmt.Fprintf(os.Stderr, "  -name=%s\n", *nameFlag)

		fmt.Fprintf(os.Stderr,
			"\nUsage:\n  %s (-in=\"</path/to/esri_file>\") "+
				"[-out=\"</path/to/file.out\">]\n"+
				"    [-dev=<\"con\">, <\"file\">, <\"db\">]\n"+
				"    [-db=\"[username[:password]@][protocol[(address)]]/dbname\"]\n"+
				"    [-name=<\"region\">]\n\n",
			filepath.Base(os.Args[0]))

		fmt.Fprintf(os.Stderr, "Options:\n")
		flag.PrintDefaults()

		fmt.Fprintf(os.Stderr, "\nExamples:\n")
		fmt.Fprintf(os.Stderr, "  %s -in=/abc.asc\n", filepath.Base(os.Args[0]))
		fmt.Fprintf(os.Stderr, "  %s -in=/abc.asc -dev=con,file\n", filepath.Base(os.Args[0]))
		fmt.Fprintf(os.Stderr, "  %s -in=/abc.asc -dev=file -out=/tmp/results.out\n", filepath.Base(os.Args[0]))
		fmt.Fprintf(os.Stderr, "  %s -in=/abc.asc -dev=db -db=usr:pwd@tcp(10.10.10.1:3306)/db\n", filepath.Base(os.Args[0]))
		fmt.Fprintf(os.Stderr, "\n")
		os.Exit(0)
	}

	flag.Parse()
	flagErr := false

	if *inFileFlag == "" {
		fmt.Fprintf(os.Stderr, "-in must be set to a fully qualified pathname for input\n")
		flagErr = true
	}

	if strings.Contains(*devFlag, "con") {
		outputToConsole = true
	}

	if strings.Contains(*devFlag, "file") {
		outputToFile = true

		if *outFileFlag == "" {
			fmt.Fprintf(os.Stderr, "when -dev=file, -out must be set to a filename\n")
			flagErr = true
		}
	}

	if strings.Contains(*devFlag, "db") {
		outputToDb = true

		if *dbFlag == "" {
			fmt.Fprintf(os.Stderr, "when -dev=db, -db must be set to a database\n")
			flagErr = true
		}

		if *nameFlag == "" {
			fmt.Fprintf(os.Stderr, "when -dev=db, -name must be set to identify the region\n")
			flagErr = true
		}

	}

	if outputToDb == false && outputToFile == false && outputToConsole == false {
		fmt.Fprintf(os.Stderr, "-dev must be con, file, and/or db\n")
		flagErr = true
	}

	if flagErr {
		flag.Usage()
	}

	return *inFileFlag, outputToConsole, outputToFile, outputToDb, *outFileFlag, *dbFlag, *nameFlag

}

// esriHeaderProcess scans only the esri raster data header block and determines if the header is valid
// and returns whether the optional NoDataValue parameter is present
func esriHeaderProcess(esriHeader esriHeaderStruct) (tableID int64, hasNoDataValue bool) {

	// noDataValue is an optional parameter in esri ascii raster data, so parse for all parameters, but special case for noDataValue
	// and return bool on presence in header block
	hasNoDataValue = true

	tableID = 0
	inputFile, err := os.Open(inFilePath)
	checkErr(err)
	defer inputFile.Close()

	scanner := bufio.NewScanner(inputFile)

	for i := 0; i < 6; i++ {

		scanner.Scan()
		parameter := strings.ToLower(strings.Fields(scanner.Text())[0])

		// parse for esriHeaderData parameters
		if esriHeader.esriHeaderData[i].parameter == parameter {
			if len(strings.Fields(scanner.Text())) < 2 {
				checkErr(errors.New("invalid esri file format: header parameter value for " + esriHeader.esriHeaderData[i].parameter + " not found"))
			}
			value, err := strconv.ParseFloat(strings.Fields(scanner.Text())[1], 16)
			checkErr(err)
			esriHeader.esriHeaderData[i].value = value
		} else {

			if i == noDataValue {
				hasNoDataValue = false                     // optional noDataValue parameter not found in header
				esriHeader.esriHeaderData[i].value = -9999 // default parameter value
			} else {
				checkErr(errors.New("invalid esri file format: header parameter " + esriHeader.esriHeaderData[i].parameter + " not found"))
			}
		}

	}

	if outputToDb {
		tableID = esriHeaderPersist(esriHeader)
	}

	if outputToConsole {
		fmt.Println("esri header processed as:\n", esriHeader, "\n")
	}

	if outputToFile {
		file, err := os.Create(outFilePath)
		checkErr(err)
		defer file.Close()

		fmt.Fprintln(file, esriHeader)
	}

	return tableID, hasNoDataValue

}

// esriDataProcess scans only the esri raster data block and builds lat/lon values from header block
// configuration details, persisting them to the specified output device
func esriDataProcess(esriHeader esriHeaderStruct, tableID int64, hasNoDataValue bool) {

	var file *os.File

	inputFile, err := os.Open(inFilePath)
	checkErr(err)
	defer inputFile.Close()
	scanner := bufio.NewScanner(inputFile)

	// advance file pointer over esri file header block
	for i := 0; i < 5+boolToInt(hasNoDataValue); i++ {
		scanner.Scan()
	}

	if outputToFile {
		file, err = os.Create(outFilePath)
		checkErr(err)
		defer file.Close()
	}

	// calculate upper-left reference datum
	lat := esriHeader.esriHeaderData[yllCorner].value + (esriHeader.esriHeaderData[nRows].value * esriHeader.esriHeaderData[cellSize].value)

	// loop through esri data block (nCols x nRows)
	for i := 0; i < int(esriHeader.esriHeaderData[nRows].value); i++ {

		scanner.Scan()
		value := strings.Split(scanner.Text(), " ")

		if int(esriHeader.esriHeaderData[nCols].value) != len(value)-1 {
			checkErr(errors.New("nCols != line length"))
		}

		lon := esriHeader.esriHeaderData[xllCorner].value
		lat = lat - esriHeader.esriHeaderData[cellSize].value

		for j := 0; j < len(value)-1; j++ {

			value, err := strconv.ParseFloat(value[j], 16)
			checkErr(err)

			if outputToConsole {
				fmt.Println("lat:", float32(lat), ", lon:", float32(lon), ", value:", value)
			}

			if outputToFile {
				fmt.Fprintln(file, float32(lat), ",", float32(lon), ",", value)
			}

			if outputToDb {
				esriDataPersist(lat, lon, value, tableID)
			}

			lon = lon + esriHeader.esriHeaderData[cellSize].value
		}
	}

}

// esriHeaderPersist takes the esriHeaderStruct object and persists into mysql database as specified
// by user through command-line flags
func esriHeaderPersist(esriHeader esriHeaderStruct) (tableID int64) {

	db, err := sql.Open("mysql", dbName)
	err = db.Ping()
	checkErr(err)
	defer db.Close()

	result, err := db.Exec("INSERT INTO esriHeader(name, nCols, nRows, xllCorner, yllCorner, cellSize, noDataValue) VALUES(?, ?, ?, ?, ?, ?, ?)",
		esriHeader.esriHeaderName.value, esriHeader.esriHeaderData[nCols].value, int(esriHeader.esriHeaderData[nRows].value), esriHeader.esriHeaderData[xllCorner].value, esriHeader.esriHeaderData[yllCorner].value,
		esriHeader.esriHeaderData[cellSize].value, esriHeader.esriHeaderData[noDataValue].value)
	checkErr(err)

	tableID, err = result.LastInsertId()
	checkErr(err)

	return tableID

}

// esriDataPersist takes the data value objects (lat, lon, and value) and persists into mysql database
func esriDataPersist(lat float64, lon float64, value float64, tableID int64) {

	// TODO: non-optimal, db open/close should be moved into separate func, since this function gets called iteratively
	db, err := sql.Open("mysql", dbName)
	err = db.Ping()
	checkErr(err)
	defer db.Close()

	_, err = db.Exec("INSERT INTO esriData(lat, lon, value, esriHeader_id_esriHeader) VALUES(?, ?, ?, ?)", lat, lon, value, tableID)
	checkErr(err)

}

//
// ================================================================================================
//

func main() {

	if outputToConsole {
		showBanner()
	}

	if useCommandLineFlags {
		inFilePath, outputToConsole, outputToFile, outputToDb, outFilePath, dbName, regionName = parseFlags()
	}

	startTime := time.Now().Local()

	var esriHeader = esriHeaderStruct{
		esriHeaderNameStruct{
			"name", regionName,
		},
		esriHeaderDataStruct{
			{"ncols", 0},
			{"nrows", 0},
			{"xllcorner", 0},
			{"yllcorner", 0},
			{"cellsize", 0},
			{"nodata_value", 0},
		},
	}

	// get esri file HEADER block
	// need to know if NoDataValue parameter is present for esri file data block parsing
	tableID, hasNoDataValue := esriHeaderProcess(esriHeader)

	// get esri file DATA block
	esriDataProcess(esriHeader, tableID, hasNoDataValue)

	if outputToConsole {
		fmt.Println("\n===========\nTime elapsed:", time.Since(startTime))
	}

}
<commit_msg>Update to license reference in source file<commit_after>//
// Copyright (C) Business Learning Incorporated (www.businesslearninginc.com)
//
// Use of this source code is governed by an MIT-style license
// that can be found in the LICENSE file
//

//
//  EsriASCIIParser - ESRI ASCII Raster Data File Parser
//
//  Validate and parse ESRI ASCII raster data file to various devices (i.e.,
//  console, file, database)
//
//  ESRI format specification reference:
//    http://resources.esri.com/help/9.3/ArcGISDesktop/com/Gp_ToolRef \
//      /Spatial_Analyst_Tools/esri_ascii_raster_format.htm
//

package main

import (
	"bufio"
	"flag"
	"fmt"
	"path/filepath"
	"strconv"
	"strings"

	"database/sql"
	_ "github.com/go-sql-driver/mysql"

	"errors"
	"os"
	"time"
)

// ================================================================================================
// local global constants/variables

// user-assigned values to override command-line interface flags (based on useCommandLineFlags)
const useCommandLineFlags bool = true

var outputToConsole = true
var outputToFile = true
var outputToDb = false

var inFilePath = "/tmp/cymuds00ag.asc"
var outFilePath = "/tmp/file.out"
var regionName = "esri_region_name"
var dbName = "root:root@/test_world"

// non-user-assigned constants
const appVersion = "0.10"

// esri ascii raster data format parameters
// http://resources.esri.com/help/9.3/ArcGISDesktop/com/Gp_ToolRef/Spatial_Analyst_Tools/esri_ascii_raster_format.htm
const (
	nCols = iota
	nRows
	xllCorner
	yllCorner
	cellSize
	noDataValue
)

type esriHeaderNameStruct struct {
	parameter string
	value     string
}

type esriHeaderDataStruct []struct {
	parameter string
	value     float64
}

type esriHeaderStruct struct {
	esriHeaderName esriHeaderNameStruct
	esriHeaderData esriHeaderDataStruct
}

// ================================================================================================
// functions

// checkErr is a simple wrapper to check err value passed to it
func checkErr(err error) {

	if err != nil {
		panic(err)
	}

}

// boolToInt converts a bool to an int (true = 1, else 0)
func boolToInt(b bool) int {

	if b {
		return 1
	}
	return 0
}

// showBanner displays the title banner to Stdout (configured for 80-columns)
func showBanner() {

	fmt.Fprintf(os.Stdout, "--------------------------------------------------------------------------------\n")
	fmt.Fprintf(os.Stdout, "  ESRI ASCII raster data file parser (v%s)\n", appVersion)
	fmt.Fprintf(os.Stdout, "  validate and parse file to console, file, or database\n")
	fmt.Fprintf(os.Stdout, "--------------------------------------------------------------------------------\n\n")

}

// parseFlags parses command-line flags and passes them on function return
//   flags used:
//     -dev=con, file, db (device output, can be cumulative)
//     -in=infile (fully qualified pathname to input esri ascii raster format file)
//     -out=outfile (relative path to output file to persist parse results)
//     -db=database (database name to persist parse results)
//	   -name=region name (when persisted to database)
func parseFlags() (inFile string, outputToConsole bool, outputToFile bool, outputToDb bool, outFile string, dbFile string, regionName string) {

	devFlag := flag.String("dev", "con", "output device(s): con, file, db, e.g,. -dev=con,file")
	inFileFlag := flag.String("in", "", "fully qualified pathname of esri ascii raster file to parse (required)")
	outFileFlag := flag.String("out", "/tmp/file.out", "file to persist parsed results (used when -dev=file)")
	dbFlag := flag.String("db", "", "database to persist parsed results (used when -dev=db)")
	nameFlag := flag.String("name", "region", "name to uniquely identify a region (used when -dev=db)")

	flag.Usage = func() {

		fmt.Fprintf(os.Stderr, "\nCurrent settings:\n")
		fmt.Fprintf(os.Stderr, "  -in=%s\n", *inFileFlag)
		fmt.Fprintf(os.Stderr, "  -out=%s\n", *outFileFlag)
		fmt.Fprintf(os.Stderr, "  -dev=%s\n", *devFlag)
		fmt.Fprintf(os.Stderr, "  -db=%s\n", *dbFlag)
		fmt.Fprintf(os.Stderr, "  -name=%s\n", *nameFlag)

		fmt.Fprintf(os.Stderr,
			"\nUsage:\n  %s (-in=\"</path/to/esri_file>\") "+
				"[-out=\"</path/to/file.out\">]\n"+
				"    [-dev=<\"con\">, <\"file\">, <\"db\">]\n"+
				"    [-db=\"[username[:password]@][protocol[(address)]]/dbname\"]\n"+
				"    [-name=<\"region\">]\n\n",
			filepath.Base(os.Args[0]))

		fmt.Fprintf(os.Stderr, "Options:\n")
		flag.PrintDefaults()

		fmt.Fprintf(os.Stderr, "\nExamples:\n")
		fmt.Fprintf(os.Stderr, "  %s -in=/abc.asc\n", filepath.Base(os.Args[0]))
		fmt.Fprintf(os.Stderr, "  %s -in=/abc.asc -dev=con,file\n", filepath.Base(os.Args[0]))
		fmt.Fprintf(os.Stderr, "  %s -in=/abc.asc -dev=file -out=/tmp/results.out\n", filepath.Base(os.Args[0]))
		fmt.Fprintf(os.Stderr, "  %s -in=/abc.asc -dev=db -db=usr:pwd@tcp(10.10.10.1:3306)/db\n", filepath.Base(os.Args[0]))
		fmt.Fprintf(os.Stderr, "\n")
		os.Exit(0)
	}

	flag.Parse()
	flagErr := false

	if *inFileFlag == "" {
		fmt.Fprintf(os.Stderr, "-in must be set to a fully qualified pathname for input\n")
		flagErr = true
	}

	if strings.Contains(*devFlag, "con") {
		outputToConsole = true
	}

	if strings.Contains(*devFlag, "file") {
		outputToFile = true

		if *outFileFlag == "" {
			fmt.Fprintf(os.Stderr, "when -dev=file, -out must be set to a filename\n")
			flagErr = true
		}
	}

	if strings.Contains(*devFlag, "db") {
		outputToDb = true

		if *dbFlag == "" {
			fmt.Fprintf(os.Stderr, "when -dev=db, -db must be set to a database\n")
			flagErr = true
		}

		if *nameFlag == "" {
			fmt.Fprintf(os.Stderr, "when -dev=db, -name must be set to identify the region\n")
			flagErr = true
		}

	}

	if outputToDb == false && outputToFile == false && outputToConsole == false {
		fmt.Fprintf(os.Stderr, "-dev must be con, file, and/or db\n")
		flagErr = true
	}

	if flagErr {
		flag.Usage()
	}

	return *inFileFlag, outputToConsole, outputToFile, outputToDb, *outFileFlag, *dbFlag, *nameFlag

}

// esriHeaderProcess scans only the esri raster data header block and determines if the header is valid
// and returns whether the optional NoDataValue parameter is present
func esriHeaderProcess(esriHeader esriHeaderStruct) (tableID int64, hasNoDataValue bool) {

	// noDataValue is an optional parameter in esri ascii raster data, so parse for all parameters, but special case for noDataValue
	// and return bool on presence in header block
	hasNoDataValue = true

	tableID = 0
	inputFile, err := os.Open(inFilePath)
	checkErr(err)
	defer inputFile.Close()

	scanner := bufio.NewScanner(inputFile)

	for i := 0; i < 6; i++ {

		scanner.Scan()
		parameter := strings.ToLower(strings.Fields(scanner.Text())[0])

		// parse for esriHeaderData parameters
		if esriHeader.esriHeaderData[i].parameter == parameter {
			if len(strings.Fields(scanner.Text())) < 2 {
				checkErr(errors.New("invalid esri file format: header parameter value for " + esriHeader.esriHeaderData[i].parameter + " not found"))
			}
			value, err := strconv.ParseFloat(strings.Fields(scanner.Text())[1], 16)
			checkErr(err)
			esriHeader.esriHeaderData[i].value = value
		} else {

			if i == noDataValue {
				hasNoDataValue = false                     // optional noDataValue parameter not found in header
				esriHeader.esriHeaderData[i].value = -9999 // default parameter value
			} else {
				checkErr(errors.New("invalid esri file format: header parameter " + esriHeader.esriHeaderData[i].parameter + " not found"))
			}
		}

	}

	if outputToDb {
		tableID = esriHeaderPersist(esriHeader)
	}

	if outputToConsole {
		fmt.Println("esri header processed as:\n", esriHeader, "\n")
	}

	if outputToFile {
		file, err := os.Create(outFilePath)
		checkErr(err)
		defer file.Close()

		fmt.Fprintln(file, esriHeader)
	}

	return tableID, hasNoDataValue

}

// esriDataProcess scans only the esri raster data block and builds lat/lon values from header block
// configuration details, persisting them to the specified output device
func esriDataProcess(esriHeader esriHeaderStruct, tableID int64, hasNoDataValue bool) {

	var file *os.File

	inputFile, err := os.Open(inFilePath)
	checkErr(err)
	defer inputFile.Close()
	scanner := bufio.NewScanner(inputFile)

	// advance file pointer over esri file header block
	for i := 0; i < 5+boolToInt(hasNoDataValue); i++ {
		scanner.Scan()
	}

	if outputToFile {
		file, err = os.Create(outFilePath)
		checkErr(err)
		defer file.Close()
	}

	// calculate upper-left reference datum
	lat := esriHeader.esriHeaderData[yllCorner].value + (esriHeader.esriHeaderData[nRows].value * esriHeader.esriHeaderData[cellSize].value)

	// loop through esri data block (nCols x nRows)
	for i := 0; i < int(esriHeader.esriHeaderData[nRows].value); i++ {

		scanner.Scan()
		value := strings.Split(scanner.Text(), " ")

		if int(esriHeader.esriHeaderData[nCols].value) != len(value)-1 {
			checkErr(errors.New("nCols != line length"))
		}

		lon := esriHeader.esriHeaderData[xllCorner].value
		lat = lat - esriHeader.esriHeaderData[cellSize].value

		for j := 0; j < len(value)-1; j++ {

			value, err := strconv.ParseFloat(value[j], 16)
			checkErr(err)

			if outputToConsole {
				fmt.Println("lat:", float32(lat), ", lon:", float32(lon), ", value:", value)
			}

			if outputToFile {
				fmt.Fprintln(file, float32(lat), ",", float32(lon), ",", value)
			}

			if outputToDb {
				esriDataPersist(lat, lon, value, tableID)
			}

			lon = lon + esriHeader.esriHeaderData[cellSize].value
		}
	}

}

// esriHeaderPersist takes the esriHeaderStruct object and persists into mysql database as specified
// by user through command-line flags
func esriHeaderPersist(esriHeader esriHeaderStruct) (tableID int64) {

	db, err := sql.Open("mysql", dbName)
	err = db.Ping()
	checkErr(err)
	defer db.Close()

	result, err := db.Exec("INSERT INTO esriHeader(name, nCols, nRows, xllCorner, yllCorner, cellSize, noDataValue) VALUES(?, ?, ?, ?, ?, ?, ?)",
		esriHeader.esriHeaderName.value, esriHeader.esriHeaderData[nCols].value, int(esriHeader.esriHeaderData[nRows].value), esriHeader.esriHeaderData[xllCorner].value, esriHeader.esriHeaderData[yllCorner].value,
		esriHeader.esriHeaderData[cellSize].value, esriHeader.esriHeaderData[noDataValue].value)
	checkErr(err)

	tableID, err = result.LastInsertId()
	checkErr(err)

	return tableID

}

// esriDataPersist takes the data value objects (lat, lon, and value) and persists into mysql database
func esriDataPersist(lat float64, lon float64, value float64, tableID int64) {

	// TODO: non-optimal, db open/close should be moved into separate func, since this function gets called iteratively
	db, err := sql.Open("mysql", dbName)
	err = db.Ping()
	checkErr(err)
	defer db.Close()

	_, err = db.Exec("INSERT INTO esriData(lat, lon, value, esriHeader_id_esriHeader) VALUES(?, ?, ?, ?)", lat, lon, value, tableID)
	checkErr(err)

}

//
// ================================================================================================
//

func main() {

	if outputToConsole {
		showBanner()
	}

	if useCommandLineFlags {
		inFilePath, outputToConsole, outputToFile, outputToDb, outFilePath, dbName, regionName = parseFlags()
	}

	startTime := time.Now().Local()

	var esriHeader = esriHeaderStruct{
		esriHeaderNameStruct{
			"name", regionName,
		},
		esriHeaderDataStruct{
			{"ncols", 0},
			{"nrows", 0},
			{"xllcorner", 0},
			{"yllcorner", 0},
			{"cellsize", 0},
			{"nodata_value", 0},
		},
	}

	// get esri file HEADER block
	// need to know if NoDataValue parameter is present for esri file data block parsing
	tableID, hasNoDataValue := esriHeaderProcess(esriHeader)

	// get esri file DATA block
	esriDataProcess(esriHeader, tableID, hasNoDataValue)

	if outputToConsole {
		fmt.Println("\n===========\nTime elapsed:", time.Since(startTime))
	}

}
<|endoftext|>


<commit_before>package ethchain

import (
	"github.com/ethereum/eth-go/ethutil"
	"math/big"
)

type Contract struct {
	Amount *big.Int
	Nonce  uint64
	//state  *ethutil.Trie
	state *State
}

func NewContract(Amount *big.Int, root []byte) *Contract {
	contract := &Contract{Amount: Amount, Nonce: 0}
	contract.state = NewState(ethutil.NewTrie(ethutil.Config.Db, string(root)))

	return contract
}

func (c *Contract) RlpEncode() []byte {
	return ethutil.Encode([]interface{}{c.Amount, c.Nonce, c.state.trie.Root})
}

func (c *Contract) RlpDecode(data []byte) {
	decoder := ethutil.NewValueFromBytes(data)

	c.Amount = decoder.Get(0).BigInt()
	c.Nonce = decoder.Get(1).Uint()
	c.state = NewState(ethutil.NewTrie(ethutil.Config.Db, decoder.Get(2).Interface()))
}

func (c *Contract) Addr(addr []byte) *ethutil.Value {
	return ethutil.NewValueFromBytes([]byte(c.state.trie.Get(string(addr))))
}

func (c *Contract) SetAddr(addr []byte, value interface{}) {
	c.state.trie.Update(string(addr), string(ethutil.NewValue(value).Encode()))
}

func (c *Contract) State() *State {
	return c.state
}

func (c *Contract) GetMem(num int) *ethutil.Value {
	nb := ethutil.BigToBytes(big.NewInt(int64(num)), 256)

	return c.Addr(nb)
}

func MakeContract(tx *Transaction, state *State) *Contract {
	// Create contract if there's no recipient
	if tx.IsContract() {
		addr := tx.Hash()[12:]

		value := tx.Value
		contract := NewContract(value, []byte(""))
		state.trie.Update(string(addr), string(contract.RlpEncode()))
		for i, val := range tx.Data {
			if len(val) > 0 {
				bytNum := ethutil.BigToBytes(big.NewInt(int64(i)), 256)
				contract.state.trie.Update(string(bytNum), string(ethutil.Encode(val)))
			}
		}
		state.trie.Update(string(addr), string(contract.RlpEncode()))

		return contract
	}

	return nil
}
<commit_msg>Comply to Callee structure<commit_after>package ethchain

import (
	"github.com/ethereum/eth-go/ethutil"
	"math/big"
)

type Contract struct {
	Amount *big.Int
	Nonce  uint64
	//state  *ethutil.Trie
	state *State
}

func NewContract(Amount *big.Int, root []byte) *Contract {
	contract := &Contract{Amount: Amount, Nonce: 0}
	contract.state = NewState(ethutil.NewTrie(ethutil.Config.Db, string(root)))

	return contract
}

func (c *Contract) RlpEncode() []byte {
	return ethutil.Encode([]interface{}{c.Amount, c.Nonce, c.state.trie.Root})
}

func (c *Contract) RlpDecode(data []byte) {
	decoder := ethutil.NewValueFromBytes(data)

	c.Amount = decoder.Get(0).BigInt()
	c.Nonce = decoder.Get(1).Uint()
	c.state = NewState(ethutil.NewTrie(ethutil.Config.Db, decoder.Get(2).Interface()))
}

func (c *Contract) Addr(addr []byte) *ethutil.Value {
	return ethutil.NewValueFromBytes([]byte(c.state.trie.Get(string(addr))))
}

func (c *Contract) SetAddr(addr []byte, value interface{}) {
	c.state.trie.Update(string(addr), string(ethutil.NewValue(value).Encode()))
}

func (c *Contract) State() *State {
	return c.state
}

func (c *Contract) GetMem(num int64) *ethutil.Value {
	nb := ethutil.BigToBytes(big.NewInt(num), 256)

	return c.Addr(nb)
}

// Return the gas back to the origin. Used by the Virtual machine or Closures
func (c *Contract) ReturnGas(val *big.Int, state *State) {
	c.Amount.Add(c.Amount, val)
}

func MakeContract(tx *Transaction, state *State) *Contract {
	// Create contract if there's no recipient
	if tx.IsContract() {
		addr := tx.Hash()[12:]

		value := tx.Value
		contract := NewContract(value, []byte(""))
		state.trie.Update(string(addr), string(contract.RlpEncode()))
		for i, val := range tx.Data {
			if len(val) > 0 {
				bytNum := ethutil.BigToBytes(big.NewInt(int64(i)), 256)
				contract.state.trie.Update(string(bytNum), string(ethutil.Encode(val)))
			}
		}
		state.trie.Update(string(addr), string(contract.RlpEncode()))

		return contract
	}

	return nil
}
<|endoftext|>


<commit_before>package koding

import (
	"time"

	"github.com/koding/kite"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"

	"koding/kites/kloud/klient"
	"koding/kites/kloud/machinestate"
	"koding/kites/kloud/protocol"
)

func (p *Provider) Info(m *protocol.Machine) (result *protocol.InfoArtifact, err error) {
	// initial machine state is the state that the storage has
	dbState := m.State

	// Assume the klient state as unknown initially
	klientState := machinestate.Unknown

	// the final state that will be sent to the caller
	var resultState machinestate.State

	// Get the klient state.
	err = klient.Exists(p.Kite, m.QueryString)
	switch err {
	case nil:
		klientState = machinestate.Running
	case kite.ErrNoKitesAvailable:
		p.Log.Warning("[%s] Klient is disconnected, I couldn't find it through Kontrol. Err: %s",
			m.Id, err)
	default:
		// Any other error will fallback to here. So assume that kontrol
		// failed or some other catastrophic failure occured. Thus, do not
		// stop or destroy the machine because of our failure.
		p.Log.Critical("[%s] couldn't get klient information to check machine status: %s ", m.Id, err)
	}

	p.Log.Debug("[%s] Info initials: Current db state: '%s'. Klient state: '%s'",
		m.Id, dbState, klientState)

	switch klientState {
	case machinestate.Running:
		p.Log.Debug("[%s] Info log: klient is running", m.Id)

		// Stop the shutdown timer if there is any.
		p.stopTimer(m)

		if dbState == machinestate.Running {
			p.Log.Debug("[%s] Info log: klient is running and dbstate is running too. Doing nothing but returning", m.Id)
			return &protocol.InfoArtifact{
				State: machinestate.Running,
			}, nil
		}
		resultState = machinestate.Running
		dbState = machinestate.Running
		p.Log.Debug("[%s] Info log: klient is running and dbstate is not running. Setting dbstate to running.", m.Id)

	case machinestate.Unknown:
		p.Log.Debug("[%s] Info log: klient is not reachable. Asking to AWS about the machine state. Also started the shutdown-timer.", m.Id)

		// Start the shutdown timer since the klient is unreachable.
		// startTimer does not turn-off always-on machines, which is good
		p.startTimer(m)

		// Since klient is not registered, we have to ask for AWS about the machine state.
		a, err := p.NewClient(m)
		if err != nil {
			return nil, err
		}
		infoResp, err := a.Info()
		if err != nil {
			return nil, err
		}

		awsState := infoResp.State

		if dbState == awsState {
			p.Log.Debug("[%s] Info log: dbState(%s) and awsState(%s) are the same. Not gonna update the db", m.Id, dbState, awsState)
			return &protocol.InfoArtifact{
				State: awsState,
			}, nil
		}

		dbState = awsState
		resultState = awsState
	}

	// auto-fix db state if the klient state is different than db state. This will
	// not break existing actions like building, starting, stopping etc...
	// because CheckAndUpdateState only update the state if there is no lock
	// available.
	p.Log.Info("[%s] Info decision: Inconsistent state between the machine and db document. Updating state to '%s'.", m.Id, dbState)
	err = p.CheckAndUpdateState(m.Id, dbState)
	if err != nil {
		p.Log.Debug("[%s] Info decision: Error while updating the machine state. Err: %v", m.Id, err)
	}

	p.Log.Debug("[%s] Info result: '%s'. Username: %s", m.Id, resultState, m.Username)

	return &protocol.InfoArtifact{
		State: resultState,
	}, nil

}

// CheckAndUpdate state updates only if the given machine id is not used by
// anyone else
func (p *Provider) CheckAndUpdateState(id string, state machinestate.State) error {
	p.Log.Info("[%s] storage state update request to state %v", id, state)
	err := p.Session.Run("jMachines", func(c *mgo.Collection) error {
		return c.Update(
			bson.M{
				"_id": bson.ObjectIdHex(id),
				"assignee.inProgress": false, // only update if it's not locked by someone else
			},
			bson.M{
				"$set": bson.M{
					"status.state":      state.String(),
					"status.modifiedAt": time.Now().UTC(),
				},
			},
		)
	})

	if err == mgo.ErrNotFound {
		p.Log.Warning("[%s] info can't update db state because lock is acquired by someone else", id)
	}

	return err
}
<commit_msg>kloud: Ping the klient to check the machine state.<commit_after>package koding

import (
	"time"

	"github.com/koding/kite"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"

	"koding/kites/kloud/klient"
	"koding/kites/kloud/machinestate"
	"koding/kites/kloud/protocol"
)

// Info checks the machine state based on the klient and AWS states.
func (p *Provider) Info(m *protocol.Machine) (result *protocol.InfoArtifact, err error) {

	dbState := m.State
	resultState := dbState
	klientState := machinestate.Unknown

	// Check if klient is running first.
	klientRef, err := klient.Connect(p.Kite, m.QueryString)
	switch err {
	case nil:
		if err = klientRef.Ping(); err != nil {
			p.Log.Info("[%s] Klient is not responding to ping request. Err: %v", m.Id, err)
		}
		klientRef.Close()
		klientState = machinestate.Running
	case kite.ErrNoKitesAvailable:
		// klient state is still machinestate.Unknown.
		p.Log.Debug("[%s] Klient is not registered to Kontrol. Err: %s", m.Id, err)
	default:
		// Any other error will fallback to here. So assume that kontrol
		// failed or some other catastrophic failure occured. So, do not
		// stop or destroy the machine because of our failure.
		klientState = machinestate.Running
		p.Log.Critical("[%s] couldn't get klient information to check machine status. Probably couldn't connect to Kontrol. Err: %s ", m.Id, err)
	}

	reason := ""
	switch klientState {
	case machinestate.Running:
		reason = "Klient is active and healthy."

		p.stopTimer(m)
		resultState = machinestate.Running
	case machinestate.Unknown:
		reason = "Klient is not reachable."

		amz, err := p.NewClient(m)
		if err != nil {
			return nil, err
		}
		infoResp, err := amz.Info()
		if err != nil {
			return nil, err
		}

		switch infoResp.State {
		case machinestate.Running:
			// this is a case where: 1) klient is unreachable 2) machine is running
			// we don't want to give away our machines without a klient is running on it,
			// so mark and return as stopped.
			resultState = machinestate.Stopped

			// startTimer does not start a timer on always-on machines. no worries.
			p.startTimer(m)

			// Check if the machine is always-on and don't send a stopped state.
			if a, ok := m.Builder["alwaysOn"]; ok {
				if isAlwaysOn, ok := a.(bool); ok && isAlwaysOn {
					// machine is always-on. return as running
					resultState = machinestate.Running
				}
			}
		default:
			// This is the place where a state transition is in place or simply
			// the machine is stopped/terminated. So we don't expect the klient
			// to be run. Return as is.
			resultState = infoResp.State
		}
	}

	// Update db state if the up-to-date state is different than the db.
	if resultState != dbState {
		p.Log.Info("[%s] Info decision: Inconsistent state between the machine and db document. Updating state to '%s'. Reason: %s", m.Id, resultState, reason)
		err = p.CheckAndUpdateState(m.Id, resultState)
		if err != nil {
			p.Log.Debug("[%s] Info decision: Error while updating the machine state. Err: %v", m.Id, err)
		}

	}

	p.Log.Debug("[%s] Info result: '%s'. Username: %s", m.Id, resultState, m.Username)
	return &protocol.InfoArtifact{
		State: resultState,
	}, nil
}

// CheckAndUpdate state updates only if the given machine id is not used by
// anyone else
func (p *Provider) CheckAndUpdateState(id string, state machinestate.State) error {
	p.Log.Info("[%s] storage state update request to state %v", id, state)
	err := p.Session.Run("jMachines", func(c *mgo.Collection) error {
		return c.Update(
			bson.M{
				"_id": bson.ObjectIdHex(id),
				"assignee.inProgress": false, // only update if it's not locked by someone else
			},
			bson.M{
				"$set": bson.M{
					"status.state":      state.String(),
					"status.modifiedAt": time.Now().UTC(),
				},
			},
		)
	})

	if err == mgo.ErrNotFound {
		p.Log.Warning("[%s] info can't update db state because lock is acquired by someone else", id)
	}

	return err
}
<|endoftext|>


<commit_before>package kodingemail

import (
	"github.com/sendgrid/sendgrid-go"
	"github.com/sendgrid/smtpapi-go"
)

var (
	DefaultFromAddress = "support@koding.com"
	DefaultFromName    = "Koding"
)

type Client interface {
	Send(*sendgrid.SGMail) error
}

type SG struct {
	FromAddress, FromName string
	Client                Client
}

func InitializeSG(username, password string) *SG {
	return &SG{
		FromAddress: DefaultFromAddress, FromName: DefaultFromName,
		Client: sendgrid.NewSendGridClient(username, password),
	}
}

func (s *SG) SendTemplateEmail(to, tId string, sub Options) error {
	message := sendgrid.NewMail()

	// sendgrid api requries not empty value, but I decided to put
	// all the required info in template itself
	message.SetSubject(" ")
	message.SetHTML(" ")

	for key, value := range sub {
		message.AddSubstitution(key, value)
	}

	filter := &smtpapi.Filter{
		Settings: map[string]string{"enabled": "1", "template_id": tId},
	}

	message.SetFilter("templates", filter)
	message.SetFrom(s.FromAddress)
	message.AddTo(to)

	return s.Client.Send(message)
}
<commit_msg>paymentwebhook: fix comments<commit_after>package kodingemail

import (
	"github.com/sendgrid/sendgrid-go"
	"github.com/sendgrid/smtpapi-go"
)

var (
	DefaultFromAddress = "support@koding.com"
	DefaultFromName    = "Koding"
)

type Client interface {
	Send(*sendgrid.SGMail) error
}

type SG struct {
	FromAddress, FromName string
	Client                Client
}

func InitializeSG(username, password string) *SG {
	return &SG{
		FromAddress: DefaultFromAddress, FromName: DefaultFromName,
		Client: sendgrid.NewSendGridClient(username, password),
	}
}

func (s *SG) SendTemplateEmail(to, tId string, sub Options) error {
	message := sendgrid.NewMail()

	// sendgrid api requries non empty values, but I decided to put
	// all the required info in template itself since it's easier to
	// edit in sendgrid ui than to edit in code and redeploy
	message.SetSubject(" ")
	message.SetHTML(" ")

	for key, value := range sub {
		message.AddSubstitution(key, value)
	}

	filter := &smtpapi.Filter{
		Settings: map[string]string{"enabled": "1", "template_id": tId},
	}

	message.SetFilter("templates", filter)
	message.SetFrom(s.FromAddress)
	message.AddTo(to)

	return s.Client.Send(message)
}
<|endoftext|>


<commit_before>// Package script is a library facilitating the creation of programs that resemble
// bash scripts.
package script

import (
	"bufio"
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"sync"
	"syscall"
)

// CommandPath finds the full path of a binary given its name.
func (c *Context) CommandPath(name string) string {
	cmd := exec.Command("which", name)
	cmdOutput, err := cmd.Output()
	if err != nil {
		return ""
	}
	return strings.Trim(string(cmdOutput), "\n")
}

// CommandExists checks if a given binary exists in PATH.
func (c *Context) CommandExists(name string) bool {
	return c.CommandPath(name) != ""
}

// MustCommandExist ensures a given binary exists in PATH, otherwise panics.
func (c *Context) MustCommandExist(name string) {
	if !c.CommandExists(name) {
		panic(fmt.Errorf("Command %s is not available. Please make sure it is installed and accessible. Ouptut of which: %s", name, c.LastOutput()))
	}
}

// ExecuteNormal execute a system command, stdout and stderr are output just
// normally
func (c *Context) ExecuteDebug(name string, args ...string) (err error) {
	err = c.Execute(false, false, name, args...)
	return
}

// ExecuteSilent executes a  system command without outputting stdout (it is
// still captured and can be retrieved using LastOutput())
func (c *Context) ExecuteSilent(name string, args ...string) (err error) {
	err = c.Execute(true, false, name, args...)
	return
}

// ExecuteFullySilent executes a system command without outputting stdout or
// stderr (both are still captured and can be retrieved using LastOutput() and
// LastError())
func (c *Context) ExecuteFullySilent(name string, args ...string) (err error) {
	err = c.Execute(true, true, name, args...)
	return
}

// MustExecute ensures a system command to be executed, otherwise panics
func (c *Context) MustExecuteDebug(name string, args ...string) {
	err := c.Execute(false, false, name, args...)
	if err != nil {
		panic(err)
	}
}

// MustExecuteSilent ensures a system command to be executed without outputting
// stdout, otherwise panics
func (c *Context) MustExecuteSilent(name string, args ...string) {
	err := c.ExecuteSilent(name, args...)
	if err != nil {
		panic(err)
	}
}

// MustExecuteFullySilent ensures a system command to be executed without
// outputting stdout and stderr, otherwise panics
func (c *Context) MustExecuteFullySilent(name string, args ...string) {
	err := c.ExecuteFullySilent(name, args...)
	if err != nil {
		panic(err)
	}
}

// Execute exceutes a system command with configurable stdout and stderr output
// https://github.com/golang/go/issues/9307
func (c *Context) Execute(stdoutSilent bool, stderrSilent bool, name string, args ...string) error {
	cmd := exec.Command(name, args...)

	cmd.Dir = c.workingDir

	// handling Stdout and Stderr
	// idea from http://nathanleclaire.com/blog/2014/12/29/shelled-out-commands-in-golang/
	var wg sync.WaitGroup
	wg.Add(2)

	cmdOutReader, err := cmd.StdoutPipe()
	if err != nil {
		panic(err)
	}

	outScanner := bufio.NewScanner(cmdOutReader)
	c.stdoutBuffer = new(bytes.Buffer)
	go func() {
		defer wg.Done()
		outputHandler(outScanner, !stdoutSilent, c.stdoutBuffer)
	}()

	cmdErrReader, err := cmd.StderrPipe()
	if err != nil {
		panic(err)
	}

	errScanner := bufio.NewScanner(cmdErrReader)
	c.stderrBuffer = new(bytes.Buffer)
	go func() {
		defer wg.Done()
		outputHandler(errScanner, !stderrSilent, c.stderrBuffer)
	}()
	cmd.Start()

	err = cmd.Wait()
	// make sure all output is captured and processed before continuing
	wg.Wait()

	c.lastProcessState = cmd.ProcessState
	return err
}

// internal
func outputHandler(scanner *bufio.Scanner, output bool, buffer *bytes.Buffer) {
	for scanner.Scan() {
		text := scanner.Text()
		if buffer.Len() > 0 {
			buffer.WriteString("\n")
		}
		_, err := buffer.WriteString(text)
		if err != nil {
			panic(err)
		}
		if output {
			fmt.Println(text, buffer.String())
		}
	}
}

// LastOutput returns the output buffer of the last command executed using one
// of the Execute* functions. stdout is captured for any command run.
func (c *Context) LastOutput() string {
	return c.stdoutBuffer.String()
}

// LastError returns the error buffer of the last command executed using one
// of the Execute* functions. stderr is captured for any command run.
func (c *Context) LastError() string {
	return c.stderrBuffer.String()
}

// LastSuccessful determins if the last command run was successful. Success is
// defined as the process' return code being zero.
func (c *Context) LastSuccessful() bool {
	return c.LastExitCode() == 0
}

// LastExitCode returns the exit code of the last command run.
func (c *Context) LastExitCode() int {
	var waitStatus syscall.WaitStatus
	if exitError, ok := c.lastProcessError.(*exec.ExitError); ok {
		waitStatus = exitError.Sys().(syscall.WaitStatus)
	} else {
		waitStatus = c.lastProcessState.Sys().(syscall.WaitStatus)
	}
	fmt.Println(waitStatus)
	return waitStatus.ExitStatus()
}

// LastProcessState returns the *os.ProcessState of the last command run.
func (c *Context) LastProcessState() *os.ProcessState {
	return c.lastProcessState
}

// PrintLastState conveniently prints statistics on the last command run. Useful
// for debugging purposes.
func (c *Context) PrintLastState() {
	state := c.lastProcessState
	fmt.Println(
		"PID:", state.Pid(),
		"Exited:", state.Exited(),
		"Exit Code:", c.LastExitCode(),
		"Success:", state.Success(),
		"User Time:", state.UserTime())
}
<commit_msg>Fix golint warnings<commit_after>// Package script is a library facilitating the creation of programs that resemble
// bash scripts.
package script

import (
	"bufio"
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"sync"
	"syscall"
)

// CommandPath finds the full path of a binary given its name.
func (c *Context) CommandPath(name string) string {
	cmd := exec.Command("which", name)
	cmdOutput, err := cmd.Output()
	if err != nil {
		return ""
	}
	return strings.Trim(string(cmdOutput), "\n")
}

// CommandExists checks if a given binary exists in PATH.
func (c *Context) CommandExists(name string) bool {
	return c.CommandPath(name) != ""
}

// MustCommandExist ensures a given binary exists in PATH, otherwise panics.
func (c *Context) MustCommandExist(name string) {
	if !c.CommandExists(name) {
		panic(fmt.Errorf("Command %s is not available. Please make sure it is installed and accessible. Ouptut of which: %s", name, c.LastOutput()))
	}
}

// ExecuteDebug executes a system command, stdout and stderr are piped
func (c *Context) ExecuteDebug(name string, args ...string) (err error) {
	err = c.Execute(false, false, name, args...)
	return
}

// ExecuteSilent executes a  system command without outputting stdout (it is
// still captured and can be retrieved using LastOutput())
func (c *Context) ExecuteSilent(name string, args ...string) (err error) {
	err = c.Execute(true, false, name, args...)
	return
}

// ExecuteFullySilent executes a system command without outputting stdout or
// stderr (both are still captured and can be retrieved using LastOutput() and
// LastError())
func (c *Context) ExecuteFullySilent(name string, args ...string) (err error) {
	err = c.Execute(true, true, name, args...)
	return
}

// MustExecuteDebug ensures a system command to be executed, otherwise panics
func (c *Context) MustExecuteDebug(name string, args ...string) {
	err := c.Execute(false, false, name, args...)
	if err != nil {
		panic(err)
	}
}

// MustExecuteSilent ensures a system command to be executed without outputting
// stdout, otherwise panics
func (c *Context) MustExecuteSilent(name string, args ...string) {
	err := c.ExecuteSilent(name, args...)
	if err != nil {
		panic(err)
	}
}

// MustExecuteFullySilent ensures a system command to be executed without
// outputting stdout and stderr, otherwise panics
func (c *Context) MustExecuteFullySilent(name string, args ...string) {
	err := c.ExecuteFullySilent(name, args...)
	if err != nil {
		panic(err)
	}
}

// Execute exceutes a system command with configurable stdout and stderr output
// https://github.com/golang/go/issues/9307
func (c *Context) Execute(stdoutSilent bool, stderrSilent bool, name string, args ...string) error {
	cmd := exec.Command(name, args...)

	cmd.Dir = c.workingDir

	// handling Stdout and Stderr
	// idea from http://nathanleclaire.com/blog/2014/12/29/shelled-out-commands-in-golang/
	var wg sync.WaitGroup
	wg.Add(2)

	cmdOutReader, err := cmd.StdoutPipe()
	if err != nil {
		panic(err)
	}

	outScanner := bufio.NewScanner(cmdOutReader)
	c.stdoutBuffer = new(bytes.Buffer)
	go func() {
		defer wg.Done()
		outputHandler(outScanner, !stdoutSilent, c.stdoutBuffer)
	}()

	cmdErrReader, err := cmd.StderrPipe()
	if err != nil {
		panic(err)
	}

	errScanner := bufio.NewScanner(cmdErrReader)
	c.stderrBuffer = new(bytes.Buffer)
	go func() {
		defer wg.Done()
		outputHandler(errScanner, !stderrSilent, c.stderrBuffer)
	}()
	cmd.Start()

	err = cmd.Wait()
	// make sure all output is captured and processed before continuing
	wg.Wait()

	c.lastProcessState = cmd.ProcessState
	return err
}

// internal
func outputHandler(scanner *bufio.Scanner, output bool, buffer *bytes.Buffer) {
	for scanner.Scan() {
		text := scanner.Text()
		if buffer.Len() > 0 {
			buffer.WriteString("\n")
		}
		_, err := buffer.WriteString(text)
		if err != nil {
			panic(err)
		}
		if output {
			fmt.Println(text, buffer.String())
		}
	}
}

// LastOutput returns the output buffer of the last command executed using one
// of the Execute* functions. stdout is captured for any command run.
func (c *Context) LastOutput() string {
	return c.stdoutBuffer.String()
}

// LastError returns the error buffer of the last command executed using one
// of the Execute* functions. stderr is captured for any command run.
func (c *Context) LastError() string {
	return c.stderrBuffer.String()
}

// LastSuccessful determins if the last command run was successful. Success is
// defined as the process' return code being zero.
func (c *Context) LastSuccessful() bool {
	return c.LastExitCode() == 0
}

// LastExitCode returns the exit code of the last command run.
func (c *Context) LastExitCode() int {
	var waitStatus syscall.WaitStatus
	if exitError, ok := c.lastProcessError.(*exec.ExitError); ok {
		waitStatus = exitError.Sys().(syscall.WaitStatus)
	} else {
		waitStatus = c.lastProcessState.Sys().(syscall.WaitStatus)
	}
	fmt.Println(waitStatus)
	return waitStatus.ExitStatus()
}

// LastProcessState returns the *os.ProcessState of the last command run.
func (c *Context) LastProcessState() *os.ProcessState {
	return c.lastProcessState
}

// PrintLastState conveniently prints statistics on the last command run. Useful
// for debugging purposes.
func (c *Context) PrintLastState() {
	state := c.lastProcessState
	fmt.Println(
		"PID:", state.Pid(),
		"Exited:", state.Exited(),
		"Exit Code:", c.LastExitCode(),
		"Success:", state.Success(),
		"User Time:", state.UserTime())
}
<|endoftext|>


<commit_before>package appdeploy

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"path"
	"strings"
	"sync"
	"text/template"

	"gopkg.in/yaml.v1"
)

type Manifest struct {
	Kind     string
	Metadata Metadata
}

func (m Manifest) Filename(folder string) string {
	name := fmt.Sprintf("%s--%s.yaml", strings.ToLower(m.Kind), m.Metadata.Name)
	if folder != "" {
		return path.Join(folder, name)
	} else {
		return name
	}
}

type Metadata struct {
	Name string
}

type ProcessVariables struct {
	Namespace        string
	ImagePullSecrets []string
	ManageCronjobs   bool
	Variables        map[string]interface{}
}

func NewProcessVariables() *ProcessVariables {
	return &ProcessVariables{
		Variables: make(map[string]interface{}),
	}
}

func Process(src ManifestSource, target Target) error {
	return ProcessWithFuncs(src, target, nil)
}

func ProcessWithFuncs(src ManifestSource, target Target, funcs template.FuncMap) error {
	names, err := src.Names()
	if err != nil {
		return err
	}

	vars, err := src.Variables()
	if err != nil {
		return err
	}

	if vars == nil {
		vars = NewProcessVariables()
	}

	// Some variables are special, extract them
	if v, ok := vars.Variables["namespace"]; ok {
		if s, ok := v.(string); ok {
			vars.Namespace = s
		}
	}
	if vars.Namespace == "" {
		vars.Namespace = "default"
	}

	// Prepare the target environment
	err = target.Prepare(vars)
	if err != nil {
		return err
	}

	seen := make([]Manifest, 0)
	var seenMutex sync.Mutex
	wg := sync.WaitGroup{}
	wg.Add(len(names))

	// Apply all resources in parallel
	for _, name := range names {
		n := name
		go func() {
			defer wg.Done()
			m, e := process(src, vars, n, target, funcs)
			if e != nil {
				err = e
			}
			if m != nil {
				seenMutex.Lock()
				seen = append(seen, *m)
				seenMutex.Unlock()
			}
		}()
	}

	wg.Wait()
	if err != nil {
		return err
	}

	return target.Cleanup(seen)
}

func process(src ManifestSource, vars *ProcessVariables, name string, target Target, funcs template.FuncMap) (*Manifest, error) {
	m, err := src.Get(name)
	if err != nil {
		return nil, err
	}
	defer m.Close()

	// Read and parse template
	data, err := ioutil.ReadAll(m)
	if err != nil {
		return nil, err
	}

	tpl, err := template.New(name).Funcs(funcs).Parse(string(data))
	if err != nil {
		return nil, err
	}

	// Execute it
	var buf bytes.Buffer
	err = tpl.Execute(&buf, vars)
	if err != nil {
		return nil, err
	}

	data = bytes.TrimSpace(buf.Bytes())
	if string(data) == "" {
		// Nothing here (entire manifest in a false if-block?
		return nil, nil
	}

	// Determine object type
	var manifest Manifest
	err = yaml.Unmarshal(data, &manifest)
	if err != nil {
		return nil, err
	}

	if manifest.Kind == "" || manifest.Metadata.Name == "" {
		return nil, fmt.Errorf("%s: missing type data, not a valid Kubernetes manifest?", name)
	}

	err = target.Apply(manifest, data)
	if err != nil {
		return nil, err
	}
	return &manifest, nil
}
<commit_msg>Don't wipe a whole namespace<commit_after>package appdeploy

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"path"
	"strings"
	"sync"
	"text/template"

	"gopkg.in/yaml.v1"
)

type Manifest struct {
	Kind     string
	Metadata Metadata
}

func (m Manifest) Filename(folder string) string {
	name := fmt.Sprintf("%s--%s.yaml", strings.ToLower(m.Kind), m.Metadata.Name)
	if folder != "" {
		return path.Join(folder, name)
	} else {
		return name
	}
}

type Metadata struct {
	Name string
}

type ProcessVariables struct {
	Namespace        string
	ImagePullSecrets []string
	ManageCronjobs   bool
	Variables        map[string]interface{}
}

func NewProcessVariables() *ProcessVariables {
	return &ProcessVariables{
		Variables: make(map[string]interface{}),
	}
}

func Process(src ManifestSource, target Target) error {
	return ProcessWithFuncs(src, target, nil)
}

func ProcessWithFuncs(src ManifestSource, target Target, funcs template.FuncMap) error {
	names, err := src.Names()
	if err != nil {
		return err
	}

	if len(names) == 0 {
		return errors.New("Cowardly refusing a source with 0 manifests, this would delete everything")
	}

	vars, err := src.Variables()
	if err != nil {
		return err
	}

	if vars == nil {
		vars = NewProcessVariables()
	}

	// Some variables are special, extract them
	if v, ok := vars.Variables["namespace"]; ok {
		if s, ok := v.(string); ok {
			vars.Namespace = s
		}
	}
	if vars.Namespace == "" {
		vars.Namespace = "default"
	}

	// Prepare the target environment
	err = target.Prepare(vars)
	if err != nil {
		return err
	}

	seen := make([]Manifest, 0)
	var seenMutex sync.Mutex
	wg := sync.WaitGroup{}
	wg.Add(len(names))

	// Apply all resources in parallel
	for _, name := range names {
		n := name
		go func() {
			defer wg.Done()
			m, e := process(src, vars, n, target, funcs)
			if e != nil {
				err = e
			}
			if m != nil {
				seenMutex.Lock()
				seen = append(seen, *m)
				seenMutex.Unlock()
			}
		}()
	}

	wg.Wait()
	if err != nil {
		return err
	}

	return target.Cleanup(seen)
}

func process(src ManifestSource, vars *ProcessVariables, name string, target Target, funcs template.FuncMap) (*Manifest, error) {
	m, err := src.Get(name)
	if err != nil {
		return nil, err
	}
	defer m.Close()

	// Read and parse template
	data, err := ioutil.ReadAll(m)
	if err != nil {
		return nil, err
	}

	tpl, err := template.New(name).Funcs(funcs).Parse(string(data))
	if err != nil {
		return nil, err
	}

	// Execute it
	var buf bytes.Buffer
	err = tpl.Execute(&buf, vars)
	if err != nil {
		return nil, err
	}

	data = bytes.TrimSpace(buf.Bytes())
	if string(data) == "" {
		// Nothing here (entire manifest in a false if-block?
		return nil, nil
	}

	// Determine object type
	var manifest Manifest
	err = yaml.Unmarshal(data, &manifest)
	if err != nil {
		return nil, err
	}

	if manifest.Kind == "" || manifest.Metadata.Name == "" {
		return nil, fmt.Errorf("%s: missing type data, not a valid Kubernetes manifest?", name)
	}

	err = target.Apply(manifest, data)
	if err != nil {
		return nil, err
	}
	return &manifest, nil
}
<|endoftext|>


<commit_before>// Package bamstats provides functions for computing several mapping statistics on a BAM file.
package bamstats

import (
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/guigolab/bamstats/annotation"
	"github.com/guigolab/bamstats/config"
	"github.com/guigolab/bamstats/sam"
	"github.com/guigolab/bamstats/stats"
	"github.com/guigolab/bamstats/utils"
)

func init() {
	log.SetLevel(log.WarnLevel)
}

// var wg sync.WaitGroup

func worker(id int, in interface{}, out chan stats.StatsMap, index *annotation.RtreeMap, cfg *config.Config, wg *sync.WaitGroup) {
	defer wg.Done()
	logger := log.WithFields(log.Fields{
		"worker": id,
	})
	logger.Debug("Starting")

	sm := stats.NewStatsMap(true, (index != nil), cfg.Uniq)

	collectStats(in, sm, index)

	logger.Debug("Done")

	out <- sm
}

func collectStats(in interface{}, sm stats.StatsMap, index *annotation.RtreeMap) {
	switch in.(type) {
	case chan *sam.Record:
		c := in.(chan *sam.Record)
		for record := range c {
			for _, s := range sm {
				s.Collect(record, index)
			}
		}
	case chan *sam.Iterator:
		iterators := in.(chan *sam.Iterator)
		for it := range iterators {
			for it.Next() {
				for _, s := range sm {
					s.Collect(it.Record(), index)
				}
			}
		}
	}
}

func process(bamFile string, index *annotation.RtreeMap, cpu int, maxBuf int, reads int, uniq bool) (stats.StatsMap, error) {

	var wg sync.WaitGroup

	conf := config.NewConfig(cpu, maxBuf, reads, uniq)

	br, err := sam.NewReader(bamFile, conf)
	defer br.Close()
	if err != nil {
		return nil, err
	}
	statChan := make(chan stats.StatsMap, cpu)
	for i := 0; i < br.Workers; i++ {
		id := i + 1
		wg.Add(1)
		go worker(id, br.Channels[i], statChan, index, conf, &wg)
	}

	br.Read()

	go waitProcess(statChan, &wg)
	stat := <-statChan
	stat.Merge(statChan)
	for k, v := range stat {
		v.Finalize()
		if k == "general" {
			s := v.(*stats.GeneralStats)
			s.Reads.Total += br.Unmapped()
		}
	}

	return stat, nil
}

func waitProcess(st chan stats.StatsMap, wg *sync.WaitGroup) {
	wg.Wait()
	close(st)
}

// Process process the input BAM file and collect different mapping stats.
func Process(bamFile string, anno string, cpu int, maxBuf int, reads int, uniq bool) (stats.StatsMap, error) {
	var index *annotation.RtreeMap
	if anno != "" {
		log.Infof("Creating index for %s", anno)
		start := time.Now()
		index = annotation.CreateIndex(anno, bamFile, cpu)
		log.Infof("Index done in %v", time.Since(start))
	}
	start := time.Now()
	log.Infof("Collecting stats for %s", bamFile)
	stats, err := process(bamFile, index, cpu, maxBuf, reads, uniq)
	if err != nil {
		return nil, err
	}
	log.Infof("Stats done in %v", time.Since(start))
	return stats, nil
}

func WriteOutput(output string, st stats.StatsMap) {
	out := utils.NewOutput(output)
	utils.OutputJSON(out, st)
}
<commit_msg>Update processing for Stats refactoring<commit_after>// Package bamstats provides functions for computing several mapping statistics on a BAM file.
package bamstats

import (
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/guigolab/bamstats/annotation"
	"github.com/guigolab/bamstats/config"
	"github.com/guigolab/bamstats/sam"
	"github.com/guigolab/bamstats/stats"
	"github.com/guigolab/bamstats/utils"
)

func init() {
	log.SetLevel(log.WarnLevel)
}

// var wg sync.WaitGroup

func worker(id int, in interface{}, out chan stats.Map, index *annotation.RtreeMap, cfg *config.Config, wg *sync.WaitGroup) {
	defer wg.Done()
	logger := log.WithFields(log.Fields{
		"worker": id,
	})
	logger.Debug("Starting")

	sm := makeStatsMap(index, cfg)

	collectStats(in, sm)

	logger.Debug("Done")

	out <- sm
}

func collectStats(in interface{}, sm stats.Map) {
	switch in.(type) {
	case chan *sam.Record:
		c := in.(chan *sam.Record)
		for record := range c {
			for _, s := range sm {
				s.Collect(record)
			}
		}
	case chan *sam.Iterator:
		iterators := in.(chan *sam.Iterator)
		for it := range iterators {
			for it.Next() {
				for _, s := range sm {
					s.Collect(it.Record())
				}
			}
		}
	}
}

func process(bamFile string, index *annotation.RtreeMap, cpu int, maxBuf int, reads int, uniq bool) (stats.Map, error) {

	var wg sync.WaitGroup

	conf := config.NewConfig(cpu, maxBuf, reads, uniq)

	br, err := sam.NewReader(bamFile, conf)
	defer br.Close()
	if err != nil {
		return nil, err
	}
	statChan := make(chan stats.Map, cpu)
	for i := 0; i < br.Workers; i++ {
		id := i + 1
		wg.Add(1)
		go worker(id, br.Channels[i], statChan, index, conf, &wg)
	}

	br.Read()

	go waitProcess(statChan, &wg)
	stat := <-statChan
	stat.Merge(statChan)
	for k, v := range stat {
		v.Finalize()
		if k == "general" {
			s := v.(*stats.GeneralStats)
			s.Reads.Total += br.Unmapped()
		}
	}

	return stat, nil
}

func waitProcess(st chan stats.Map, wg *sync.WaitGroup) {
	wg.Wait()
	close(st)
}

// Process process the input BAM file and collect different mapping stats.
func Process(bamFile string, anno string, cpu int, maxBuf int, reads int, uniq bool) (stats.Map, error) {
	var index *annotation.RtreeMap
	if anno != "" {
		log.Infof("Creating index for %s", anno)
		start := time.Now()
		index = annotation.CreateIndex(anno, bamFile, cpu)
		log.Infof("Index done in %v", time.Since(start))
	}
	start := time.Now()
	log.Infof("Collecting stats for %s", bamFile)
	stats, err := process(bamFile, index, cpu, maxBuf, reads, uniq)
	if err != nil {
		return nil, err
	}
	log.Infof("Stats done in %v", time.Since(start))
	return stats, nil
}

func writeOutput(output string, st stats.Map) {
	out := utils.NewOutput(output)
	utils.OutputJSON(out, st)
}

func makeStatsMap(index *annotation.RtreeMap, cfg *config.Config) stats.Map {
	m := make(stats.Map)
	m.Add("general", stats.NewGeneralStats())
	if index != nil {
		m.Add("coverage", stats.NewCoverageStats(index, false))
		if cfg.Uniq {
			m.Add("coverageUniq", stats.NewCoverageStats(index, true))
		}
	}
	return m
}
<|endoftext|>