<commit_before>// Copyright © 2016-2019 Wei Shen <shenwei356@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"runtime"
	"syscall"

	// "runtime/debug"

	"github.com/shenwei356/bio/seq"
	"github.com/shenwei356/bio/seqio/fastx"
	"github.com/shenwei356/util/byteutil"
	"github.com/spf13/cobra"
)

// seqCmd represents the seq command
var seqCmd = &cobra.Command{
	Use:   "seq",
	Short: "transform sequences (revserse, complement, extract ID...)",
	Long: `transform sequences (revserse, complement, extract ID...)

`,
	Run: func(cmd *cobra.Command, args []string) {
		config := getConfigs(cmd)
		alphabet := config.Alphabet
		idRegexp := config.IDRegexp
		lineWidth := config.LineWidth
		outFile := config.OutFile
		quiet := config.Quiet
		seq.AlphabetGuessSeqLengthThreshold = config.AlphabetGuessSeqLength
		runtime.GOMAXPROCS(config.Threads)

		reverse := getFlagBool(cmd, "reverse")
		complement := getFlagBool(cmd, "complement")
		onlyName := getFlagBool(cmd, "name")
		onlySeq := getFlagBool(cmd, "seq")
		onlyQual := getFlagBool(cmd, "qual")
		onlyID := getFlagBool(cmd, "only-id")
		removeGaps := getFlagBool(cmd, "remove-gaps")
		gapLetters := getFlagString(cmd, "gap-letters")
		lowerCase := getFlagBool(cmd, "lower-case")
		upperCase := getFlagBool(cmd, "upper-case")
		dna2rna := getFlagBool(cmd, "dna2rna")
		rna2dna := getFlagBool(cmd, "rna2dna")
		color := getFlagBool(cmd, "color")
		validateSeq := getFlagBool(cmd, "validate-seq")
		validateSeqLength := getFlagValidateSeqLength(cmd, "validate-seq-length")
		minLen := getFlagInt(cmd, "min-len")
		maxLen := getFlagInt(cmd, "max-len")
		qBase := getFlagPositiveInt(cmd, "qual-ascii-base")
		minQual := getFlagFloat64(cmd, "min-qual")
		maxQual := getFlagFloat64(cmd, "max-qual")

		if gapLetters == "" {
			checkError(fmt.Errorf("value of flag -G (--gap-letters) should not be empty"))
		}
		for _, c := range gapLetters {
			if c > 127 {
				checkError(fmt.Errorf("value of -G (--gap-letters) contains non-ASCII characters"))
			}
		}

		if minLen >= 0 && maxLen >= 0 && minLen > maxLen {
			checkError(fmt.Errorf("value of flag -m (--min-len) should be >= value of flag -M (--max-len)"))
		}
		if minQual >= 0 && maxQual >= 0 && minQual > maxQual {
			checkError(fmt.Errorf("value of flag -Q (--min-qual) should be <= value of flag -R (--max-qual)"))
		}
		// if minLen >= 0 || maxLen >= 0 {
		// 	removeGaps = true
		// 	if !quiet {
		// 		log.Infof("flag -g (--remove-gaps) is switched on when using -m (--min-len) or -M (--max-len)")
		// 	}
		// }
		if (minLen >= 0 || maxLen >= 0) && !removeGaps {
			log.Warning("you may switch on flag -g/--remove-gaps to remove spaces")
		}

		seq.ValidateSeq = validateSeq
		seq.ValidateWholeSeq = false
		seq.ValidSeqLengthThreshold = validateSeqLength
		seq.ValidSeqThreads = config.Threads
		seq.ComplementThreads = config.Threads

		if complement && (alphabet == nil || alphabet == seq.Protein) {
			log.Warningf("flag -t (--seq-type) (DNA/RNA) is recommended for computing complement sequences")
		}

		if !validateSeq && !(alphabet == nil || alphabet == seq.Unlimit) {
			if !quiet {
				log.Info("when flag -t (--seq-type) given, flag -v (--validate-seq) is automatically switched on")
			}
			validateSeq = true
			seq.ValidateSeq = true
		}

		if lowerCase && upperCase {
			checkError(fmt.Errorf("could not give both flags -l (--lower-case) and -u (--upper-case)"))
		}

		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)

		var seqCol *SeqColorizer
		if color {
			switch alphabet {
			case seq.DNA, seq.DNAredundant, seq.RNA, seq.RNAredundant:
				seqCol = NewSeqColorizer("nucleic")
			case seq.Protein:
				seqCol = NewSeqColorizer("amino")
			default:
				seqCol = NewSeqColorizer("nucleic")
			}
		}
		var outfh *os.File
		var err error
		if outFile == "-" {
			outfh = os.Stdout
		} else {
			outfh, err = os.Open(outFile)
			checkError(err)
		}
		defer outfh.Close()
		var outbw io.Writer
		outbw = outfh
		if color {
			outbw = seqCol.WrapWriter(outfh)
		}

		var checkSeqType bool
		var isFastq bool
		var printName, printSeq, printQual bool
		var head []byte
		var sequence *seq.Seq
		var text []byte
		var b *bytes.Buffer
		var record *fastx.Record
		var fastxReader *fastx.Reader

		for _, file := range files {
			fastxReader, err = fastx.NewReader(alphabet, file, idRegexp)
			checkError(err)

			checkSeqType = true
			printQual = false
			once := true
			if onlySeq || onlyQual {
				config.LineWidth = 0
			}
			for {
				record, err = fastxReader.Read()
				if err != nil {
					if err == io.EOF {
						break
					}
					checkError(err)
					break
				}

				if checkSeqType {
					isFastq = fastxReader.IsFastq
					if isFastq {
						config.LineWidth = 0
						printQual = true
					}
					checkSeqType = false
				}

				if removeGaps {
					record.Seq.RemoveGapsInplace(gapLetters)
				}

				if minLen >= 0 && len(record.Seq.Seq) < minLen {
					continue
				}

				if maxLen >= 0 && len(record.Seq.Seq) > maxLen {
					continue
				}

				if minQual > 0 || maxQual > 0 {
					avgQual := record.Seq.AvgQual(qBase)
					if minQual > 0 && avgQual < minQual {
						continue
					}
					if maxQual > 0 && avgQual >= maxQual {
						continue
					}
				}

				printName, printSeq = true, true
				if onlyName && onlySeq {
					printName, printSeq = true, true
				} else if onlyName {
					printName, printSeq, printQual = true, false, false
				} else if onlySeq {
					printName, printSeq, printQual = false, true, false
				} else if onlyQual {
					if !isFastq {
						checkError(fmt.Errorf("FASTA format has no quality. So do not just use flag -q (--qual)"))
					}
					printName, printSeq, printQual = false, false, true
				}
				if printName {
					if onlyID {
						head = record.ID
					} else {
						head = record.Name
					}

					if printSeq {
						if isFastq {
							outbw.Write([]byte("@"))
							outbw.Write(head)
							outbw.Write([]byte("\n"))
						} else {
							outbw.Write([]byte(">"))
							outbw.Write(head)
							outbw.Write([]byte("\n"))
						}
					} else {
						outbw.Write(head)
						outbw.Write([]byte("\n"))
					}
				}

				sequence = record.Seq
				if reverse {
					sequence = sequence.ReverseInplace()
				}
				if complement {
					if !config.Quiet && record.Seq.Alphabet == seq.Protein || record.Seq.Alphabet == seq.Unlimit {
						log.Warning("complement does no take effect on protein/unlimit sequence")
					}
					sequence = sequence.ComplementInplace()
				}

				if printSeq {
					if dna2rna {
						ab := fastxReader.Alphabet()
						if ab == seq.RNA || ab == seq.RNAredundant {
							if once {
								log.Warningf("it's already RNA, no need to convert")
								once = false
							}
						} else {
							for i, b := range sequence.Seq {
								switch b {
								case 't':
									sequence.Seq[i] = 'u'
								case 'T':
									sequence.Seq[i] = 'U'
								}
							}
						}
					}
					if rna2dna {
						ab := fastxReader.Alphabet()
						if ab == seq.DNA || ab == seq.DNAredundant {
							if once {
								log.Warningf("it's already DNA, no need to convert")
								once = false
							}
						} else {
							for i, b := range sequence.Seq {
								switch b {
								case 'u':
									sequence.Seq[i] = 't'
								case 'U':
									sequence.Seq[i] = 'T'
								}
							}
						}
					}
					if lowerCase {
						sequence.Seq = bytes.ToLower(sequence.Seq)
					} else if upperCase {
						sequence.Seq = bytes.ToUpper(sequence.Seq)
					}

					if len(sequence.Seq) <= pageSize {
						text := byteutil.WrapByteSlice(sequence.Seq, config.LineWidth)
						if color {
							if sequence.Qual != nil {
								text = seqCol.ColorWithQuals(text, sequence.Qual)
							} else {
								text = seqCol.Color(text)
							}
						}
						outbw.Write(text)
					} else {
						if bufferedByteSliceWrapper == nil {
							bufferedByteSliceWrapper = byteutil.NewBufferedByteSliceWrapper2(1, len(sequence.Seq), config.LineWidth)
						}
						text, b = bufferedByteSliceWrapper.Wrap(sequence.Seq, config.LineWidth)
						if color {
							text = seqCol.Color(text)
						}
						outbw.Write(text)
						bufferedByteSliceWrapper.Recycle(b)
					}

					outbw.Write([]byte("\n"))
				}

				if printQual {
					if !onlyQual {
						outbw.Write([]byte("+\n"))
					}

					if len(sequence.Qual) <= pageSize {
						if color {
							outbw.Write(byteutil.WrapByteSlice(seqCol.ColorQuals(sequence.Qual), config.LineWidth))
						} else {
							outbw.Write(byteutil.WrapByteSlice(sequence.Qual, config.LineWidth))
						}
					} else {
						if bufferedByteSliceWrapper == nil {
							bufferedByteSliceWrapper = byteutil.NewBufferedByteSliceWrapper2(1, len(sequence.Qual), config.LineWidth)
						}
						text, b = bufferedByteSliceWrapper.Wrap(sequence.Qual, config.LineWidth)
						if color {
							text = seqCol.ColorQuals(text)
						}
						outbw.Write(text)
						bufferedByteSliceWrapper.Recycle(b)
					}

					outbw.Write([]byte("\n"))
				}
			}

			config.LineWidth = lineWidth
		}

		outfh.Close()
	},
}

var pageSize = syscall.Getpagesize()

func init() {
	RootCmd.AddCommand(seqCmd)

	seqCmd.Flags().BoolP("reverse", "r", false, "reverse sequence")
	seqCmd.Flags().BoolP("complement", "p", false, "complement sequence, flag '-v' is recommended to switch on")
	seqCmd.Flags().BoolP("name", "n", false, "only print names")
	seqCmd.Flags().BoolP("seq", "s", false, "only print sequences")
	seqCmd.Flags().BoolP("qual", "q", false, "only print qualities")
	seqCmd.Flags().BoolP("only-id", "i", false, "print ID instead of full head")
	seqCmd.Flags().BoolP("remove-gaps", "g", false, "remove gaps")
	seqCmd.Flags().StringP("gap-letters", "G", "- 	.", "gap letters")
	seqCmd.Flags().BoolP("lower-case", "l", false, "print sequences in lower case")
	seqCmd.Flags().BoolP("upper-case", "u", false, "print sequences in upper case")
	seqCmd.Flags().BoolP("dna2rna", "", false, "DNA to RNA")
	seqCmd.Flags().BoolP("rna2dna", "", false, "RNA to DNA")
	seqCmd.Flags().BoolP("color", "k", false, "colorize sequences - to be piped into \"less -R\"")
	seqCmd.Flags().BoolP("validate-seq", "v", false, "validate bases according to the alphabet")
	seqCmd.Flags().IntP("validate-seq-length", "V", 10000, "length of sequence to validate (0 for whole seq)")
	seqCmd.Flags().IntP("min-len", "m", -1, "only print sequences longer than the minimum length (-1 for no limit)")
	seqCmd.Flags().IntP("max-len", "M", -1, "only print sequences shorter than the maximum length (-1 for no limit)")
	seqCmd.Flags().IntP("qual-ascii-base", "b", 33, "ASCII BASE, 33 for Phred+33")
	seqCmd.Flags().Float64P("min-qual", "Q", -1, "only print sequences with average quality qreater or equal than this limit (-1 for no limit)")
	seqCmd.Flags().Float64P("max-qual", "R", -1, "only print sequences with average quality less than this limit (-1 for no limit)")
}
<commit_msg>seqkit seq: fix bug when using -o/--out-file. fix #155<commit_after>// Copyright © 2016-2019 Wei Shen <shenwei356@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"runtime"
	"syscall"

	// "runtime/debug"

	"github.com/shenwei356/bio/seq"
	"github.com/shenwei356/bio/seqio/fastx"
	"github.com/shenwei356/util/byteutil"
	"github.com/spf13/cobra"
)

// seqCmd represents the seq command
var seqCmd = &cobra.Command{
	Use:   "seq",
	Short: "transform sequences (revserse, complement, extract ID...)",
	Long: `transform sequences (revserse, complement, extract ID...)

`,
	Run: func(cmd *cobra.Command, args []string) {
		config := getConfigs(cmd)
		alphabet := config.Alphabet
		idRegexp := config.IDRegexp
		lineWidth := config.LineWidth
		outFile := config.OutFile
		quiet := config.Quiet
		seq.AlphabetGuessSeqLengthThreshold = config.AlphabetGuessSeqLength
		runtime.GOMAXPROCS(config.Threads)

		reverse := getFlagBool(cmd, "reverse")
		complement := getFlagBool(cmd, "complement")
		onlyName := getFlagBool(cmd, "name")
		onlySeq := getFlagBool(cmd, "seq")
		onlyQual := getFlagBool(cmd, "qual")
		onlyID := getFlagBool(cmd, "only-id")
		removeGaps := getFlagBool(cmd, "remove-gaps")
		gapLetters := getFlagString(cmd, "gap-letters")
		lowerCase := getFlagBool(cmd, "lower-case")
		upperCase := getFlagBool(cmd, "upper-case")
		dna2rna := getFlagBool(cmd, "dna2rna")
		rna2dna := getFlagBool(cmd, "rna2dna")
		color := getFlagBool(cmd, "color")
		validateSeq := getFlagBool(cmd, "validate-seq")
		validateSeqLength := getFlagValidateSeqLength(cmd, "validate-seq-length")
		minLen := getFlagInt(cmd, "min-len")
		maxLen := getFlagInt(cmd, "max-len")
		qBase := getFlagPositiveInt(cmd, "qual-ascii-base")
		minQual := getFlagFloat64(cmd, "min-qual")
		maxQual := getFlagFloat64(cmd, "max-qual")

		if gapLetters == "" {
			checkError(fmt.Errorf("value of flag -G (--gap-letters) should not be empty"))
		}
		for _, c := range gapLetters {
			if c > 127 {
				checkError(fmt.Errorf("value of -G (--gap-letters) contains non-ASCII characters"))
			}
		}

		if minLen >= 0 && maxLen >= 0 && minLen > maxLen {
			checkError(fmt.Errorf("value of flag -m (--min-len) should be >= value of flag -M (--max-len)"))
		}
		if minQual >= 0 && maxQual >= 0 && minQual > maxQual {
			checkError(fmt.Errorf("value of flag -Q (--min-qual) should be <= value of flag -R (--max-qual)"))
		}
		// if minLen >= 0 || maxLen >= 0 {
		// 	removeGaps = true
		// 	if !quiet {
		// 		log.Infof("flag -g (--remove-gaps) is switched on when using -m (--min-len) or -M (--max-len)")
		// 	}
		// }
		if (minLen >= 0 || maxLen >= 0) && !removeGaps {
			log.Warning("you may switch on flag -g/--remove-gaps to remove spaces")
		}

		seq.ValidateSeq = validateSeq
		seq.ValidateWholeSeq = false
		seq.ValidSeqLengthThreshold = validateSeqLength
		seq.ValidSeqThreads = config.Threads
		seq.ComplementThreads = config.Threads

		if complement && (alphabet == nil || alphabet == seq.Protein) {
			log.Warningf("flag -t (--seq-type) (DNA/RNA) is recommended for computing complement sequences")
		}

		if !validateSeq && !(alphabet == nil || alphabet == seq.Unlimit) {
			if !quiet {
				log.Info("when flag -t (--seq-type) given, flag -v (--validate-seq) is automatically switched on")
			}
			validateSeq = true
			seq.ValidateSeq = true
		}

		if lowerCase && upperCase {
			checkError(fmt.Errorf("could not give both flags -l (--lower-case) and -u (--upper-case)"))
		}

		files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true)

		var seqCol *SeqColorizer
		if color {
			switch alphabet {
			case seq.DNA, seq.DNAredundant, seq.RNA, seq.RNAredundant:
				seqCol = NewSeqColorizer("nucleic")
			case seq.Protein:
				seqCol = NewSeqColorizer("amino")
			default:
				seqCol = NewSeqColorizer("nucleic")
			}
		}
		var outfh *os.File
		var err error
		if outFile == "-" {
			outfh = os.Stdout
		} else {
			outfh, err = os.Create(outFile)
			checkError(err)
		}
		defer outfh.Close()
		var outbw io.Writer
		outbw = outfh
		if color {
			outbw = seqCol.WrapWriter(outfh)
		}

		var checkSeqType bool
		var isFastq bool
		var printName, printSeq, printQual bool
		var head []byte
		var sequence *seq.Seq
		var text []byte
		var b *bytes.Buffer
		var record *fastx.Record
		var fastxReader *fastx.Reader

		for _, file := range files {
			fastxReader, err = fastx.NewReader(alphabet, file, idRegexp)
			checkError(err)

			checkSeqType = true
			printQual = false
			once := true
			if onlySeq || onlyQual {
				config.LineWidth = 0
			}
			for {
				record, err = fastxReader.Read()
				if err != nil {
					if err == io.EOF {
						break
					}
					checkError(err)
					break
				}

				if checkSeqType {
					isFastq = fastxReader.IsFastq
					if isFastq {
						config.LineWidth = 0
						printQual = true
					}
					checkSeqType = false
				}

				if removeGaps {
					record.Seq.RemoveGapsInplace(gapLetters)
				}

				if minLen >= 0 && len(record.Seq.Seq) < minLen {
					continue
				}

				if maxLen >= 0 && len(record.Seq.Seq) > maxLen {
					continue
				}

				if minQual > 0 || maxQual > 0 {
					avgQual := record.Seq.AvgQual(qBase)
					if minQual > 0 && avgQual < minQual {
						continue
					}
					if maxQual > 0 && avgQual >= maxQual {
						continue
					}
				}

				printName, printSeq = true, true
				if onlyName && onlySeq {
					printName, printSeq = true, true
				} else if onlyName {
					printName, printSeq, printQual = true, false, false
				} else if onlySeq {
					printName, printSeq, printQual = false, true, false
				} else if onlyQual {
					if !isFastq {
						checkError(fmt.Errorf("FASTA format has no quality. So do not just use flag -q (--qual)"))
					}
					printName, printSeq, printQual = false, false, true
				}
				if printName {
					if onlyID {
						head = record.ID
					} else {
						head = record.Name
					}

					if printSeq {
						if isFastq {
							outbw.Write([]byte("@"))
							outbw.Write(head)
							outbw.Write([]byte("\n"))
						} else {
							outbw.Write([]byte(">"))
							outbw.Write(head)
							outbw.Write([]byte("\n"))
						}
					} else {
						outbw.Write(head)
						outbw.Write([]byte("\n"))
					}
				}

				sequence = record.Seq
				if reverse {
					sequence = sequence.ReverseInplace()
				}
				if complement {
					if !config.Quiet && record.Seq.Alphabet == seq.Protein || record.Seq.Alphabet == seq.Unlimit {
						log.Warning("complement does no take effect on protein/unlimit sequence")
					}
					sequence = sequence.ComplementInplace()
				}

				if printSeq {
					if dna2rna {
						ab := fastxReader.Alphabet()
						if ab == seq.RNA || ab == seq.RNAredundant {
							if once {
								log.Warningf("it's already RNA, no need to convert")
								once = false
							}
						} else {
							for i, b := range sequence.Seq {
								switch b {
								case 't':
									sequence.Seq[i] = 'u'
								case 'T':
									sequence.Seq[i] = 'U'
								}
							}
						}
					}
					if rna2dna {
						ab := fastxReader.Alphabet()
						if ab == seq.DNA || ab == seq.DNAredundant {
							if once {
								log.Warningf("it's already DNA, no need to convert")
								once = false
							}
						} else {
							for i, b := range sequence.Seq {
								switch b {
								case 'u':
									sequence.Seq[i] = 't'
								case 'U':
									sequence.Seq[i] = 'T'
								}
							}
						}
					}
					if lowerCase {
						sequence.Seq = bytes.ToLower(sequence.Seq)
					} else if upperCase {
						sequence.Seq = bytes.ToUpper(sequence.Seq)
					}

					if len(sequence.Seq) <= pageSize {
						text := byteutil.WrapByteSlice(sequence.Seq, config.LineWidth)
						if color {
							if sequence.Qual != nil {
								text = seqCol.ColorWithQuals(text, sequence.Qual)
							} else {
								text = seqCol.Color(text)
							}
						}
						outbw.Write(text)
					} else {
						if bufferedByteSliceWrapper == nil {
							bufferedByteSliceWrapper = byteutil.NewBufferedByteSliceWrapper2(1, len(sequence.Seq), config.LineWidth)
						}
						text, b = bufferedByteSliceWrapper.Wrap(sequence.Seq, config.LineWidth)
						if color {
							text = seqCol.Color(text)
						}
						outbw.Write(text)
						bufferedByteSliceWrapper.Recycle(b)
					}

					outbw.Write([]byte("\n"))
				}

				if printQual {
					if !onlyQual {
						outbw.Write([]byte("+\n"))
					}

					if len(sequence.Qual) <= pageSize {
						if color {
							outbw.Write(byteutil.WrapByteSlice(seqCol.ColorQuals(sequence.Qual), config.LineWidth))
						} else {
							outbw.Write(byteutil.WrapByteSlice(sequence.Qual, config.LineWidth))
						}
					} else {
						if bufferedByteSliceWrapper == nil {
							bufferedByteSliceWrapper = byteutil.NewBufferedByteSliceWrapper2(1, len(sequence.Qual), config.LineWidth)
						}
						text, b = bufferedByteSliceWrapper.Wrap(sequence.Qual, config.LineWidth)
						if color {
							text = seqCol.ColorQuals(text)
						}
						outbw.Write(text)
						bufferedByteSliceWrapper.Recycle(b)
					}

					outbw.Write([]byte("\n"))
				}
			}

			config.LineWidth = lineWidth
		}

		outfh.Close()
	},
}

var pageSize = syscall.Getpagesize()

func init() {
	RootCmd.AddCommand(seqCmd)

	seqCmd.Flags().BoolP("reverse", "r", false, "reverse sequence")
	seqCmd.Flags().BoolP("complement", "p", false, "complement sequence, flag '-v' is recommended to switch on")
	seqCmd.Flags().BoolP("name", "n", false, "only print names")
	seqCmd.Flags().BoolP("seq", "s", false, "only print sequences")
	seqCmd.Flags().BoolP("qual", "q", false, "only print qualities")
	seqCmd.Flags().BoolP("only-id", "i", false, "print ID instead of full head")
	seqCmd.Flags().BoolP("remove-gaps", "g", false, "remove gaps")
	seqCmd.Flags().StringP("gap-letters", "G", "- 	.", "gap letters")
	seqCmd.Flags().BoolP("lower-case", "l", false, "print sequences in lower case")
	seqCmd.Flags().BoolP("upper-case", "u", false, "print sequences in upper case")
	seqCmd.Flags().BoolP("dna2rna", "", false, "DNA to RNA")
	seqCmd.Flags().BoolP("rna2dna", "", false, "RNA to DNA")
	seqCmd.Flags().BoolP("color", "k", false, "colorize sequences - to be piped into \"less -R\"")
	seqCmd.Flags().BoolP("validate-seq", "v", false, "validate bases according to the alphabet")
	seqCmd.Flags().IntP("validate-seq-length", "V", 10000, "length of sequence to validate (0 for whole seq)")
	seqCmd.Flags().IntP("min-len", "m", -1, "only print sequences longer than the minimum length (-1 for no limit)")
	seqCmd.Flags().IntP("max-len", "M", -1, "only print sequences shorter than the maximum length (-1 for no limit)")
	seqCmd.Flags().IntP("qual-ascii-base", "b", 33, "ASCII BASE, 33 for Phred+33")
	seqCmd.Flags().Float64P("min-qual", "Q", -1, "only print sequences with average quality qreater or equal than this limit (-1 for no limit)")
	seqCmd.Flags().Float64P("max-qual", "R", -1, "only print sequences with average quality less than this limit (-1 for no limit)")
}
<|endoftext|>


<commit_before>// +build windows

package serial

import (
	"fmt"
	"io"
	"os"
	"sync"
	"syscall"
	"unsafe"
	"time"
)

type serialPort struct {
	f  *os.File
	fd syscall.Handle
	rl sync.Mutex
	wl sync.Mutex
	st structTimeouts
	h uintptr
	ro *syscall.Overlapped
	wo *syscall.Overlapped
}

type structDCB struct {
	DCBlength, BaudRate                            uint32
	flags                                          [4]byte
	wReserved, XonLim, XoffLim                     uint16
	ByteSize, Parity, StopBits                     byte
	XonChar, XoffChar, ErrorChar, EofChar, EvtChar byte
	wReserved1                                     uint16
}

type structTimeouts struct {
	ReadIntervalTimeout         uint32
	ReadTotalTimeoutMultiplier  uint32
	ReadTotalTimeoutConstant    uint32
	WriteTotalTimeoutMultiplier uint32
	WriteTotalTimeoutConstant   uint32
}

func openPort(name string, baud int) (rwc io.ReadWriteCloser, err error) {
	if len(name) > 0 && name[0] != '\\' {
		name = "\\\\.\\" + name
	}

	h, err := syscall.CreateFile(syscall.StringToUTF16Ptr(name),
		syscall.GENERIC_READ|syscall.GENERIC_WRITE,
		0,
		nil,
		syscall.OPEN_EXISTING,
		syscall.FILE_ATTRIBUTE_NORMAL|syscall.FILE_FLAG_OVERLAPPED,
		0)
	if err != nil {
		return nil, err
	}
	f := os.NewFile(uintptr(h), name)
	defer func() {
		if err != nil {
			f.Close()
		}
	}()

	if err = setCommState(h, baud); err != nil {
		return
	}
	if err = setupComm(h, 64, 64); err != nil {
		return
	}
	if err = setCommTimeouts(h); err != nil {
		return
	}
	if err = setCommMask(h); err != nil {
		return
	}

	ro, err := newOverlapped()
	if err != nil {
		return
	}
	wo, err := newOverlapped()
	if err != nil {
		return
	}
	port := new(serialPort)
	port.f = f
	port.fd = h
	port.ro = ro
	port.wo = wo

	return port, nil
}

func (p *serialPort) Close() error {
	return p.f.Close()
}
func (p *serialPort) SetTimeout(m Millisecond){

}
func (p *serialPort) Write(buf []byte) (int, error) {
	p.wl.Lock()
	defer p.wl.Unlock()

	if err := resetEvent(p.wo.HEvent); err != nil {
		return 0, err
	}
	var n uint32
	err := syscall.WriteFile(p.fd, buf, &n, p.wo)
	if err != nil && err != syscall.ERROR_IO_PENDING {
		return int(n), err
	}
	return getOverlappedResult(p.fd, p.wo)
}

func (p *serialPort) Read(buf []byte) (int, error) {
	if p == nil || p.f == nil {
		return 0, fmt.Errorf("Invalid port on read %v %v", p, p.f)
	}

	p.rl.Lock()
	defer p.rl.Unlock()

	if err := resetEvent(p.ro.HEvent); err != nil {
		return 0, err
	}
	var done uint32
	err := syscall.ReadFile(p.fd, buf, &done, p.ro)
	if err != nil && err != syscall.ERROR_IO_PENDING {
		return int(done), err
	}
	return getOverlappedResult(p.fd, p.ro)
}

var (
	nSetCommState,
	nSetCommTimeouts,
	nSetCommMask,
	nSetupComm,
	nGetOverlappedResult,
	nCreateEvent,
	nResetEvent uintptr
)

func init() {
	k32, err := syscall.LoadLibrary("kernel32.dll")
	if err != nil {
		panic("LoadLibrary " + err.Error())
	}
	defer syscall.FreeLibrary(k32)

	nSetCommState = getProcAddr(k32, "SetCommState")
	nSetCommTimeouts = getProcAddr(k32, "SetCommTimeouts")
	nSetCommMask = getProcAddr(k32, "SetCommMask")
	nSetupComm = getProcAddr(k32, "SetupComm")
	nGetOverlappedResult = getProcAddr(k32, "GetOverlappedResult")
	nCreateEvent = getProcAddr(k32, "CreateEventW")
	nResetEvent = getProcAddr(k32, "ResetEvent")
}

func getProcAddr(lib syscall.Handle, name string) uintptr {
	addr, err := syscall.GetProcAddress(lib, name)
	if err != nil {
		panic(name + " " + err.Error())
	}
	return addr
}

func setCommState(h syscall.Handle, baud int) error {
	var params structDCB
	params.DCBlength = uint32(unsafe.Sizeof(params))

	params.flags[0] = 0x01  // fBinary
	params.flags[0] |= 0x10 // Assert DSR

	params.BaudRate = uint32(baud)
	params.ByteSize = 8

	r, _, err := syscall.Syscall(nSetCommState, 2, uintptr(h), uintptr(unsafe.Pointer(&params)), 0)
	if r == 0 {
		return err
	}
	return nil
}

func setCommTimeouts(h syscall.Handle) error {
	var timeouts structTimeouts
	const MAXDWORD = 1<<32 - 1
	timeouts.ReadIntervalTimeout = MAXDWORD
	timeouts.ReadTotalTimeoutMultiplier = MAXDWORD
	timeouts.ReadTotalTimeoutConstant = MAXDWORD - 1

	/* From http://msdn.microsoft.com/en-us/library/aa363190(v=VS.85).aspx

		 For blocking I/O see below:

		 Remarks:

		 If an application sets ReadIntervalTimeout and
		 ReadTotalTimeoutMultiplier to MAXDWORD and sets
		 ReadTotalTimeoutConstant to a value greater than zero and
		 less than MAXDWORD, one of the following occurs when the
		 ReadFile function is called:

		 If there are any bytes in the input buffer, ReadFile returns
		       immediately with the bytes in the buffer.

		 If there are no bytes in the input buffer, ReadFile waits
	               until a byte arrives and then returns immediately.

		 If no bytes arrive within the time specified by
		       ReadTotalTimeoutConstant, ReadFile times out.
	*/

	r, _, err := syscall.Syscall(nSetCommTimeouts, 2, uintptr(h), uintptr(unsafe.Pointer(&timeouts)), 0)
	if r == 0 {
		return err
	}
	return nil
}

func setupComm(h syscall.Handle, in, out int) error {
	r, _, err := syscall.Syscall(nSetupComm, 3, uintptr(h), uintptr(in), uintptr(out))
	if r == 0 {
		return err
	}
	return nil
}

func setCommMask(h syscall.Handle) error {
	const EV_RXCHAR = 0x0001
	r, _, err := syscall.Syscall(nSetCommMask, 2, uintptr(h), EV_RXCHAR, 0)
	if r == 0 {
		return err
	}
	return nil
}

func resetEvent(h syscall.Handle) error {
	r, _, err := syscall.Syscall(nResetEvent, 1, uintptr(h), 0, 0)
	if r == 0 {
		return err
	}
	return nil
}

func newOverlapped() (*syscall.Overlapped, error) {
	var overlapped syscall.Overlapped
	r, _, err := syscall.Syscall6(nCreateEvent, 4, 0, 1, 0, 0, 0, 0)
	if r == 0 {
		return nil, err
	}
	overlapped.HEvent = syscall.Handle(r)
	return &overlapped, nil
}

func getOverlappedResult(h syscall.Handle, overlapped *syscall.Overlapped) (int, error) {
	var n int
	r, _, err := syscall.Syscall6(nGetOverlappedResult, 4,
		uintptr(h),
		uintptr(unsafe.Pointer(overlapped)),
		uintptr(unsafe.Pointer(&n)), 1, 0, 0)
	if r == 0 {
		return n, err
	}

	return n, nil
}
<commit_msg>add SetTimeouts to windows.<commit_after>// +build windows

package serial

import (
	"fmt"
	"io"
	"os"
	"sync"
	"syscall"
	"unsafe"
	"time"
)

type serialPort struct {
	f  *os.File
	fd syscall.Handle
	rl sync.Mutex
	wl sync.Mutex
	st *structTimeouts
	h syscall.Handle
	ro *syscall.Overlapped
	wo *syscall.Overlapped
}

type structDCB struct {
	DCBlength, BaudRate                            uint32
	flags                                          [4]byte
	wReserved, XonLim, XoffLim                     uint16
	ByteSize, Parity, StopBits                     byte
	XonChar, XoffChar, ErrorChar, EofChar, EvtChar byte
	wReserved1                                     uint16
}

type structTimeouts struct {
	ReadIntervalTimeout         uint32
	ReadTotalTimeoutMultiplier  uint32
	ReadTotalTimeoutConstant    uint32
	WriteTotalTimeoutMultiplier uint32
	WriteTotalTimeoutConstant   uint32
}

func openPort(name string, baud int) (rwc io.ReadWriteCloser, err error) {
	if len(name) > 0 && name[0] != '\\' {
		name = "\\\\.\\" + name
	}

	h, err := syscall.CreateFile(syscall.StringToUTF16Ptr(name),
		syscall.GENERIC_READ|syscall.GENERIC_WRITE,
		0,
		nil,
		syscall.OPEN_EXISTING,
		syscall.FILE_ATTRIBUTE_NORMAL|syscall.FILE_FLAG_OVERLAPPED,
		0)
	if err != nil {
		return nil, err
	}
	f := os.NewFile(uintptr(h), name)
	defer func() {
		if err != nil {
			f.Close()
		}
	}()

	if err = setCommState(h, baud); err != nil {
		return
	}
	if err = setupComm(h, 64, 64); err != nil {
		return
	}
	if err = setCommMask(h); err != nil {
		return
	}

	ro, err := newOverlapped()
	if err != nil {
		return
	}
	wo, err := newOverlapped()
	if err != nil {
		return
	}
	port := new(serialPort)
	port.f = f
	port.fd = h
	port.ro = ro
	port.wo = wo
    var timeouts structTimeouts
    port.st = &timeouts
//	if err = setCommTimeouts(&port); err != nil {
//		return
//	}
    port.SetTimeouts(100)

	return port, nil
}

func (p *serialPort) Close() error {
	return p.f.Close()
}
func (p *serialPort) SetTimeouts(msec uint32){
    timeouts := p.st
	timeouts.ReadIntervalTimeout = msec/10
	timeouts.ReadTotalTimeoutMultiplier = msec
	timeouts.ReadTotalTimeoutConstant = msec

	/* From http://msdn.microsoft.com/en-us/library/aa363190(v=VS.85).aspx

		 For blocking I/O see below:

		 Remarks:

		 If an application sets ReadIntervalTimeout and
		 ReadTotalTimeoutMultiplier to MAXDWORD and sets
		 ReadTotalTimeoutConstant to a value greater than zero and
		 less than MAXDWORD, one of the following occurs when the
		 ReadFile function is called:

		 If there are any bytes in the input buffer, ReadFile returns
		       immediately with the bytes in the buffer.

		 If there are no bytes in the input buffer, ReadFile waits
	               until a byte arrives and then returns immediately.

		 If no bytes arrive within the time specified by
		       ReadTotalTimeoutConstant, ReadFile times out.
	*/

    p.st = timeouts
    setCommTimeouts(p.h, timeouts)

}
func (p *serialPort) Write(buf []byte) (int, error) {
	p.wl.Lock()
	defer p.wl.Unlock()

	if err := resetEvent(p.wo.HEvent); err != nil {
		return 0, err
	}
	var n uint32
	err := syscall.WriteFile(p.fd, buf, &n, p.wo)
	if err != nil && err != syscall.ERROR_IO_PENDING {
		return int(n), err
	}
	return getOverlappedResult(p.fd, p.wo)
}

func (p *serialPort) Read(buf []byte) (int, error) {
	if p == nil || p.f == nil {
		return 0, fmt.Errorf("Invalid port on read %v %v", p, p.f)
	}

	p.rl.Lock()
	defer p.rl.Unlock()

	if err := resetEvent(p.ro.HEvent); err != nil {
		return 0, err
	}
	var done uint32
	err := syscall.ReadFile(p.fd, buf, &done, p.ro)
	if err != nil && err != syscall.ERROR_IO_PENDING {
		return int(done), err
	}
	return getOverlappedResult(p.fd, p.ro)
}

var (
	nSetCommState,
	nSetCommTimeouts,
	nSetCommMask,
	nSetupComm,
	nGetOverlappedResult,
	nCreateEvent,
	nResetEvent uintptr
)

func init() {
	k32, err := syscall.LoadLibrary("kernel32.dll")
	if err != nil {
		panic("LoadLibrary " + err.Error())
	}
	defer syscall.FreeLibrary(k32)

	nSetCommState = getProcAddr(k32, "SetCommState")
	nSetCommTimeouts = getProcAddr(k32, "SetCommTimeouts")
	nSetCommMask = getProcAddr(k32, "SetCommMask")
	nSetupComm = getProcAddr(k32, "SetupComm")
	nGetOverlappedResult = getProcAddr(k32, "GetOverlappedResult")
	nCreateEvent = getProcAddr(k32, "CreateEventW")
	nResetEvent = getProcAddr(k32, "ResetEvent")
}

func getProcAddr(lib syscall.Handle, name string) uintptr {
	addr, err := syscall.GetProcAddress(lib, name)
	if err != nil {
		panic(name + " " + err.Error())
	}
	return addr
}

func setCommState(h syscall.Handle, baud int) error {
	var params structDCB
	params.DCBlength = uint32(unsafe.Sizeof(params))

	params.flags[0] = 0x01  // fBinary
	params.flags[0] |= 0x10 // Assert DSR

	params.BaudRate = uint32(baud)
	params.ByteSize = 8

	r, _, err := syscall.Syscall(nSetCommState, 2, uintptr(h), uintptr(unsafe.Pointer(&params)), 0)
	if r == 0 {
		return err
	}
	return nil
}

func setCommTimeouts(h syscall.Handle, timeouts *structTimeouts) error {
	r, _, err := syscall.Syscall(nSetCommTimeouts, 2, uintptr(h), uintptr(unsafe.Pointer(timeouts)), 0)
	if r == 0 {
		return err
	}
	return nil
}

func setupComm(h syscall.Handle, in, out int) error {
	r, _, err := syscall.Syscall(nSetupComm, 3, uintptr(h), uintptr(in), uintptr(out))
	if r == 0 {
		return err
	}
	return nil
}

func setCommMask(h syscall.Handle) error {
	const EV_RXCHAR = 0x0001
	r, _, err := syscall.Syscall(nSetCommMask, 2, uintptr(h), EV_RXCHAR, 0)
	if r == 0 {
		return err
	}
	return nil
}

func resetEvent(h syscall.Handle) error {
	r, _, err := syscall.Syscall(nResetEvent, 1, uintptr(h), 0, 0)
	if r == 0 {
		return err
	}
	return nil
}

func newOverlapped() (*syscall.Overlapped, error) {
	var overlapped syscall.Overlapped
	r, _, err := syscall.Syscall6(nCreateEvent, 4, 0, 1, 0, 0, 0, 0)
	if r == 0 {
		return nil, err
	}
	overlapped.HEvent = syscall.Handle(r)
	return &overlapped, nil
}

func getOverlappedResult(h syscall.Handle, overlapped *syscall.Overlapped) (int, error) {
	var n int
	r, _, err := syscall.Syscall6(nGetOverlappedResult, 4,
		uintptr(h),
		uintptr(unsafe.Pointer(overlapped)),
		uintptr(unsafe.Pointer(&n)), 1, 0, 0)
	if r == 0 {
		return n, err
	}

	return n, nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt" //for debugging //TODO Remove
	"strconv"
	"strings"
	"sync"
	"time"

	ais "github.com/andmarios/aislib"
	"github.com/tormol/AIS/storage"
)

//The Archive stores the information about the ships (and works as a temp. solution for the RTree concurrency)
type Archive struct {
	rt *storage.RTree //Stores the points
	rw *sync.RWMutex  //works as a lock for the RTree (#TODO: RTree should be improved to handle concurrency on its own)

	si *storage.ShipInfo //Contains tracklog and other info for each ship
}

//Returns a pointer to the new Archive
func NewArchive() *Archive {
	return &Archive{
		rt: storage.NewRTree(),
		rw: &sync.RWMutex{},
		si: storage.NewShipInfo(),
	}
}

// Stores the information recieved form the channel
func (a *Archive) Save(msg chan *Message) {
	counter := 0 //TODO Remove
	for {
		select {
		case m := <-msg:
			var err error
			ps := (*ais.PositionReport)(nil)
			switch m.Type {
			case 1, 2, 3: // class A position report (longest)
				cApr, e := ais.DecodeClassAPositionReport(m.ArmoredPayload())
				ps = &cApr.PositionReport
				if e != nil {
					continue //TODO
				}
				err = a.updatePos(ps)
			case 5: // static voyage data
				svd, e := ais.DecodeStaticVoyageData(m.ArmoredPayload())
				if e != nil && svd.MMSI <= 0 {
					continue //TODO
				}
				err = a.si.UpdateSVD(svd.MMSI, svd.Callsign, svd.Destination, svd.VesselName, svd.ToBow, svd.ToStern)
			case 18: // basic class B position report (shorter)
				cBpr, e := ais.DecodeClassBPositionReport(m.ArmoredPayload())
				ps = &cBpr.PositionReport
				if e != nil {
					continue //TODO
				}
				err = a.updatePos(ps)
			}
			if err != nil {
				//fmt.Printf("Had an error saving to Archive... %v\n", err)
				continue //TODO do something...
			}
			counter++              //TODO Remove
			if counter%1000 == 0 { //TODO Remove
				fmt.Printf("Number of boats: %d\n", a.rt.NumOfBoats())
				fmt.Println(a.FindWithin(59.0, 5.54, 59.15, 5.8))
				//fmt.Println(a.FindAll())
			}
		}
	}
}

//Updates the ships position in the structures (message type 1,2,3,18)
func (a *Archive) updatePos(ps *ais.PositionReport) error {
	mmsi := ps.MMSI
	if !okCoords(ps.Lat, ps.Lon) || mmsi <= 0 { //This happends quite frequently (coordinates are set to 91,181)
		return errors.New(fmt.Sprintf("Cannot update position... MMSI: %d, lat: %f, long %f", mmsi, ps.Lat, ps.Lon))
	}
	//Check if it is a known ship
	if a.si.IsKnown(mmsi) {
		oldLat, oldLong := a.si.GetCoords(mmsi) //get the previous coordinates
		a.rw.Lock()
		a.rt.Update(mmsi, oldLat, oldLong, ps.Lat, ps.Lon) //update the position in the R*Tree
		a.rw.Unlock()
	} else {
		a.rw.Lock()
		a.rt.InsertData(ps.Lat, ps.Lon, mmsi) //insert a new ship into the R*Tree
		a.rw.Unlock()
	} //TODO check for error?
	err := a.si.AddCheckpoint(ps.MMSI, ps.Lat, ps.Lon, time.Now(), ps.Heading) //Adds the position to the ships tracklog
	return err
}

// Returns a GeoJSON FeatureCollection containing all the known ships
func (a *Archive) FindAll() string {
	geoJsonFC, _ := a.FindWithin(-79.999999, -179.999999, 79.999999, 179.999999)
	return geoJsonFC
}

/*
Public func for finding all known boats that overlaps a given rectangle of the map [13], [14]
	input:
		minLatitude, minLongitude, maxLatitude, maxLongitude	float64
	output:
		string	-	All matching ships in GeoJSON FeatureCollection

*/
func (a *Archive) FindWithin(minLat, minLong, maxLat, maxLong float64) (string, error) {
	r, err := storage.NewRectangle(minLat, minLong, maxLat, maxLong)
	if err != nil {
		return "{}", fmt.Errorf("ERROR, invalid rectangle coordinates")
	}
	a.rw.RLock()
	matchingShips := a.rt.FindWithin(r)
	a.rw.RUnlock()
	features := []string{}
	var name string
	var length, heading uint16
	for _, s := range *matchingShips {
		name, length, heading = a.si.GetFeatures(s.MMSI)
		f := `{
				"type": "Feature",
				"id": ` + strconv.Itoa(int(s.MMSI)) + `,
				"geometry": {
					"type": "Point",
					"coordinates": ` + "[" + strconv.FormatFloat(s.Long, 'f', 6, 64) + ", " + strconv.FormatFloat(s.Lat, 'f', 6, 64) + "]" + `},
				"properties": {
					"name": "` + name + `" ,
					"length": ` + strconv.Itoa(int(length)) + `,
					"heading": ` + strconv.Itoa(int(heading)) + `
				}

			}`
		features = append(features, f)
	}
	return "{ \"type\": \"FeatureCollection\", \"features\": [" + strings.Join(features, ", ") + "]}", nil
}

// Check if the coordinates are ok.	(<91, 181> seems to be a fallback value for the coordinates)
func okCoords(lat, long float64) bool {
	if lat <= 90 && long <= 180 && lat >= -90 && long >= -180 {
		return true
	}
	return false
}

/*
TODO:
	- Fix rStarTree so that it handles concurrency by itself?
		- Archive controls the concurrency of the RTree at the moment...
				- need not be much point using a RWMutex for the rtree... there are a lot more writes than reads atm ... could use a normal mutex, and thereby save some overhead..
		- This could be improved in the future by modifying the rtree structure

References:
	[1]	http://geojsonlint.com/
	[2]	http://stackoverflow.com/questions/7933460/how-do-you-write-multiline-strings-in-go#7933487
*/
<commit_msg>JSON encode ship names<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt" //for debugging //TODO Remove
	"strconv"
	"strings"
	"sync"
	"time"

	ais "github.com/andmarios/aislib"
	"github.com/tormol/AIS/storage"
)

//The Archive stores the information about the ships (and works as a temp. solution for the RTree concurrency)
type Archive struct {
	rt *storage.RTree //Stores the points
	rw *sync.RWMutex  //works as a lock for the RTree (#TODO: RTree should be improved to handle concurrency on its own)

	si *storage.ShipInfo //Contains tracklog and other info for each ship
}

//Returns a pointer to the new Archive
func NewArchive() *Archive {
	return &Archive{
		rt: storage.NewRTree(),
		rw: &sync.RWMutex{},
		si: storage.NewShipInfo(),
	}
}

// Stores the information recieved form the channel
func (a *Archive) Save(msg chan *Message) {
	counter := 0 //TODO Remove
	for {
		select {
		case m := <-msg:
			var err error
			ps := (*ais.PositionReport)(nil)
			switch m.Type {
			case 1, 2, 3: // class A position report (longest)
				cApr, e := ais.DecodeClassAPositionReport(m.ArmoredPayload())
				ps = &cApr.PositionReport
				if e != nil {
					continue //TODO
				}
				err = a.updatePos(ps)
			case 5: // static voyage data
				svd, e := ais.DecodeStaticVoyageData(m.ArmoredPayload())
				if e != nil && svd.MMSI <= 0 {
					continue //TODO
				}
				err = a.si.UpdateSVD(svd.MMSI, svd.Callsign, svd.Destination, svd.VesselName, svd.ToBow, svd.ToStern)
			case 18: // basic class B position report (shorter)
				cBpr, e := ais.DecodeClassBPositionReport(m.ArmoredPayload())
				ps = &cBpr.PositionReport
				if e != nil {
					continue //TODO
				}
				err = a.updatePos(ps)
			}
			if err != nil {
				//fmt.Printf("Had an error saving to Archive... %v\n", err)
				continue //TODO do something...
			}
			counter++              //TODO Remove
			if counter%1000 == 0 { //TODO Remove
				fmt.Printf("Number of boats: %d\n", a.rt.NumOfBoats())
				fmt.Println(a.FindWithin(59.0, 5.54, 59.15, 5.8))
				//fmt.Println(a.FindAll())
			}
		}
	}
}

//Updates the ships position in the structures (message type 1,2,3,18)
func (a *Archive) updatePos(ps *ais.PositionReport) error {
	mmsi := ps.MMSI
	if !okCoords(ps.Lat, ps.Lon) || mmsi <= 0 { //This happends quite frequently (coordinates are set to 91,181)
		return errors.New(fmt.Sprintf("Cannot update position... MMSI: %d, lat: %f, long %f", mmsi, ps.Lat, ps.Lon))
	}
	//Check if it is a known ship
	if a.si.IsKnown(mmsi) {
		oldLat, oldLong := a.si.GetCoords(mmsi) //get the previous coordinates
		a.rw.Lock()
		a.rt.Update(mmsi, oldLat, oldLong, ps.Lat, ps.Lon) //update the position in the R*Tree
		a.rw.Unlock()
	} else {
		a.rw.Lock()
		a.rt.InsertData(ps.Lat, ps.Lon, mmsi) //insert a new ship into the R*Tree
		a.rw.Unlock()
	} //TODO check for error?
	err := a.si.AddCheckpoint(ps.MMSI, ps.Lat, ps.Lon, time.Now(), ps.Heading) //Adds the position to the ships tracklog
	return err
}

// Returns a GeoJSON FeatureCollection containing all the known ships
func (a *Archive) FindAll() string {
	geoJsonFC, _ := a.FindWithin(-79.999999, -179.999999, 79.999999, 179.999999)
	return geoJsonFC
}

/*
Public func for finding all known boats that overlaps a given rectangle of the map [13], [14]
	input:
		minLatitude, minLongitude, maxLatitude, maxLongitude	float64
	output:
		string	-	All matching ships in GeoJSON FeatureCollection

*/
func (a *Archive) FindWithin(minLat, minLong, maxLat, maxLong float64) (string, error) {
	r, err := storage.NewRectangle(minLat, minLong, maxLat, maxLong)
	if err != nil {
		return "{}", fmt.Errorf("ERROR, invalid rectangle coordinates")
	}
	a.rw.RLock()
	matchingShips := a.rt.FindWithin(r)
	a.rw.RUnlock()
	features := []string{}
	var name string
	var length, heading uint16
	for _, s := range *matchingShips {
		name, length, heading = a.si.GetFeatures(s.MMSI)
		name, _ := json.Marshal(name)
		f := `{
				"type": "Feature",
				"id": ` + strconv.Itoa(int(s.MMSI)) + `,
				"geometry": {
					"type": "Point",
					"coordinates": ` + "[" + strconv.FormatFloat(s.Long, 'f', 6, 64) + ", " + strconv.FormatFloat(s.Lat, 'f', 6, 64) + "]" + `},
				"properties": {
					"name": ` + string(name) + `,
					"length": ` + strconv.Itoa(int(length)) + `,
					"heading": ` + strconv.Itoa(int(heading)) + `
				}

			}`
		features = append(features, f)
	}
	return "{ \"type\": \"FeatureCollection\", \"features\": [" + strings.Join(features, ", ") + "]}", nil
}

// Check if the coordinates are ok.	(<91, 181> seems to be a fallback value for the coordinates)
func okCoords(lat, long float64) bool {
	if lat <= 90 && long <= 180 && lat >= -90 && long >= -180 {
		return true
	}
	return false
}

/*
TODO:
	- Fix rStarTree so that it handles concurrency by itself?
		- Archive controls the concurrency of the RTree at the moment...
				- need not be much point using a RWMutex for the rtree... there are a lot more writes than reads atm ... could use a normal mutex, and thereby save some overhead..
		- This could be improved in the future by modifying the rtree structure

References:
	[1]	http://geojsonlint.com/
	[2]	http://stackoverflow.com/questions/7933460/how-do-you-write-multiline-strings-in-go#7933487
*/
<|endoftext|>


<commit_before>package server

import (
	"errors"

	"bitbucket.org/cmaiorano/golang-wol/types"
	wol "github.com/sabhiram/go-wol"
)

var deviceChan = make(chan *types.Alias)
var getChan = make(chan *types.GetDev)
var passHandlingChan = make(chan *types.PasswordHandling)
var updatePassChan = make(chan *types.PasswordUpdate)

func registerDevice(alias, mac, iface string) error {
	dev := &types.Device{Iface: iface, Mac: mac}
	resp := make(chan struct{}, 1)
	aliasStr := &types.Alias{Device: dev, Name: alias, Response: resp}
	deviceChan <- aliasStr

	if value, ok := <-resp; !ok {
		return errors.New("Error adding device")
	}
	return nil
}

func sendPacket(computerName, localIface string) error {
	macAddr, bcastAddr = "", ""
	err := wol.SendMagicPacket(macAddr, bcastAddr, localIface)
	return err
}
<commit_msg>redirecting<commit_after>package server

import (
	"errors"
	"net/http"

	"bitbucket.org/cmaiorano/golang-wol/types"
	wol "github.com/sabhiram/go-wol"
)

var initialized = false
var deviceChan = make(chan *types.Alias)
var getChan = make(chan *types.GetDev)
var passHandlingChan = make(chan *types.PasswordHandling)
var updatePassChan = make(chan *types.PasswordUpdate)

func handleRoot(w http.ResponseWriter, r *http.Request) {
	if !initialized {
		redirectToConfig(w, r)
		return
	}
}

func redirectToConfig(w http.ResponseWriter, r *http.Request) {
	http.Redirect(w, r, "/config", 301)
}

func registerDevice(alias, mac, iface string) error {
	dev := &types.Device{Iface: iface, Mac: mac}
	resp := make(chan struct{}, 1)
	aliasStr := &types.Alias{Device: dev, Name: alias, Response: resp}
	deviceChan <- aliasStr

	if value, ok := <-resp; !ok {
		return errors.New("Error adding device")
	}
	return nil
}

func sendPacket(computerName, localIface string) error {
	macAddr, bcastAddr = "", ""
	err := wol.SendMagicPacket(macAddr, bcastAddr, localIface)
	return err
}
<|endoftext|>


<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Daniel Theophanes (kardianos@gmail.com)

package server

import (
	"errors"
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/cockroachdb/cockroach/gossip"
	"github.com/cockroachdb/cockroach/proto"
	"github.com/cockroachdb/cockroach/storage/engine"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/log"
)

// Context defaults.
const (
	defaultAddr           = ":8080"
	defaultMaxOffset      = 250 * time.Millisecond
	defaultGossipInterval = 2 * time.Second
	defaultCacheSize      = 1 << 30 // GB
)

// Context holds parameters needed to setup a server.
// Calling "server/cli".InitFlags(ctx *Context) will initialize Context using
// command flags. Keep in sync with "server/cli/flags.go".
type Context struct {
	// Addr is the host:port to bind for HTTP/RPC traffic.
	Addr string

	// Certs specifies a directory containing RSA key and x509 certs.
	Certs string

	// Stores is specified to enable durable key-value storage.
	// Memory-backed key value stores may be optionally specified
	// via mem=<integer byte size>.
	//
	// Stores specify a comma-separated list of stores specified by a
	// colon-separated list of device attributes followed by '=' and
	// either a filepath for a persistent store or an integer size in bytes for an
	// in-memory store. Device attributes typically include whether the store is
	// flash (ssd), spinny disk (hdd), fusion-io (fio), in-memory (mem); device
	// attributes might also include speeds and other specs (7200rpm, 200kiops, etc.).
	// For example, -store=hdd:7200rpm=/mnt/hda1,ssd=/mnt/ssd01,ssd=/mnt/ssd02,mem=1073741824
	Stores string

	// Attrs specifies a colon-separated list of node topography or machine
	// capabilities, used to match capabilities or location preferences specified
	// in zone configs.
	Attrs string

	// Maximum clock offset for the cluster.
	MaxOffset time.Duration

	// GossipBootstrap is a comma-separated list of node addresses that
	// act as bootstrap hosts for connecting to the gossip network.
	GossipBootstrap string

	// GossipInterval is a time interval specifying how often gossip is
	// communicated between hosts on the gossip network.
	GossipInterval time.Duration

	// Enables linearizable behaviour of operations on this node by making sure
	// that no commit timestamp is reported back to the client until all other
	// node clocks have necessarily passed it.
	Linearizable bool

	// CacheSize is the amount of memory in bytes to use for caching data.
	// The value is split evenly between the stores if there are more than one.
	CacheSize int64

	// Parsed values.

	// Engines is the storage instances specified by Stores.
	Engines []engine.Engine

	// NodeAttributes is the parsed representation of Attrs.
	NodeAttributes proto.Attributes

	// GossipBootstrapResolvers is a list of gossip resolvers used
	// to find bootstrap nodes for connecting to the gossip network.
	GossipBootstrapResolvers []gossip.Resolver
}

// NewContext returns a Context with default values.
func NewContext() *Context {
	return &Context{
		Addr:           defaultAddr,
		MaxOffset:      defaultMaxOffset,
		GossipInterval: defaultGossipInterval,
		CacheSize:      defaultCacheSize,
	}
}

// Init interprets the stores parameter to initialize a slice of
// engine.Engine objects, parses node attributes, and initializes
// the gossip bootstrap resolvers.
func (ctx *Context) Init() error {
	var err error
	storesRE := regexp.MustCompile(`([^=]+)=([^,]+)(,|$)`)
	// Error if regexp doesn't match.
	storeSpecs := storesRE.FindAllStringSubmatch(ctx.Stores, -1)
	if storeSpecs == nil || len(storeSpecs) == 0 {
		return fmt.Errorf("invalid or empty engines specification %q, "+
			"did you specify -stores?", ctx.Stores)
	}

	ctx.Engines = nil
	for _, store := range storeSpecs {
		if len(store) != 4 {
			return util.Errorf("unable to parse attributes and path from store %q", store[0])
		}
		// There are two matches for each store specification: the colon-separated
		// list of attributes and the path.
		engine, err := ctx.initEngine(store[1], store[2])
		if err != nil {
			return util.Errorf("unable to init engine for store %q: %s", store[0], err)
		}
		ctx.Engines = append(ctx.Engines, engine)
	}
	log.Infof("initialized %d storage engine(s)", len(ctx.Engines))

	ctx.NodeAttributes = parseAttributes(ctx.Attrs)

	resolvers, err := ctx.parseGossipBootstrapResolvers()
	if err != nil {
		return err
	}
	if len(resolvers) == 0 {
		return errors.New("no gossip addresses found, did you specify -gossip?")
	}
	ctx.GossipBootstrapResolvers = resolvers

	return nil
}

// initEngine parses the store attributes as a colon-separated list
// and instantiates an engine based on the dir parameter. If dir parses
// to an integer, it's taken to mean an in-memory engine; otherwise,
// dir is treated as a path and a RocksDB engine is created.
func (ctx *Context) initEngine(attrsStr, path string) (engine.Engine, error) {
	attrs := parseAttributes(attrsStr)
	if size, err := strconv.ParseUint(path, 10, 64); err == nil {
		if size == 0 {
			return nil, util.Errorf("unable to initialize an in-memory store with capacity 0")
		}
		return engine.NewInMem(attrs, int64(size)), nil
		// TODO(spencer): should be using rocksdb for in-memory stores and
		// relegate the InMem engine to usage only from unittests.
	}
	return engine.NewRocksDB(attrs, path, ctx.CacheSize), nil
}

// parseGossipBootstrapResolvers parses a comma-separated list of
// gossip bootstrap resolvers.
func (ctx *Context) parseGossipBootstrapResolvers() ([]*gossip.Resolver, error) {
	var bootstrapResolvers []gossip.Resolver
	addresses := strings.Split(ctx.GossipBootstrap, ",")
	for _, address := range addresses {
		if len(address) == 0 {
			continue
		}
		// Special case self:// to pick a nice address that resolves
		// uniquely for use in Gossip. This avoids having to specify
		// the port for single-node clusters twice (once in -addr,
		// once in -gossip).
		if strings.HasPrefix(address, "self://") {
			address = util.EnsureHost(ctx.Addr)
		}
		resolver, err := gossip.NewResolver(address)
		if err != nil {
			return nil, err
		}
		bootstrapResolvers = append(bootstrapResolvers, resolver)
	}

	return bootstrapResolvers, nil
}

// parseAttributes parses a colon-separated list of strings,
// filtering empty strings (i.e. "::" will yield no attributes.
// Returns the list of strings as Attributes.
func parseAttributes(attrsStr string) proto.Attributes {
	var filtered []string
	for _, attr := range strings.Split(attrsStr, ":") {
		if len(attr) != 0 {
			filtered = append(filtered, attr)
		}
	}
	return proto.Attributes{Attrs: filtered}
}
<commit_msg>buf fix<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Daniel Theophanes (kardianos@gmail.com)

package server

import (
	"errors"
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/cockroachdb/cockroach/gossip"
	"github.com/cockroachdb/cockroach/proto"
	"github.com/cockroachdb/cockroach/storage/engine"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/log"
)

// Context defaults.
const (
	defaultAddr           = ":8080"
	defaultMaxOffset      = 250 * time.Millisecond
	defaultGossipInterval = 2 * time.Second
	defaultCacheSize      = 1 << 30 // GB
)

// Context holds parameters needed to setup a server.
// Calling "server/cli".InitFlags(ctx *Context) will initialize Context using
// command flags. Keep in sync with "server/cli/flags.go".
type Context struct {
	// Addr is the host:port to bind for HTTP/RPC traffic.
	Addr string

	// Certs specifies a directory containing RSA key and x509 certs.
	Certs string

	// Stores is specified to enable durable key-value storage.
	// Memory-backed key value stores may be optionally specified
	// via mem=<integer byte size>.
	//
	// Stores specify a comma-separated list of stores specified by a
	// colon-separated list of device attributes followed by '=' and
	// either a filepath for a persistent store or an integer size in bytes for an
	// in-memory store. Device attributes typically include whether the store is
	// flash (ssd), spinny disk (hdd), fusion-io (fio), in-memory (mem); device
	// attributes might also include speeds and other specs (7200rpm, 200kiops, etc.).
	// For example, -store=hdd:7200rpm=/mnt/hda1,ssd=/mnt/ssd01,ssd=/mnt/ssd02,mem=1073741824
	Stores string

	// Attrs specifies a colon-separated list of node topography or machine
	// capabilities, used to match capabilities or location preferences specified
	// in zone configs.
	Attrs string

	// Maximum clock offset for the cluster.
	MaxOffset time.Duration

	// GossipBootstrap is a comma-separated list of node addresses that
	// act as bootstrap hosts for connecting to the gossip network.
	GossipBootstrap string

	// GossipInterval is a time interval specifying how often gossip is
	// communicated between hosts on the gossip network.
	GossipInterval time.Duration

	// Enables linearizable behaviour of operations on this node by making sure
	// that no commit timestamp is reported back to the client until all other
	// node clocks have necessarily passed it.
	Linearizable bool

	// CacheSize is the amount of memory in bytes to use for caching data.
	// The value is split evenly between the stores if there are more than one.
	CacheSize int64

	// Parsed values.

	// Engines is the storage instances specified by Stores.
	Engines []engine.Engine

	// NodeAttributes is the parsed representation of Attrs.
	NodeAttributes proto.Attributes

	// GossipBootstrapResolvers is a list of gossip resolvers used
	// to find bootstrap nodes for connecting to the gossip network.
	GossipBootstrapResolvers []gossip.Resolver
}

// NewContext returns a Context with default values.
func NewContext() *Context {
	return &Context{
		Addr:           defaultAddr,
		MaxOffset:      defaultMaxOffset,
		GossipInterval: defaultGossipInterval,
		CacheSize:      defaultCacheSize,
	}
}

// Init interprets the stores parameter to initialize a slice of
// engine.Engine objects, parses node attributes, and initializes
// the gossip bootstrap resolvers.
func (ctx *Context) Init() error {
	var err error
	storesRE := regexp.MustCompile(`([^=]+)=([^,]+)(,|$)`)
	// Error if regexp doesn't match.
	storeSpecs := storesRE.FindAllStringSubmatch(ctx.Stores, -1)
	if storeSpecs == nil || len(storeSpecs) == 0 {
		return fmt.Errorf("invalid or empty engines specification %q, "+
			"did you specify -stores?", ctx.Stores)
	}

	ctx.Engines = nil
	for _, store := range storeSpecs {
		if len(store) != 4 {
			return util.Errorf("unable to parse attributes and path from store %q", store[0])
		}
		// There are two matches for each store specification: the colon-separated
		// list of attributes and the path.
		engine, err := ctx.initEngine(store[1], store[2])
		if err != nil {
			return util.Errorf("unable to init engine for store %q: %s", store[0], err)
		}
		ctx.Engines = append(ctx.Engines, engine)
	}
	log.Infof("initialized %d storage engine(s)", len(ctx.Engines))

	ctx.NodeAttributes = parseAttributes(ctx.Attrs)

	resolvers, err := ctx.parseGossipBootstrapResolvers()
	if err != nil {
		return err
	}
	if len(resolvers) == 0 {
		return errors.New("no gossip addresses found, did you specify -gossip?")
	}
	ctx.GossipBootstrapResolvers = resolvers

	return nil
}

// initEngine parses the store attributes as a colon-separated list
// and instantiates an engine based on the dir parameter. If dir parses
// to an integer, it's taken to mean an in-memory engine; otherwise,
// dir is treated as a path and a RocksDB engine is created.
func (ctx *Context) initEngine(attrsStr, path string) (engine.Engine, error) {
	attrs := parseAttributes(attrsStr)
	if size, err := strconv.ParseUint(path, 10, 64); err == nil {
		if size == 0 {
			return nil, util.Errorf("unable to initialize an in-memory store with capacity 0")
		}
		return engine.NewInMem(attrs, int64(size)), nil
		// TODO(spencer): should be using rocksdb for in-memory stores and
		// relegate the InMem engine to usage only from unittests.
	}
	return engine.NewRocksDB(attrs, path, ctx.CacheSize), nil
}

// parseGossipBootstrapResolvers parses a comma-separated list of
// gossip bootstrap resolvers.
func (ctx *Context) parseGossipBootstrapResolvers() ([]gossip.Resolver, error) {
	var bootstrapResolvers []gossip.Resolver
	addresses := strings.Split(ctx.GossipBootstrap, ",")
	for _, address := range addresses {
		if len(address) == 0 {
			continue
		}
		// Special case self:// to pick a nice address that resolves
		// uniquely for use in Gossip. This avoids having to specify
		// the port for single-node clusters twice (once in -addr,
		// once in -gossip).
		if strings.HasPrefix(address, "self://") {
			address = util.EnsureHost(ctx.Addr)
		}
		resolver, err := gossip.NewResolver(address)
		if err != nil {
			return nil, err
		}
		bootstrapResolvers = append(bootstrapResolvers, resolver)
	}

	return bootstrapResolvers, nil
}

// parseAttributes parses a colon-separated list of strings,
// filtering empty strings (i.e. "::" will yield no attributes.
// Returns the list of strings as Attributes.
func parseAttributes(attrsStr string) proto.Attributes {
	var filtered []string
	for _, attr := range strings.Split(attrsStr, ":") {
		if len(attr) != 0 {
			filtered = append(filtered, attr)
		}
	}
	return proto.Attributes{Attrs: filtered}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"github.com/1lann/lol-replay/record"
	"github.com/1lann/lol-replay/recording"
	"log"
	"net/http"
	"os"
	"runtime/debug"
	"strconv"
	"time"
)

var platformToRegion = map[string]string{
	"NA1":  "na",
	"OC1":  "oce",
	"EUN1": "eune",
	"EUW1": "euw",
	"KR":   "kr",
	"BR1":  "br",
	"LA1":  "lan",
	"LA2":  "las",
	"RU":   "ru",
	"TR1":  "tr",
	"PBE1": "pbe",
}

type player struct {
	ID       string `json:"id"`
	Platform string `json:"platform"`
}

type gameInfoMetadata struct {
	BannedChampions []struct {
		ChampionID int `json:"championId"`
		PickTurn   int `json:"pickTurn"`
		TeamID     int `json:"teamId"`
	} `json:"bannedChampions"`
	GameID            int64  `json:"gameId"`
	GameLength        int    `json:"gameLength"`
	GameMode          string `json:"gameMode"`
	GameQueueConfigID int    `json:"gameQueueConfigId"`
	GameStartTime     int64  `json:"gameStartTime"`
	GameType          string `json:"gameType"`
	MapID             int    `json:"mapId"`
	Observers         struct {
		EncryptionKey string `json:"encryptionKey"`
	} `json:"observers"`
	Participants []struct {
		Bot        bool `json:"bot"`
		ChampionID int  `json:"championId"`
		Masteries  []struct {
			MasteryID int `json:"masteryId"`
			Rank      int `json:"rank"`
		} `json:"masteries"`
		ProfileIconID int `json:"profileIconId"`
		Runes         []struct {
			Count  int `json:"count"`
			RuneID int `json:"runeId"`
		} `json:"runes"`
		Spell1Id     int    `json:"spell1Id"`
		Spell2Id     int    `json:"spell2Id"`
		SummonerID   int64  `json:"summonerId"`
		SummonerName string `json:"summonerName"`
		TeamID       int    `json:"teamId"`
	} `json:"participants"`
	PlatformID string `json:"platformId"`
}

func monitorPlayers() {
	waitSeconds := float64(config.RefreshRate) / float64(len(config.Players))
	waitPeriod := time.Millisecond * time.Duration(waitSeconds*1000.0)

	for {
		for _, player := range config.Players {
			time.Sleep(waitPeriod)

			info, ok := player.currentGameInfo(config.RiotAPIKey)
			if !ok {
				continue
			}

			gameId := strconv.FormatInt(info.GameID, 10)
			keyName := info.PlatformID + "_" + gameId
			resume := false

			recordingsMutex.RLock()

			if _, found := recordings[keyName]; found {
				if recordings[keyName].temporary ||
					recordings[keyName].recording ||
					recordings[keyName].rec.IsComplete() {
					recordingsMutex.RUnlock()
					continue
				}

				if !recordings[keyName].rec.IsComplete() {
					resume = true
				}
			}

			recordingsMutex.RUnlock()
			recordingsMutex.Lock()

			if !resume {
				recordings[keyName] = &internalRecording{
					temporary: true,
					recording: false,
				}
			} else {
				recordings[keyName].temporary = true
				recordings[keyName].recording = false
			}

			cleanUp()
			recordingsMutex.Unlock()
			go recordGame(info, resume)
		}
	}
}

// recordingsMutex must be Locked before cleanUp is called.
func cleanUp() {
	for len(recordings) >= config.KeepNumRecordings {
		deleteRecording := sortedRecordings[0]
		deleteRecording.temporary = true
		deleteRecording.file.Close()
		err := os.Remove(deleteRecording.location)
		if err != nil {
			log.Println("failed to delete "+
				deleteRecording.location+":", err)
		} else {
			log.Println("deleted: " + deleteRecording.location)
		}

		sortedRecordings = sortedRecordings[1:]

		for key, rec := range recordings {
			if rec == deleteRecording {
				delete(recordings, key)
				break
			}
		}
	}
}

func recordGame(info gameInfoMetadata, resume bool) {
	gameId := strconv.FormatInt(info.GameID, 10)
	keyName := info.PlatformID + "_" + gameId

	defer func() {
		if e := recover(); e != nil {
			log.Printf("record game panic: %s: %s", e, debug.Stack())

			recordingsMutex.Lock()
			recordings[keyName].recording = false
			recordingsMutex.Unlock()
		}
	}()

	var file *os.File
	var rec *recording.Recording
	var err error
	sortedKey := -1

	if !resume {
		file, err = os.Create(config.RecordingsDirectory + "/" + keyName + ".glr")
		if err != nil {
			log.Println("create recording error:", err)
			return
		}

		rec, err = recording.NewRecording(file)
		if err != nil {
			log.Println("failed to initialize recording:", err)
			return
		}
	} else {
		recordingsMutex.RLock()
		rec = recordings[keyName].rec
		file = recordings[keyName].file

		for i, internalRec := range sortedRecordings {
			if internalRec.rec == rec {
				sortedKey = i
				break
			}
		}
		recordingsMutex.RUnlock()
	}

	recordingsMutex.Lock()
	recordings[keyName] = &internalRecording{
		file:      file,
		location:  config.RecordingsDirectory + "/" + file.Name(),
		rec:       rec,
		temporary: false,
		recording: true,
	}

	if resume && sortedKey >= 0 {
		sortedRecordings[sortedKey] = recordings[keyName]
	} else {
		sortedRecordings = append(sortedRecordings, recordings[keyName])
	}
	recordingsMutex.Unlock()

	if !rec.HasUserMetadata() {
		if err := rec.StoreUserMetadata(&info); err != nil {
			log.Println("recording failed to store game metadata:", err)
			return
		}
	}

	if resume {
		log.Println("resuming recording " + keyName)
	} else {
		log.Println("recording " + keyName)
	}

	err = record.Record(info.PlatformID, gameId,
		info.Observers.EncryptionKey, rec)

	recordingsMutex.Lock()
	recordings[keyName].recording = false
	recordingsMutex.Unlock()

	if err != nil {
		log.Println("error while recording "+keyName+":", err)
		return
	}

	log.Println("recording " + keyName + " complete")
}

func (p player) currentGameInfo(apiKey string) (gameInfoMetadata, bool) {
	url := "https://" + platformToRegion[p.Platform] + ".api.pvp.net/observer-mode/rest" +
		"/consumer/getSpectatorGameInfo/" + p.Platform + "/" + p.ID +
		"?api_key=" + apiKey

	for i := 0; i < 3; i++ {
		resp, err := http.Get(url)
		if err != nil {
			log.Println("URL:", url)
			log.Println("current game error:", err)
			continue
		}

		if resp.StatusCode == http.StatusNotFound {
			resp.Body.Close()
			return gameInfoMetadata{}, false
		}

		if resp.StatusCode != http.StatusOK {
			log.Println("URL:", url)
			log.Println("current game: not OK:", resp.Status)
			resp.Body.Close()
			continue
		}

		dec := json.NewDecoder(resp.Body)
		var info gameInfoMetadata
		err = dec.Decode(&info)
		resp.Body.Close()
		if err != nil {
			return gameInfoMetadata{}, false
		}

		return info, true
	}

	return gameInfoMetadata{}, false
}
<commit_msg>Lock before deleting<commit_after>package main

import (
	"encoding/json"
	"github.com/1lann/lol-replay/record"
	"github.com/1lann/lol-replay/recording"
	"log"
	"net/http"
	"os"
	"runtime/debug"
	"strconv"
	"time"
)

var platformToRegion = map[string]string{
	"NA1":  "na",
	"OC1":  "oce",
	"EUN1": "eune",
	"EUW1": "euw",
	"KR":   "kr",
	"BR1":  "br",
	"LA1":  "lan",
	"LA2":  "las",
	"RU":   "ru",
	"TR1":  "tr",
	"PBE1": "pbe",
}

type player struct {
	ID       string `json:"id"`
	Platform string `json:"platform"`
}

type gameInfoMetadata struct {
	BannedChampions []struct {
		ChampionID int `json:"championId"`
		PickTurn   int `json:"pickTurn"`
		TeamID     int `json:"teamId"`
	} `json:"bannedChampions"`
	GameID            int64  `json:"gameId"`
	GameLength        int    `json:"gameLength"`
	GameMode          string `json:"gameMode"`
	GameQueueConfigID int    `json:"gameQueueConfigId"`
	GameStartTime     int64  `json:"gameStartTime"`
	GameType          string `json:"gameType"`
	MapID             int    `json:"mapId"`
	Observers         struct {
		EncryptionKey string `json:"encryptionKey"`
	} `json:"observers"`
	Participants []struct {
		Bot        bool `json:"bot"`
		ChampionID int  `json:"championId"`
		Masteries  []struct {
			MasteryID int `json:"masteryId"`
			Rank      int `json:"rank"`
		} `json:"masteries"`
		ProfileIconID int `json:"profileIconId"`
		Runes         []struct {
			Count  int `json:"count"`
			RuneID int `json:"runeId"`
		} `json:"runes"`
		Spell1Id     int    `json:"spell1Id"`
		Spell2Id     int    `json:"spell2Id"`
		SummonerID   int64  `json:"summonerId"`
		SummonerName string `json:"summonerName"`
		TeamID       int    `json:"teamId"`
	} `json:"participants"`
	PlatformID string `json:"platformId"`
}

func monitorPlayers() {
	waitSeconds := float64(config.RefreshRate) / float64(len(config.Players))
	waitPeriod := time.Millisecond * time.Duration(waitSeconds*1000.0)

	for {
		for _, player := range config.Players {
			time.Sleep(waitPeriod)

			info, ok := player.currentGameInfo(config.RiotAPIKey)
			if !ok {
				continue
			}

			gameId := strconv.FormatInt(info.GameID, 10)
			keyName := info.PlatformID + "_" + gameId
			resume := false

			recordingsMutex.RLock()

			if _, found := recordings[keyName]; found {
				if recordings[keyName].temporary ||
					recordings[keyName].recording ||
					recordings[keyName].rec.IsComplete() {
					recordingsMutex.RUnlock()
					continue
				}

				if !recordings[keyName].rec.IsComplete() {
					resume = true
				}
			}

			recordingsMutex.RUnlock()
			recordingsMutex.Lock()

			if !resume {
				recordings[keyName] = &internalRecording{
					temporary: true,
					recording: false,
				}
			} else {
				recordings[keyName].temporary = true
				recordings[keyName].recording = false
			}

			cleanUp()
			recordingsMutex.Unlock()
			go recordGame(info, resume)
		}
	}
}

// recordingsMutex must be Locked before cleanUp is called.
func cleanUp() {
	for len(recordings) >= config.KeepNumRecordings {
		deleteRecording := sortedRecordings[0]
		deleteRecording.rec.Lock()
		deleteRecording.temporary = true
		deleteRecording.file.Close()
		err := os.Remove(deleteRecording.location)
		if err != nil {
			log.Println("failed to delete "+
				deleteRecording.location+":", err)
		} else {
			log.Println("deleted: " + deleteRecording.location)
		}

		sortedRecordings = sortedRecordings[1:]

		for key, rec := range recordings {
			if rec == deleteRecording {
				delete(recordings, key)
				break
			}
		}
	}
}

func recordGame(info gameInfoMetadata, resume bool) {
	gameId := strconv.FormatInt(info.GameID, 10)
	keyName := info.PlatformID + "_" + gameId

	defer func() {
		if e := recover(); e != nil {
			log.Printf("record game panic: %s: %s", e, debug.Stack())

			recordingsMutex.Lock()
			recordings[keyName].recording = false
			recordingsMutex.Unlock()
		}
	}()

	var file *os.File
	var rec *recording.Recording
	var err error
	sortedKey := -1

	if !resume {
		file, err = os.Create(config.RecordingsDirectory + "/" + keyName + ".glr")
		if err != nil {
			log.Println("create recording error:", err)
			return
		}

		rec, err = recording.NewRecording(file)
		if err != nil {
			log.Println("failed to initialize recording:", err)
			return
		}
	} else {
		recordingsMutex.RLock()
		rec = recordings[keyName].rec
		file = recordings[keyName].file

		for i, internalRec := range sortedRecordings {
			if internalRec.rec == rec {
				sortedKey = i
				break
			}
		}
		recordingsMutex.RUnlock()
	}

	recordingsMutex.Lock()
	recordings[keyName] = &internalRecording{
		file:      file,
		location:  config.RecordingsDirectory + "/" + file.Name(),
		rec:       rec,
		temporary: false,
		recording: true,
	}

	if resume && sortedKey >= 0 {
		sortedRecordings[sortedKey] = recordings[keyName]
	} else {
		sortedRecordings = append(sortedRecordings, recordings[keyName])
	}
	recordingsMutex.Unlock()

	if !rec.HasUserMetadata() {
		if err := rec.StoreUserMetadata(&info); err != nil {
			log.Println("recording failed to store game metadata:", err)
			return
		}
	}

	if resume {
		log.Println("resuming recording " + keyName)
	} else {
		log.Println("recording " + keyName)
	}

	err = record.Record(info.PlatformID, gameId,
		info.Observers.EncryptionKey, rec)

	recordingsMutex.Lock()
	recordings[keyName].recording = false
	recordingsMutex.Unlock()

	if err != nil {
		log.Println("error while recording "+keyName+":", err)
		return
	}

	log.Println("recording " + keyName + " complete")
}

func (p player) currentGameInfo(apiKey string) (gameInfoMetadata, bool) {
	url := "https://" + platformToRegion[p.Platform] + ".api.pvp.net/observer-mode/rest" +
		"/consumer/getSpectatorGameInfo/" + p.Platform + "/" + p.ID +
		"?api_key=" + apiKey

	for i := 0; i < 3; i++ {
		resp, err := http.Get(url)
		if err != nil {
			log.Println("URL:", url)
			log.Println("current game error:", err)
			continue
		}

		if resp.StatusCode == http.StatusNotFound {
			resp.Body.Close()
			return gameInfoMetadata{}, false
		}

		if resp.StatusCode != http.StatusOK {
			log.Println("URL:", url)
			log.Println("current game: not OK:", resp.Status)
			resp.Body.Close()
			continue
		}

		dec := json.NewDecoder(resp.Body)
		var info gameInfoMetadata
		err = dec.Decode(&info)
		resp.Body.Close()
		if err != nil {
			return gameInfoMetadata{}, false
		}

		return info, true
	}

	return gameInfoMetadata{}, false
}
<|endoftext|>


<commit_before>package dbmaker

import (
	"log"
	"streamdb/dbutil"
	"streamdb/users"
	"streamdb/config"
)

//MakeUser creates a user directly from a streamdb directory, without needing to start up all of streamdb
func MakeUser(username, password, email string) error {
	if err := StartSqlDatabase(); err != nil {
		return err
	}
	defer StopSqlDatabase()

	log.Printf("Creating user %s (%s)\n", username, email)

	spath := config.GetDatabaseConnectionString()

	db, driver, err := dbutil.OpenSqlDatabase(spath)
	if err != nil {
		return err
	}

	var udb users.UserDatabase
	udb.InitUserDatabase(db, string(driver))
	return udb.CreateUser(username, email, password)
}
<commit_msg>third time's the charm<commit_after>package dbmaker

import (
	"log"
	"streamdb/dbutil"
	"streamdb/users"
	"streamdb/config"
)

//MakeUser creates a user directly from a streamdb directory, without needing to start up all of streamdb
func MakeUser(username, password, email string, isadmin bool) error {
	if err := StartSqlDatabase(); err != nil {
		return err
	}
	defer StopSqlDatabase()

	log.Printf("Creating user %s (%s)\n", username, email)

	spath := config.GetDatabaseConnectionString()

	db, driver, err := dbutil.OpenSqlDatabase(spath)
	if err != nil {
		return err
	}

	var udb users.UserDatabase
	udb.InitUserDatabase(db, string(driver))
	err = udb.CreateUser(username, email, password)
	if err != nil {
		return err
	}

	usr, err := udb.ReadUserByName(username)
	if err != nil {
		return err
	}

	usr.Admin = true
	return udb.UpdateUser(usr)
}
<|endoftext|>


<commit_before>package gce

import (
	"context"
	"fmt"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/compute/v1"
	"io/ioutil"
	"net/http"
	"tes/config"
	pbr "tes/server/proto"
)

type gceClientI interface {
	// Templates returns available worker types,
	// which are described using GCE instance templates.
	Templates() []*pbr.Worker
	StartWorker(*pbr.Worker) error
}

type gceClient struct {
	templates []*pbr.Worker
	conf      config.Config
	svc       *compute.Service
}

// service creates a Google Compute service client
func newGCEClient(ctx context.Context, conf config.Config) (gceClientI, error) {
	var client *http.Client
	if conf.Schedulers.GCE.AccountFile != "" {
		// Pull the client configuration (e.g. auth) from a given account file.
		// This is likely downloaded from Google Cloud manually via IAM & Admin > Service accounts.
		bytes, rerr := ioutil.ReadFile(conf.Schedulers.GCE.AccountFile)
		if rerr != nil {
			return nil, rerr
		}

		config, tserr := google.JWTConfigFromJSON(bytes, compute.ComputeScope)
		if tserr != nil {
			return nil, tserr
		}
		client = config.Client(ctx)
	} else {
		// Pull the information (auth and other config) from the environment,
		// which is useful when this code is running in a Google Compute instance.
		client, _ = google.DefaultClient(ctx, compute.ComputeScope)
		// TODO catch error
	}

	svc, cerr := compute.New(client)
	if cerr != nil {
		return nil, cerr
	}

	return &gceClient{
		conf: conf,
		svc:  svc,
	}, nil
}

// Templates queries the GCE API to get details about GCE instance templates.
// If the API client fails to connect, this returns an empty list.
func (s *gceClient) Templates() []*pbr.Worker {
	// templates are cached after the first call
	if s.templates != nil {
		return s.templates
	}

	templates := []*pbr.Worker{}
	project := s.conf.Schedulers.GCE.Project
	machineTypes := map[string]pbr.Resources{}

	// Get the machine types available to the project + zone
	resp, err := s.svc.MachineTypes.List(project, s.conf.Schedulers.GCE.Zone).Do()
	if err != nil {
		log.Error("Couldn't get GCE machine list.")
		return templates
	}

	for _, m := range resp.Items {
		machineTypes[m.Name] = pbr.Resources{
			Cpus: uint32(m.GuestCpus),
			Ram:  float64(m.MemoryMb) / float64(1024),
		}
	}

	for _, t := range s.conf.Schedulers.GCE.Templates {
		// Get the instance template from the GCE API
		tpl, err := s.svc.InstanceTemplates.Get(project, t).Do()
		if err != nil {
			log.Error("Couldn't get GCE template. Skipping.", "error", err, "template", t)
			continue
		}
		// Map the machine type ID string to a pbr.Resources struct
		res := machineTypes[tpl.Properties.MachineType]
		// TODO is there always at least one disk? Is the first the best choice?
		//      how to know which to pick if there are multiple?
		res.Disk = float64(tpl.Properties.Disks[0].InitializeParams.DiskSizeGb)
		templates = append(templates, &pbr.Worker{
			Resources: &res,
		})
	}

	// Looks like we have a successful response (no errors above)
	// so cache the templates array.
	s.templates = templates
	return templates
}

// StartWorker calls out to GCE APIs to create a VM instance
// with a Funnel worker.
func (s *gceClient) StartWorker(w *pbr.Worker) error {
	project := s.conf.Schedulers.GCE.Project
	zone := s.conf.Schedulers.GCE.Zone

	// Get the instance template from the GCE API
	tpl, terr := s.svc.InstanceTemplates.Get(project, w.Gce.Template).Do()
	if terr != nil {
		log.Error("Couldn't retrieve GCE instance template",
			"error", terr,
			"template", w.Gce.Template)
		return terr
	}

	// TODO
	confYaml := ""

	// Add the funnel config yaml string to the template metadata
	props := tpl.Properties
	metadata := compute.Metadata{
		Items: append(props.Metadata.Items,
			&compute.MetadataItems{
				Key:   "funnel-config",
				Value: &confYaml,
			},
		),
	}

	// Prepare disk details by setting the specific zone
	for _, disk := range props.Disks {
		dt := localize(zone, "diskTypes", disk.InitializeParams.DiskType)
		disk.InitializeParams.DiskType = dt
	}

	instance := compute.Instance{
		Name:              w.Id,
		CanIpForward:      props.CanIpForward,
		Description:       props.Description,
		Disks:             props.Disks,
		MachineType:       localize(zone, "machineTypes", props.MachineType),
		NetworkInterfaces: props.NetworkInterfaces,
		Scheduling:        props.Scheduling,
		ServiceAccounts:   props.ServiceAccounts,
		Tags:              props.Tags,
		Metadata:          &metadata,
	}

	op, ierr := s.svc.Instances.Insert(project, zone, &instance).Do()
	if ierr != nil {
		log.Error("Couldn't insert GCE VM instance", ierr)
		return ierr
	}

	log.Debug("GCE VM instance created", "details", op)
	return nil
}

func localize(zone, resourceType, val string) string {
	return fmt.Sprintf("zones/%s/%s/%s", zone, resourceType, val)
}
<commit_msg>GCE worker config yaml<commit_after>package gce

import (
	"context"
	"fmt"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/compute/v1"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"tes/config"
	pbr "tes/server/proto"
)

type gceClientI interface {
	// Templates returns available worker types,
	// which are described using GCE instance templates.
	Templates() []*pbr.Worker
	StartWorker(*pbr.Worker) error
}

type gceClient struct {
	templates []*pbr.Worker
	conf      config.Config
	svc       *compute.Service
}

// service creates a Google Compute service client
func newGCEClient(ctx context.Context, conf config.Config) (gceClientI, error) {
	var client *http.Client
	if conf.Schedulers.GCE.AccountFile != "" {
		// Pull the client configuration (e.g. auth) from a given account file.
		// This is likely downloaded from Google Cloud manually via IAM & Admin > Service accounts.
		bytes, rerr := ioutil.ReadFile(conf.Schedulers.GCE.AccountFile)
		if rerr != nil {
			return nil, rerr
		}

		config, tserr := google.JWTConfigFromJSON(bytes, compute.ComputeScope)
		if tserr != nil {
			return nil, tserr
		}
		client = config.Client(ctx)
	} else {
		// Pull the information (auth and other config) from the environment,
		// which is useful when this code is running in a Google Compute instance.
		client, _ = google.DefaultClient(ctx, compute.ComputeScope)
		// TODO catch error
	}

	svc, cerr := compute.New(client)
	if cerr != nil {
		return nil, cerr
	}

	return &gceClient{
		conf: conf,
		svc:  svc,
	}, nil
}

// Templates queries the GCE API to get details about GCE instance templates.
// If the API client fails to connect, this returns an empty list.
func (s *gceClient) Templates() []*pbr.Worker {
	// templates are cached after the first call
	if s.templates != nil {
		return s.templates
	}

	templates := []*pbr.Worker{}
	project := s.conf.Schedulers.GCE.Project
	machineTypes := map[string]pbr.Resources{}

	// Get the machine types available to the project + zone
	resp, err := s.svc.MachineTypes.List(project, s.conf.Schedulers.GCE.Zone).Do()
	if err != nil {
		log.Error("Couldn't get GCE machine list.")
		return templates
	}

	for _, m := range resp.Items {
		machineTypes[m.Name] = pbr.Resources{
			Cpus: uint32(m.GuestCpus),
			Ram:  float64(m.MemoryMb) / float64(1024),
		}
	}

	for _, t := range s.conf.Schedulers.GCE.Templates {
		// Get the instance template from the GCE API
		tpl, err := s.svc.InstanceTemplates.Get(project, t).Do()
		if err != nil {
			log.Error("Couldn't get GCE template. Skipping.", "error", err, "template", t)
			continue
		}
		// Map the machine type ID string to a pbr.Resources struct
		res := machineTypes[tpl.Properties.MachineType]
		// TODO is there always at least one disk? Is the first the best choice?
		//      how to know which to pick if there are multiple?
		res.Disk = float64(tpl.Properties.Disks[0].InitializeParams.DiskSizeGb)
		templates = append(templates, &pbr.Worker{
			Resources: &res,
		})
	}

	// Looks like we have a successful response (no errors above)
	// so cache the templates array.
	s.templates = templates
	return templates
}

// StartWorker calls out to GCE APIs to create a VM instance
// with a Funnel worker.
func (s *gceClient) StartWorker(w *pbr.Worker) error {
	project := s.conf.Schedulers.GCE.Project
	zone := s.conf.Schedulers.GCE.Zone

	// Get the instance template from the GCE API
	tpl, terr := s.svc.InstanceTemplates.Get(project, w.Gce.Template).Do()
	if terr != nil {
		log.Error("Couldn't retrieve GCE instance template",
			"error", terr,
			"template", w.Gce.Template)
		return terr
	}

	c := s.conf.Worker
	c.ID = w.Id
	c.Timeout = -1
	c.Storage = s.conf.Storage
	workdir := path.Join(s.conf.WorkDir, w.Id)
	workdir, _ = filepath.Abs(workdir)
	os.MkdirAll(workdir, 0755)
	confYaml := string(c.ToYaml())

	// Add the funnel config yaml string to the template metadata
	props := tpl.Properties
	metadata := compute.Metadata{
		Items: append(props.Metadata.Items,
			&compute.MetadataItems{
				Key:   "funnel-config",
				Value: &confYaml,
			},
		),
	}

	// Prepare disk details by setting the specific zone
	for _, disk := range props.Disks {
		dt := localize(zone, "diskTypes", disk.InitializeParams.DiskType)
		disk.InitializeParams.DiskType = dt
	}

	instance := compute.Instance{
		Name:              w.Id,
		CanIpForward:      props.CanIpForward,
		Description:       props.Description,
		Disks:             props.Disks,
		MachineType:       localize(zone, "machineTypes", props.MachineType),
		NetworkInterfaces: props.NetworkInterfaces,
		Scheduling:        props.Scheduling,
		ServiceAccounts:   props.ServiceAccounts,
		Tags:              props.Tags,
		Metadata:          &metadata,
	}

	op, ierr := s.svc.Instances.Insert(project, zone, &instance).Do()
	if ierr != nil {
		log.Error("Couldn't insert GCE VM instance", ierr)
		return ierr
	}

	log.Debug("GCE VM instance created", "details", op)
	return nil
}

func localize(zone, resourceType, val string) string {
	return fmt.Sprintf("zones/%s/%s/%s", zone, resourceType, val)
}
<|endoftext|>