<commit_before>// Copyright 2015 Martin Hebnes Pedersen (LA5NTA). All rights reserved.
// Use of this source code is governed by the MIT-license that can be
// found in the LICENSE file.

package ardop

import (
	"fmt"
	"io"
	"net"
)

type listener struct {
	incoming <-chan net.Conn
	quit     chan struct{}
	errors   <-chan error
	addr     Addr
}

func (l listener) Accept() (c net.Conn, err error) {
	select {
	case c, ok := <-l.incoming:
		if !ok {
			return nil, io.EOF
		}
		return c, nil
	case err = <-l.errors:
		return nil, err
	}
}

func (l listener) Addr() net.Addr {
	return l.addr
}

func (l listener) Close() error {
	close(l.quit)
	return nil
}

func (tnc *TNC) Listen() (ln net.Listener, err error) {
	if tnc.closed {
		return nil, ErrTNCClosed
	}

	if tnc.listenerActive {
		return nil, ErrActiveListenerExists
	}
	tnc.listenerActive = true

	incoming := make(chan net.Conn)
	quit := make(chan struct{})
	errors := make(chan error)

	mycall, err := tnc.MyCall()
	if err != nil {
		return nil, fmt.Errorf("Unable to get mycall: %s", err)
	}

	if err := tnc.SetListenEnabled(true); err != nil {
		return nil, fmt.Errorf("TNC failed to enable listening: %s", err)
	}

	go func() {
		defer func() {
			close(incoming) // Important to close this first!
			close(errors)
			tnc.listenerActive = false
		}()

		msgListener := tnc.in.Listen()
		defer msgListener.Close()
		msgs := msgListener.Msgs()

		var remotecall, targetcall string
		for {
			select {
			case <-quit:
				tnc.SetListenEnabled(false) // Should return this in listener.Close()
				errors <- fmt.Errorf("Closed")
				return
			case msg, ok := <-msgs:
				switch {
				case !ok:
					errors <- ErrTNCClosed
					return
				case msg.cmd == cmdCancelPending:
					remotecall, targetcall = "", ""
				case msg.cmd == cmdConnected:
					remotecall = msg.value.([]string)[0]
				case msg.cmd == cmdTarget:
					targetcall = msg.String()
				}

				if len(remotecall) > 0 && len(targetcall) > 0 {
					tnc.data = &tncConn{
						remoteAddr: Addr{remotecall},
						localAddr:  Addr{targetcall},
						ctrlOut:    tnc.out,
						dataOut:    tnc.dataOut,
						ctrlIn:     tnc.in,
						dataIn:     tnc.dataIn,
						eofChan:    make(chan struct{}),
						isTCP:      tnc.isTCP,
					}

					tnc.connected = true
					incoming <- tnc.data

					remotecall, targetcall = "", ""
				}
			}
		}
	}()

	return listener{incoming, quit, errors, Addr{mycall}}, nil
}
<commit_msg>ardop: Fix bug in Listener<commit_after>// Copyright 2015 Martin Hebnes Pedersen (LA5NTA). All rights reserved.
// Use of this source code is governed by the MIT-license that can be
// found in the LICENSE file.

package ardop

import (
	"fmt"
	"io"
	"net"
)

type listener struct {
	incoming <-chan net.Conn
	quit     chan struct{}
	errors   <-chan error
	addr     Addr
}

func (l listener) Accept() (c net.Conn, err error) {
	select {
	case c, ok := <-l.incoming:
		if !ok {
			return nil, io.EOF
		}
		return c, nil
	case err = <-l.errors:
		return nil, err
	}
}

func (l listener) Addr() net.Addr {
	return l.addr
}

func (l listener) Close() error {
	close(l.quit)
	return nil
}

func (tnc *TNC) Listen() (ln net.Listener, err error) {
	if tnc.closed {
		return nil, ErrTNCClosed
	}

	if tnc.listenerActive {
		return nil, ErrActiveListenerExists
	}
	tnc.listenerActive = true

	incoming := make(chan net.Conn)
	quit := make(chan struct{})
	errors := make(chan error)

	mycall, err := tnc.MyCall()
	if err != nil {
		return nil, fmt.Errorf("Unable to get mycall: %s", err)
	}

	if err := tnc.SetListenEnabled(true); err != nil {
		return nil, fmt.Errorf("TNC failed to enable listening: %s", err)
	}

	go func() {
		defer func() {
			close(incoming) // Important to close this first!
			close(errors)
			tnc.listenerActive = false
		}()

		msgListener := tnc.in.Listen()
		defer msgListener.Close()
		msgs := msgListener.Msgs()

		var targetcall string
		for {
			select {
			case <-quit:
				tnc.SetListenEnabled(false) // Should return this in listener.Close()
				errors <- fmt.Errorf("Closed")
				return
			case msg, ok := <-msgs:
				if !ok {
					errors <- ErrTNCClosed
					return
				}
				switch msg.cmd {
				case cmdCancelPending, cmdDisconnected:
					targetcall = "" // Reset
				case cmdTarget:
					targetcall = msg.String()
				case cmdConnected:
					if targetcall == "" {
						// This can not be an incoming connection.
						// Incoming connections always gets cmdTarget before cmdConnected according to the spec
						continue
					}
					remotecall := msg.value.([]string)[0]
					tnc.data = &tncConn{
						remoteAddr: Addr{remotecall},
						localAddr:  Addr{targetcall},
						ctrlOut:    tnc.out,
						dataOut:    tnc.dataOut,
						ctrlIn:     tnc.in,
						dataIn:     tnc.dataIn,
						eofChan:    make(chan struct{}),
						isTCP:      tnc.isTCP,
					}
					tnc.connected = true
					incoming <- tnc.data
					targetcall = ""
				}
			}
		}
	}()

	return listener{incoming, quit, errors, Addr{mycall}}, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package zoekt

import (
	"bytes"
	"log"
	"sort"
	"unicode/utf8"
)

var _ = log.Println

// contentProvider is an abstraction to treat matches for names and
// content with the same code.
type contentProvider struct {
	id    *indexData
	stats *Stats

	// mutable
	err      error
	idx      uint32
	_data    []byte
	_nl      []uint32
	_nlBuf   []uint32
	_sects   []DocumentSection
	_sectBuf []DocumentSection
	fileSize uint32
}

// setDocument skips to the given document.
func (p *contentProvider) setDocument(docID uint32) {
	fileStart := p.id.boundaries[docID]

	p.idx = docID
	p.fileSize = p.id.boundaries[docID+1] - fileStart

	p._nl = nil
	p._sects = nil
	p._data = nil
}

func (p *contentProvider) docSections() []DocumentSection {
	if p._sects == nil {
		var sz uint32
		p._sects, sz, p.err = p.id.readDocSections(p.idx, p._sectBuf)
		p.stats.ContentBytesLoaded += int64(sz)
		p._sectBuf = p._sects
	}
	return p._sects
}

func (p *contentProvider) newlines() []uint32 {
	if p._nl == nil {
		var sz uint32
		p._nl, sz, p.err = p.id.readNewlines(p.idx, p._nlBuf)
		p._nlBuf = p._nl
		p.stats.ContentBytesLoaded += int64(sz)
	}
	return p._nl
}

func (p *contentProvider) data(fileName bool) []byte {
	if fileName {
		return p.id.fileNameContent[p.id.fileNameIndex[p.idx]:p.id.fileNameIndex[p.idx+1]]
	}

	if p._data == nil {
		p._data, p.err = p.id.readContents(p.idx)
		p.stats.FilesLoaded++
		p.stats.ContentBytesLoaded += int64(len(p._data))
	}
	return p._data
}

// Find offset in bytes (relative to corpus start) for an offset in
// runes (relative to document start). If filename is set, the corpus
// is the set of filenames, with the document being the name itself.
func (p *contentProvider) findOffset(filename bool, r uint32) uint32 {
	if p.id.metaData.PlainASCII {
		return r
	}

	sample := p.id.runeOffsets
	runeEnds := p.id.fileEndRunes
	fileStartByte := p.id.boundaries[p.idx]
	if filename {
		sample = p.id.fileNameRuneOffsets
		runeEnds = p.id.fileNameEndRunes
		fileStartByte = p.id.fileNameIndex[p.idx]
	}

	absR := r
	if p.idx > 0 {
		absR += runeEnds[p.idx-1]
	}

	byteOff := sample[absR/runeOffsetFrequency]
	left := absR % runeOffsetFrequency

	var data []byte

	if filename {
		data = p.id.fileNameContent[byteOff:]
	} else {
		data, p.err = p.id.readContentSlice(byteOff, 3*runeOffsetFrequency)
		if p.err != nil {
			return 0
		}
	}
	for left > 0 {
		_, sz := utf8.DecodeRune(data)
		byteOff += uint32(sz)
		data = data[sz:]
		left--
	}

	byteOff -= fileStartByte
	return byteOff
}

func (p *contentProvider) fillMatches(ms []*candidateMatch) []LineMatch {
	var result []LineMatch
	if ms[0].fileName {
		// There is only "line" in a filename.
		res := LineMatch{
			Line:     p.id.fileName(p.idx),
			FileName: true,
		}

		for _, m := range ms {
			res.LineFragments = append(res.LineFragments, LineFragmentMatch{
				LineOffset:  int(m.byteOffset),
				MatchLength: int(m.byteMatchSz),
				Offset:      m.byteOffset,
			})

			result = []LineMatch{res}
		}
	} else {
		ms = breakMatchesOnNewlines(ms, p.data(false))
		result = p.fillContentMatches(ms)
	}

	sects := p.docSections()
	for i, m := range result {
		result[i].Score = matchScore(sects, &m)
	}

	return result
}

func (p *contentProvider) fillContentMatches(ms []*candidateMatch) []LineMatch {
	var result []LineMatch
	for len(ms) > 0 {
		m := ms[0]
		num, lineStart, lineEnd := m.line(p.newlines(), p.fileSize)

		var lineCands []*candidateMatch

		endMatch := m.byteOffset + m.byteMatchSz

		for len(ms) > 0 {
			m := ms[0]
			if int(m.byteOffset) <= lineEnd {
				endMatch = m.byteOffset + m.byteMatchSz
				lineCands = append(lineCands, m)
				ms = ms[1:]
			} else {
				break
			}
		}

		if len(lineCands) == 0 {
			log.Panicf(
				"%s %v infinite loop: num %d start,end %d,%d, offset %d",
				p.id.fileName(p.idx), p.id.metaData,
				num, lineStart, lineEnd,
				m.byteOffset)
		}

		data := p.data(false)

		// Due to merging matches, we may have a match that
		// crosses a line boundary. Prevent confusion by
		// taking lines until we pass the last match
		for lineEnd < len(data) && endMatch > uint32(lineEnd) {
			next := bytes.IndexByte(data[lineEnd+1:], '\n')
			if next == -1 {
				lineEnd = len(data)
			} else {
				// TODO(hanwen): test that checks "+1" part here.
				lineEnd += next + 1
			}
		}

		finalMatch := LineMatch{
			LineStart:  lineStart,
			LineEnd:    lineEnd,
			LineNumber: num,
		}
		finalMatch.Line = p.data(false)[lineStart:lineEnd]

		for _, m := range lineCands {
			fragment := LineFragmentMatch{
				Offset:      m.byteOffset,
				LineOffset:  int(m.byteOffset) - lineStart,
				MatchLength: int(m.byteMatchSz),
			}
			finalMatch.LineFragments = append(finalMatch.LineFragments, fragment)
		}
		result = append(result, finalMatch)
	}
	return result
}

const (
	// TODO - how to scale this relative to rank?
	scorePartialWordMatch   = 50.0
	scoreWordMatch          = 500.0
	scoreImportantThreshold = 2000.0
	scorePartialSymbol      = 4000.0
	scoreSymbol             = 7000.0
	scoreFactorAtomMatch    = 400.0
	scoreShardRankFactor    = 20.0
	scoreFileOrderFactor    = 10.0
	scoreLineOrderFactor    = 1.0
)

func findSection(secs []DocumentSection, off, sz uint32) *DocumentSection {
	j := sort.Search(len(secs), func(i int) bool {
		return secs[i].End >= off+sz
	})

	if j == len(secs) {
		return nil
	}

	if secs[j].Start <= off && off+sz <= secs[j].End {
		return &secs[j]
	}
	return nil
}

func matchScore(secs []DocumentSection, m *LineMatch) float64 {
	var maxScore float64
	for _, f := range m.LineFragments {
		startBoundary := f.LineOffset < len(m.Line) && (f.LineOffset == 0 || byteClass(m.Line[f.LineOffset-1]) != byteClass(m.Line[f.LineOffset]))

		end := int(f.LineOffset) + f.MatchLength
		endBoundary := end > 0 && (end == len(m.Line) || byteClass(m.Line[end-1]) != byteClass(m.Line[end]))

		score := 0.0
		if startBoundary && endBoundary {
			score = scoreWordMatch
		} else if startBoundary || endBoundary {
			score = scorePartialWordMatch
		}

		sec := findSection(secs, f.Offset, uint32(f.MatchLength))
		if sec != nil {
			startMatch := sec.Start == f.Offset
			endMatch := sec.End == f.Offset+uint32(f.MatchLength)
			if startMatch && endMatch {
				score += scoreSymbol
			} else if startMatch || endMatch {
				score += (scoreSymbol + scorePartialSymbol) / 2
			} else {
				score += scorePartialSymbol
			}
		}
		if score > maxScore {
			maxScore = score
		}
	}
	return maxScore
}

type matchScoreSlice []LineMatch

func (m matchScoreSlice) Len() int           { return len(m) }
func (m matchScoreSlice) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }
func (m matchScoreSlice) Less(i, j int) bool { return m[i].Score > m[j].Score }

type fileMatchSlice []FileMatch

func (m fileMatchSlice) Len() int           { return len(m) }
func (m fileMatchSlice) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }
func (m fileMatchSlice) Less(i, j int) bool { return m[i].Score > m[j].Score }

func sortMatchesByScore(ms []LineMatch) {
	sort.Sort(matchScoreSlice(ms))
}

// Sort a slice of results.
func SortFilesByScore(ms []FileMatch) {
	sort.Sort(fileMatchSlice(ms))
}
<commit_msg>Reuse existing data buffer in contentprovider<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package zoekt

import (
	"bytes"
	"log"
	"sort"
	"unicode/utf8"
)

var _ = log.Println

// contentProvider is an abstraction to treat matches for names and
// content with the same code.
type contentProvider struct {
	id    *indexData
	stats *Stats

	// mutable
	err      error
	idx      uint32
	_data    []byte
	_nl      []uint32
	_nlBuf   []uint32
	_sects   []DocumentSection
	_sectBuf []DocumentSection
	fileSize uint32
}

// setDocument skips to the given document.
func (p *contentProvider) setDocument(docID uint32) {
	fileStart := p.id.boundaries[docID]

	p.idx = docID
	p.fileSize = p.id.boundaries[docID+1] - fileStart

	p._nl = nil
	p._sects = nil
	p._data = nil
}

func (p *contentProvider) docSections() []DocumentSection {
	if p._sects == nil {
		var sz uint32
		p._sects, sz, p.err = p.id.readDocSections(p.idx, p._sectBuf)
		p.stats.ContentBytesLoaded += int64(sz)
		p._sectBuf = p._sects
	}
	return p._sects
}

func (p *contentProvider) newlines() []uint32 {
	if p._nl == nil {
		var sz uint32
		p._nl, sz, p.err = p.id.readNewlines(p.idx, p._nlBuf)
		p._nlBuf = p._nl
		p.stats.ContentBytesLoaded += int64(sz)
	}
	return p._nl
}

func (p *contentProvider) data(fileName bool) []byte {
	if fileName {
		return p.id.fileNameContent[p.id.fileNameIndex[p.idx]:p.id.fileNameIndex[p.idx+1]]
	}

	if p._data == nil {
		p._data, p.err = p.id.readContents(p.idx)
		p.stats.FilesLoaded++
		p.stats.ContentBytesLoaded += int64(len(p._data))
	}
	return p._data
}

// Find offset in bytes (relative to corpus start) for an offset in
// runes (relative to document start). If filename is set, the corpus
// is the set of filenames, with the document being the name itself.
func (p *contentProvider) findOffset(filename bool, r uint32) uint32 {
	if p.id.metaData.PlainASCII {
		return r
	}

	sample := p.id.runeOffsets
	runeEnds := p.id.fileEndRunes
	fileStartByte := p.id.boundaries[p.idx]
	if filename {
		sample = p.id.fileNameRuneOffsets
		runeEnds = p.id.fileNameEndRunes
		fileStartByte = p.id.fileNameIndex[p.idx]
	}

	absR := r
	if p.idx > 0 {
		absR += runeEnds[p.idx-1]
	}

	byteOff := sample[absR/runeOffsetFrequency]
	left := absR % runeOffsetFrequency

	var data []byte

	if filename {
		data = p.id.fileNameContent[byteOff:]
	} else {
		data, p.err = p.id.readContentSlice(byteOff, 3*runeOffsetFrequency)
		if p.err != nil {
			return 0
		}
	}
	for left > 0 {
		_, sz := utf8.DecodeRune(data)
		byteOff += uint32(sz)
		data = data[sz:]
		left--
	}

	byteOff -= fileStartByte
	return byteOff
}

func (p *contentProvider) fillMatches(ms []*candidateMatch) []LineMatch {
	var result []LineMatch
	if ms[0].fileName {
		// There is only "line" in a filename.
		res := LineMatch{
			Line:     p.id.fileName(p.idx),
			FileName: true,
		}

		for _, m := range ms {
			res.LineFragments = append(res.LineFragments, LineFragmentMatch{
				LineOffset:  int(m.byteOffset),
				MatchLength: int(m.byteMatchSz),
				Offset:      m.byteOffset,
			})

			result = []LineMatch{res}
		}
	} else {
		ms = breakMatchesOnNewlines(ms, p.data(false))
		result = p.fillContentMatches(ms)
	}

	sects := p.docSections()
	for i, m := range result {
		result[i].Score = matchScore(sects, &m)
	}

	return result
}

func (p *contentProvider) fillContentMatches(ms []*candidateMatch) []LineMatch {
	var result []LineMatch
	for len(ms) > 0 {
		m := ms[0]
		num, lineStart, lineEnd := m.line(p.newlines(), p.fileSize)

		var lineCands []*candidateMatch

		endMatch := m.byteOffset + m.byteMatchSz

		for len(ms) > 0 {
			m := ms[0]
			if int(m.byteOffset) <= lineEnd {
				endMatch = m.byteOffset + m.byteMatchSz
				lineCands = append(lineCands, m)
				ms = ms[1:]
			} else {
				break
			}
		}

		if len(lineCands) == 0 {
			log.Panicf(
				"%s %v infinite loop: num %d start,end %d,%d, offset %d",
				p.id.fileName(p.idx), p.id.metaData,
				num, lineStart, lineEnd,
				m.byteOffset)
		}

		data := p.data(false)

		// Due to merging matches, we may have a match that
		// crosses a line boundary. Prevent confusion by
		// taking lines until we pass the last match
		for lineEnd < len(data) && endMatch > uint32(lineEnd) {
			next := bytes.IndexByte(data[lineEnd+1:], '\n')
			if next == -1 {
				lineEnd = len(data)
			} else {
				// TODO(hanwen): test that checks "+1" part here.
				lineEnd += next + 1
			}
		}

		finalMatch := LineMatch{
			LineStart:  lineStart,
			LineEnd:    lineEnd,
			LineNumber: num,
		}
		finalMatch.Line = data[lineStart:lineEnd]

		for _, m := range lineCands {
			fragment := LineFragmentMatch{
				Offset:      m.byteOffset,
				LineOffset:  int(m.byteOffset) - lineStart,
				MatchLength: int(m.byteMatchSz),
			}
			finalMatch.LineFragments = append(finalMatch.LineFragments, fragment)
		}
		result = append(result, finalMatch)
	}
	return result
}

const (
	// TODO - how to scale this relative to rank?
	scorePartialWordMatch   = 50.0
	scoreWordMatch          = 500.0
	scoreImportantThreshold = 2000.0
	scorePartialSymbol      = 4000.0
	scoreSymbol             = 7000.0
	scoreFactorAtomMatch    = 400.0
	scoreShardRankFactor    = 20.0
	scoreFileOrderFactor    = 10.0
	scoreLineOrderFactor    = 1.0
)

func findSection(secs []DocumentSection, off, sz uint32) *DocumentSection {
	j := sort.Search(len(secs), func(i int) bool {
		return secs[i].End >= off+sz
	})

	if j == len(secs) {
		return nil
	}

	if secs[j].Start <= off && off+sz <= secs[j].End {
		return &secs[j]
	}
	return nil
}

func matchScore(secs []DocumentSection, m *LineMatch) float64 {
	var maxScore float64
	for _, f := range m.LineFragments {
		startBoundary := f.LineOffset < len(m.Line) && (f.LineOffset == 0 || byteClass(m.Line[f.LineOffset-1]) != byteClass(m.Line[f.LineOffset]))

		end := int(f.LineOffset) + f.MatchLength
		endBoundary := end > 0 && (end == len(m.Line) || byteClass(m.Line[end-1]) != byteClass(m.Line[end]))

		score := 0.0
		if startBoundary && endBoundary {
			score = scoreWordMatch
		} else if startBoundary || endBoundary {
			score = scorePartialWordMatch
		}

		sec := findSection(secs, f.Offset, uint32(f.MatchLength))
		if sec != nil {
			startMatch := sec.Start == f.Offset
			endMatch := sec.End == f.Offset+uint32(f.MatchLength)
			if startMatch && endMatch {
				score += scoreSymbol
			} else if startMatch || endMatch {
				score += (scoreSymbol + scorePartialSymbol) / 2
			} else {
				score += scorePartialSymbol
			}
		}
		if score > maxScore {
			maxScore = score
		}
	}
	return maxScore
}

type matchScoreSlice []LineMatch

func (m matchScoreSlice) Len() int           { return len(m) }
func (m matchScoreSlice) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }
func (m matchScoreSlice) Less(i, j int) bool { return m[i].Score > m[j].Score }

type fileMatchSlice []FileMatch

func (m fileMatchSlice) Len() int           { return len(m) }
func (m fileMatchSlice) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }
func (m fileMatchSlice) Less(i, j int) bool { return m[i].Score > m[j].Score }

func sortMatchesByScore(ms []LineMatch) {
	sort.Sort(matchScoreSlice(ms))
}

// Sort a slice of results.
func SortFilesByScore(ms []FileMatch) {
	sort.Sort(fileMatchSlice(ms))
}
<|endoftext|>


<commit_before>package bencode

import (
	"bytes"

	"fmt"
	"io"
	"reflect"
	"sort"
)

type sortValues []reflect.Value

func (p sortValues) Len() int           { return len(p) }
func (p sortValues) Less(i, j int) bool { return p[i].String() < p[j].String() }
func (p sortValues) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

type sortFields []reflect.StructField

func (p sortFields) Len() int { return len(p) }
func (p sortFields) Less(i, j int) bool {
	iName, jName := p[i].Name, p[j].Name
	if p[i].Tag.Get("bencode") != "" {
		iName = p[i].Tag.Get("bencode")
	}
	if p[j].Tag.Get("bencode") != "" {
		iName = p[j].Tag.Get("bencode")
	}
	return iName < jName
}
func (p sortFields) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

//An Encoder writes bencoded objects to an output stream.
type Encoder struct {
	w io.Writer
}

//NewEncoder returns a new encoder that writes to w.
func NewEncoder(w io.Writer) *Encoder {
	return &Encoder{w}
}

//Encode writes the bencoded data of val to its output stream.
//See the documentation for Decode about the conversion of Go values to
//bencoded data.
func (e *Encoder) Encode(val interface{}) error {
	return encodeValue(e.w, reflect.ValueOf(val))
}

//EncodeString returns the bencoded data of val as a string.
func EncodeString(val interface{}) (string, error) {
	buf := new(bytes.Buffer)
	e := NewEncoder(buf)
	if err := e.Encode(val); err != nil {
		return "", err
	}
	return buf.String(), nil
}

func encodeValue(w io.Writer, val reflect.Value) error {
	//inspect the val to check
	v := indirect(val)

	switch v.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		_, err := fmt.Fprintf(w, "i%de", v.Int())
		return err

	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		_, err := fmt.Fprintf(w, "i%de", v.Uint())
		return err

	case reflect.String:
		_, err := fmt.Fprintf(w, "%d:%s", len(v.String()), v.String())
		return err

	case reflect.Slice, reflect.Array:
		if _, err := fmt.Fprint(w, "l"); err != nil {
			return err
		}
		for i := 0; i < v.Len(); i++ {
			if err := encodeValue(w, v.Index(i)); err != nil {
				return err
			}
		}

		_, err := fmt.Fprint(w, "e")
		return err

	case reflect.Map:
		if _, err := fmt.Fprint(w, "d"); err != nil {
			return err
		}
		var (
			keys sortValues = v.MapKeys()
			mval reflect.Value
		)
		sort.Sort(keys)
		for i := range keys {
			if err := encodeValue(w, keys[i]); err != nil {
				return err
			}
			mval = v.MapIndex(keys[i])
			if err := encodeValue(w, mval); err != nil {
				return err
			}
		}
		_, err := fmt.Fprint(w, "e")
		return err

	case reflect.Struct:
		t := v.Type()
		if _, err := fmt.Fprint(w, "d"); err != nil {
			return err
		}
		//put keys into keys
		var (
			keys = make(sortFields, t.NumField())
			mval reflect.Value
			rkey reflect.Value
		)
		for i := range keys {
			keys[i] = t.Field(i)
		}
		sort.Sort(keys)
		for _, key := range keys {
			//determine if key has a tag
			if tag := key.Tag.Get("bencode"); tag != "" {
				rkey = reflect.ValueOf(tag)
			} else {
				rkey = reflect.ValueOf(key.Name)
			}

			//encode the key
			if err := encodeValue(w, rkey); err != nil {
				return err
			}
			//encode the value
			mval = v.FieldByIndex(key.Index)
			if err := encodeValue(w, mval); err != nil {
				return err
			}
		}
		_, err := fmt.Fprint(w, "e")
		return err
	}

	return fmt.Errorf("Can't encode type: %s", v.Type())
}
<commit_msg>fix for sortFields by tag<commit_after>package bencode

import (
	"bytes"

	"fmt"
	"io"
	"reflect"
	"sort"
)

type sortValues []reflect.Value

func (p sortValues) Len() int           { return len(p) }
func (p sortValues) Less(i, j int) bool { return p[i].String() < p[j].String() }
func (p sortValues) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

type sortFields []reflect.StructField

func (p sortFields) Len() int { return len(p) }
func (p sortFields) Less(i, j int) bool {
	iName, jName := p[i].Name, p[j].Name
	if p[i].Tag.Get("bencode") != "" {
		iName = p[i].Tag.Get("bencode")
	}
	if p[j].Tag.Get("bencode") != "" {
		jName = p[j].Tag.Get("bencode")
	}
	return iName < jName
}
func (p sortFields) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

//An Encoder writes bencoded objects to an output stream.
type Encoder struct {
	w io.Writer
}

//NewEncoder returns a new encoder that writes to w.
func NewEncoder(w io.Writer) *Encoder {
	return &Encoder{w}
}

//Encode writes the bencoded data of val to its output stream.
//See the documentation for Decode about the conversion of Go values to
//bencoded data.
func (e *Encoder) Encode(val interface{}) error {
	return encodeValue(e.w, reflect.ValueOf(val))
}

//EncodeString returns the bencoded data of val as a string.
func EncodeString(val interface{}) (string, error) {
	buf := new(bytes.Buffer)
	e := NewEncoder(buf)
	if err := e.Encode(val); err != nil {
		return "", err
	}
	return buf.String(), nil
}

func encodeValue(w io.Writer, val reflect.Value) error {
	//inspect the val to check
	v := indirect(val)

	switch v.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		_, err := fmt.Fprintf(w, "i%de", v.Int())
		return err

	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		_, err := fmt.Fprintf(w, "i%de", v.Uint())
		return err

	case reflect.String:
		_, err := fmt.Fprintf(w, "%d:%s", len(v.String()), v.String())
		return err

	case reflect.Slice, reflect.Array:
		if _, err := fmt.Fprint(w, "l"); err != nil {
			return err
		}
		for i := 0; i < v.Len(); i++ {
			if err := encodeValue(w, v.Index(i)); err != nil {
				return err
			}
		}

		_, err := fmt.Fprint(w, "e")
		return err

	case reflect.Map:
		if _, err := fmt.Fprint(w, "d"); err != nil {
			return err
		}
		var (
			keys sortValues = v.MapKeys()
			mval reflect.Value
		)
		sort.Sort(keys)
		for i := range keys {
			if err := encodeValue(w, keys[i]); err != nil {
				return err
			}
			mval = v.MapIndex(keys[i])
			if err := encodeValue(w, mval); err != nil {
				return err
			}
		}
		_, err := fmt.Fprint(w, "e")
		return err

	case reflect.Struct:
		t := v.Type()
		if _, err := fmt.Fprint(w, "d"); err != nil {
			return err
		}
		//put keys into keys
		var (
			keys = make(sortFields, t.NumField())
			mval reflect.Value
			rkey reflect.Value
		)
		for i := range keys {
			keys[i] = t.Field(i)
		}
		sort.Sort(keys)
		for _, key := range keys {
			//determine if key has a tag
			if tag := key.Tag.Get("bencode"); tag != "" {
				rkey = reflect.ValueOf(tag)
			} else {
				rkey = reflect.ValueOf(key.Name)
			}

			//encode the key
			if err := encodeValue(w, rkey); err != nil {
				return err
			}
			//encode the value
			mval = v.FieldByIndex(key.Index)
			if err := encodeValue(w, mval); err != nil {
				return err
			}
		}
		_, err := fmt.Fprint(w, "e")
		return err
	}

	return fmt.Errorf("Can't encode type: %s", v.Type())
}
<|endoftext|>


<commit_before>package hitGox

import (
	"bytes"
	"encoding/json"
	"github.com/valyala/fasthttp"
)

// User contains information about user.
type FeaturedMedia struct {
	MediaID          string `json:"media_id,omitempty"`
	MediaDisplayName string `json:"media_display_name,omitempty"`
	MediaName        string `json:"media_name,omitempty"`
	Backdrop         string `json:"backdrop,omitempty"`
	BackdropHTML     string `json:"backdrop_html,omitempty"`
}

// GetFeaturedMedia return information about user.
//
// When a user isn’t found, this API returns a regular response but with all values containing null.
func GetFeaturedMedia() (FeaturedMedia, error) {
	_, body, err := fasthttp.Get(nil, API+"/mediafeatured")
	if err != nil {
		return FeaturedMedia{}, err
	}
	var obj FeaturedMedia
	if err = json.NewDecoder(bytes.NewReader(body)).Decode(&obj); err != nil {
		return FeaturedMedia{}, err
	}
	return obj, nil
}
<commit_msg>✅ Approved getFeaturedMedia<commit_after>package hitGox

import (
	"bytes"
	"encoding/json"
)

// FeaturedMedia is about featured live stream.
type FeaturedMedia struct {
	MediaID          string `json:"media_id"`
	MediaDisplayName string `json:"media_display_name"`
	MediaName        string `json:"media_name"`
	Backdrop         string `json:"backdrop"`
	BackdropHTML     string `json:"backdrop_html"`
}

// GetFeaturedMedia returns a featured live stream.
func GetFeaturedMedia() (*FeaturedMedia, error) {
	url := APIEndpoint + "/mediafeatured"
	resp, err := get(url, nil)
	if err != nil {
		return nil, err
	}

	var obj FeaturedMedia
	if err = json.NewDecoder(bytes.NewReader(resp)).Decode(&obj); err != nil {
		return nil, err
	}

	return &obj, nil
}
<|endoftext|>


<commit_before>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package gateway

import (
	"fmt"
	"github.com/thethingsnetwork/core/lorawan/semtech"
	"github.com/thethingsnetwork/core/utils/log"
	"github.com/thethingsnetwork/core/utils/pointer"
	"io"
	"time"
)

type Forwarder struct {
	Id       [8]byte // Gateway's Identifier
	Logger   log.Logger
	alti     int                  // GPS altitude in RX meters
	upnb     uint                 // Number of upstream datagrams sent
	ackn     uint                 // Number of upstream datagrams that were acknowledged
	dwnb     uint                 // Number of downlink datagrams received
	lati     float64              // GPS latitude, North is +
	long     float64              // GPS longitude, East is +
	rxfw     uint                 // Number of radio packets forwarded
	rxnb     uint                 // Number of radio packets received
	adapters []io.ReadWriteCloser // List of downlink adapters
	packets  []semtech.Packet     // Downlink packets received
	acks     map[[4]byte]uint     // adapterIndex | packet.Identifier | packet.Token
	commands chan command         // Concurrent access on gateway stats
	Errors   chan error           // Done channel
}

type commandName string
type command struct {
	name commandName
	data interface{}
}

const (
	cmd_ACK     commandName = "Acknowledged"
	cmd_EMIT    commandName = "Emitted"
	cmd_RECVUP  commandName = "Radio Packet Received"
	cmd_RECVDWN commandName = "Dowlink Datagram Received"
	cmd_FWD     commandName = "Forwarded"
	cmd_FLUSH   commandName = "Flush"
	cmd_STATS   commandName = "Stats"
)

// NewForwarder create a forwarder instance bound to a set of routers.
func NewForwarder(id [8]byte, adapters ...io.ReadWriteCloser) (*Forwarder, error) {
	if len(adapters) == 0 {
		return nil, fmt.Errorf("At least one adapter must be supplied")
	}

	if len(adapters) > 255 { // cf fwd.acks
		return nil, fmt.Errorf("Cannot connect more than 255 adapters")
	}

	fwd := &Forwarder{
		Id:       id,
		Logger:   log.VoidLogger{},
		alti:     120,
		lati:     53.3702,
		long:     4.8952,
		adapters: adapters,
		packets:  make([]semtech.Packet, 0),
		acks:     make(map[[4]byte]uint),
		commands: make(chan command),
		Errors:   make(chan error, len(adapters)),
	}

	go fwd.handleCommands()

	// Star listening to each adapter Read() method
	for i, adapter := range fwd.adapters {
		go fwd.listenAdapter(adapter, i)
	}

	return fwd, nil
}

// log wraps the Logger.log method, this is nothing more than a shortcut
func (fwd Forwarder) log(format string, a ...interface{}) {
	fwd.Logger.Log(format, a...)
}

// listenAdapter listen to incoming datagrams from an adapter. Non-valid packets are ignored.
func (fwd Forwarder) listenAdapter(adapter io.ReadWriteCloser, index int) {
	for {
		buf := make([]byte, 1024)
		n, err := adapter.Read(buf)
		fwd.log("%d bytes received by adapter\n", n)
		if err != nil {
			fwd.log("Error: %+v", err)
			fwd.Errors <- err
			return // Error on reading, we assume the connection is closed / lost
		}
		fwd.log("Forwarder unmarshals datagram %x\n", buf[:n])
		packet, err := semtech.Unmarshal(buf[:n])
		if err != nil {
			fwd.log("Error: %+v", err)
			continue
		}

		switch packet.Identifier {
		case semtech.PUSH_ACK, semtech.PULL_ACK:
			fwd.commands <- command{cmd_ACK, ackToken(index, *packet)}
		case semtech.PULL_RESP:
			fwd.commands <- command{cmd_RECVDWN, packet}
		default:
			fwd.log("Forwarder ignores contingent packet %+v\n", packet)
		}

	}
}

// handleCommands acts as a monitor between all goroutines that attempt to modify the forwarder
// attributes. All sensitive operations are done by commands sent through an appropriate channel.
// This method consumes commands from the channel until it's closed.
func (fwd *Forwarder) handleCommands() {
	for cmd := range fwd.commands {
		fwd.log("Fowarder executes command: %v\n", cmd.name)

		switch cmd.name {
		case cmd_ACK:
			token := cmd.data.([4]byte)
			if fwd.acks[token] > 0 {
				fwd.acks[token] -= 1
				fwd.ackn += 1
			}
		case cmd_FWD:
			fwd.rxfw += 1
		case cmd_EMIT:
			token := cmd.data.([4]byte)
			fwd.acks[token] += 1
			fwd.upnb += 1
		case cmd_RECVUP:
			fwd.rxnb += 1
		case cmd_RECVDWN:
			fwd.dwnb += 1
			fwd.packets = append(fwd.packets, *cmd.data.(*semtech.Packet))
		case cmd_FLUSH:
			cmd.data.(chan []semtech.Packet) <- fwd.packets
			fwd.packets = make([]semtech.Packet, 0)
		case cmd_STATS:
			var ackr float64
			if fwd.upnb > 0 {
				ackr = float64(fwd.ackn) / float64(fwd.upnb)
			}
			cmd.data.(chan semtech.Stat) <- semtech.Stat{
				Ackr: &ackr,
				Alti: pointer.Int(fwd.alti),
				Dwnb: pointer.Uint(fwd.dwnb),
				Lati: pointer.Float64(fwd.lati),
				Long: pointer.Float64(fwd.long),
				Rxfw: pointer.Uint(fwd.rxfw),
				Rxnb: pointer.Uint(fwd.rxnb),
				Rxok: pointer.Uint(fwd.rxnb),
				Time: pointer.Time(time.Now()),
				Txnb: pointer.Uint(0),
			}
		}
	}
}

// Forward dispatch a packet to all connected routers.
func (fwd Forwarder) Forward(packet semtech.Packet) error {
	fwd.commands <- command{cmd_RECVUP, nil}
	if packet.Identifier != semtech.PUSH_DATA {
		return fmt.Errorf("Unable to forward with identifier %x", packet.Identifier)
	}

	raw, err := semtech.Marshal(packet)
	if err != nil {
		return err
	}

	for i, adapter := range fwd.adapters {
		n, err := adapter.Write(raw)
		if err != nil {
			return err
		}
		if n < len(raw) {
			return fmt.Errorf("Packet was too long")
		}
		fwd.commands <- command{cmd_EMIT, ackToken(i, packet)}
	}

	fwd.commands <- command{cmd_FWD, nil}
	return nil
}

// Flush spits out all downlink packet received by the forwarder since the last flush.
func (fwd Forwarder) Flush() []semtech.Packet {
	chpkt := make(chan []semtech.Packet)
	fwd.commands <- command{cmd_FLUSH, chpkt}
	return <-chpkt
}

// Stats computes and return the forwarder statistics since it was created
func (fwd Forwarder) Stats() semtech.Stat {
	chstats := make(chan semtech.Stat)
	fwd.commands <- command{cmd_STATS, chstats}
	return <-chstats
}

// Stop terminate the forwarder activity. Closing all routers connections
func (fwd Forwarder) Stop() error {
	var errors []error

	// Close the uplink adapters
	for _, adapter := range fwd.adapters {
		err := adapter.Close()
		if err != nil {
			errors = append(errors, err)
		}
	}

	if len(errors) > 0 {
		return fmt.Errorf("Unable to stop the forwarder: %+v", errors)
	}

	return nil
}
<commit_msg>[simulators.gateway] Fix stop goroutines leaks.<commit_after>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package gateway

import (
	"fmt"
	"github.com/thethingsnetwork/core/lorawan/semtech"
	"github.com/thethingsnetwork/core/utils/log"
	"github.com/thethingsnetwork/core/utils/pointer"
	"io"
	"time"
)

type Forwarder struct {
	Id       [8]byte // Gateway's Identifier
	Logger   log.Logger
	alti     int                  // GPS altitude in RX meters
	upnb     uint                 // Number of upstream datagrams sent
	ackn     uint                 // Number of upstream datagrams that were acknowledged
	dwnb     uint                 // Number of downlink datagrams received
	lati     float64              // GPS latitude, North is +
	long     float64              // GPS longitude, East is +
	rxfw     uint                 // Number of radio packets forwarded
	rxnb     uint                 // Number of radio packets received
	adapters []io.ReadWriteCloser // List of downlink adapters
	packets  []semtech.Packet     // Downlink packets received
	acks     map[[4]byte]uint     // adapterIndex | packet.Identifier | packet.Token
	commands chan command         // Concurrent access on gateway stats
	quit     chan error           // Adapter which loses connection spit here
}

type commandName string
type command struct {
	name commandName
	data interface{}
}

const (
	cmd_ACK     commandName = "Acknowledged"
	cmd_EMIT    commandName = "Emitted"
	cmd_RECVUP  commandName = "Radio Packet Received"
	cmd_RECVDWN commandName = "Dowlink Datagram Received"
	cmd_FWD     commandName = "Forwarded"
	cmd_FLUSH   commandName = "Flush"
	cmd_STATS   commandName = "Stats"
)

// NewForwarder create a forwarder instance bound to a set of routers.
func NewForwarder(id [8]byte, adapters ...io.ReadWriteCloser) (*Forwarder, error) {
	if len(adapters) == 0 {
		return nil, fmt.Errorf("At least one adapter must be supplied")
	}

	if len(adapters) > 255 { // cf fwd.acks
		return nil, fmt.Errorf("Cannot connect more than 255 adapters")
	}

	fwd := &Forwarder{
		Id:       id,
		Logger:   log.VoidLogger{},
		alti:     120,
		lati:     53.3702,
		long:     4.8952,
		adapters: adapters,
		packets:  make([]semtech.Packet, 0),
		acks:     make(map[[4]byte]uint),
		commands: make(chan command),
		quit:     make(chan error, len(adapters)),
	}

	go fwd.handleCommands()

	// Star listening to each adapter Read() method
	for i, adapter := range fwd.adapters {
		go fwd.listenAdapter(adapter, i)
	}

	return fwd, nil
}

// log wraps the Logger.log method, this is nothing more than a shortcut
func (fwd Forwarder) log(format string, a ...interface{}) {
	fwd.Logger.Log(format, a...) // NOTE: concurrent-safe ?
}

// listenAdapter listen to incoming datagrams from an adapter. Non-valid packets are ignored.
func (fwd Forwarder) listenAdapter(adapter io.ReadWriteCloser, index int) {
	for {
		buf := make([]byte, 1024)
		n, err := adapter.Read(buf)
		fwd.log("%d bytes received by adapter\n", n)
		if err != nil {
			fwd.log("Error: %+v", err)
			fwd.quit <- err
			return // Connection lost / closed
		}
		fwd.log("Forwarder unmarshals datagram %x\n", buf[:n])
		packet, err := semtech.Unmarshal(buf[:n])
		if err != nil {
			fwd.log("Error: %+v", err)
			continue
		}

		switch packet.Identifier {
		case semtech.PUSH_ACK, semtech.PULL_ACK:
			fwd.commands <- command{cmd_ACK, ackToken(index, *packet)}
		case semtech.PULL_RESP:
			fwd.commands <- command{cmd_RECVDWN, packet}
		default:
			fwd.log("Forwarder ignores contingent packet %+v\n", packet)
		}
	}
}

// handleCommands acts as a monitor between all goroutines that attempt to modify the forwarder
// attributes. All sensitive operations are done by commands sent through an appropriate channel.
// This method consumes commands from the channel until it's closed.
func (fwd *Forwarder) handleCommands() {
	for cmd := range fwd.commands {
		fwd.log("Fowarder executes command: %v\n", cmd.name)

		switch cmd.name {
		case cmd_ACK:
			token := cmd.data.([4]byte)
			if fwd.acks[token] > 0 {
				fwd.acks[token] -= 1
				fwd.ackn += 1
			}
		case cmd_FWD:
			fwd.rxfw += 1
		case cmd_EMIT:
			token := cmd.data.([4]byte)
			fwd.acks[token] += 1
			fwd.upnb += 1
		case cmd_RECVUP:
			fwd.rxnb += 1
		case cmd_RECVDWN:
			fwd.dwnb += 1
			fwd.packets = append(fwd.packets, *cmd.data.(*semtech.Packet))
		case cmd_FLUSH:
			cmd.data.(chan []semtech.Packet) <- fwd.packets
			fwd.packets = make([]semtech.Packet, 0)
		case cmd_STATS:
			var ackr float64
			if fwd.upnb > 0 {
				ackr = float64(fwd.ackn) / float64(fwd.upnb)
			}
			cmd.data.(chan semtech.Stat) <- semtech.Stat{
				Ackr: &ackr,
				Alti: pointer.Int(fwd.alti),
				Dwnb: pointer.Uint(fwd.dwnb),
				Lati: pointer.Float64(fwd.lati),
				Long: pointer.Float64(fwd.long),
				Rxfw: pointer.Uint(fwd.rxfw),
				Rxnb: pointer.Uint(fwd.rxnb),
				Rxok: pointer.Uint(fwd.rxnb),
				Time: pointer.Time(time.Now()),
				Txnb: pointer.Uint(0),
			}
		}
	}
}

// Forward dispatch a packet to all connected routers.
func (fwd Forwarder) Forward(packet semtech.Packet) error {
	fwd.commands <- command{cmd_RECVUP, nil}
	if packet.Identifier != semtech.PUSH_DATA {
		return fmt.Errorf("Unable to forward with identifier %x", packet.Identifier)
	}

	raw, err := semtech.Marshal(packet)
	if err != nil {
		return err
	}

	for i, adapter := range fwd.adapters {
		n, err := adapter.Write(raw)
		if err != nil {
			return err
		}
		if n < len(raw) {
			return fmt.Errorf("Packet was too long")
		}
		fwd.commands <- command{cmd_EMIT, ackToken(i, packet)}
	}

	fwd.commands <- command{cmd_FWD, nil}
	return nil
}

// Flush spits out all downlink packet received by the forwarder since the last flush.
func (fwd Forwarder) Flush() []semtech.Packet {
	chpkt := make(chan []semtech.Packet)
	fwd.commands <- command{cmd_FLUSH, chpkt}
	return <-chpkt
}

// Stats computes and return the forwarder statistics since it was created
func (fwd Forwarder) Stats() semtech.Stat {
	chstats := make(chan semtech.Stat)
	fwd.commands <- command{cmd_STATS, chstats}
	return <-chstats
}

// Stop terminate the forwarder activity. Closing all routers connections
func (fwd Forwarder) Stop() error {
	var errors []error

	// Close the uplink adapters
	for _, adapter := range fwd.adapters {
		err := adapter.Close()
		if err != nil {
			errors = append(errors, err)
		}
	}

	// Wait for each adapter to terminate
	for range fwd.adapters {
		<-fwd.quit
	}

	close(fwd.commands)

	if len(errors) > 0 {
		return fmt.Errorf("Unable to stop the forwarder: %+v", errors)
	}
	return nil
}
<|endoftext|>


<commit_before>package citadel

import (
	"crypto/tls"
	"fmt"
	"io"
	"strings"
	"time"

	"github.com/samalba/dockerclient"
)

type Engine struct {
	ID     string   `json:"id,omitempty"`
	Addr   string   `json:"addr,omitempty"`
	Cpus   float64  `json:"cpus,omitempty"`
	Memory float64  `json:"memory,omitempty"`
	Labels []string `json:"labels,omitempty"`

	client       *dockerclient.DockerClient
	eventHandler EventHandler
}

func (e *Engine) Connect(config *tls.Config) error {
	c, err := dockerclient.NewDockerClient(e.Addr, config)
	if err != nil {
		return err
	}

	e.client = c

	return nil
}

func (e *Engine) SetClient(c *dockerclient.DockerClient) {
	e.client = c
}

// IsConnected returns true if the engine is connected to a remote docker API
func (e *Engine) IsConnected() bool {
	return e.client != nil
}

func (e *Engine) Pull(image string) error {
	imageInfo := ParseImageName(image)
	if err := e.client.PullImage(imageInfo.Name, imageInfo.Tag); err != nil {
		return err
	}
	return nil
}

func (e *Engine) Start(c *Container, pullImage bool) error {
	var (
		err    error
		env    = []string{}
		client = e.client
		i      = c.Image
	)
	c.Engine = e

	for k, v := range i.Environment {
		env = append(env, fmt.Sprintf("%s=%s", k, v))
	}

	env = append(env,
		fmt.Sprintf("_citadel_type=%s", i.Type),
		fmt.Sprintf("_citadel_labels=%s", strings.Join(i.Labels, ",")),
	)

	vols := make(map[string]struct{})
	binds := []string{}
	for _, v := range i.Volumes {
		if strings.Index(v, ":") > -1 {
			cv := strings.Split(v, ":")
			binds = append(binds, v)
			v = cv[1]
		}
		vols[v] = struct{}{}
	}
	config := &dockerclient.ContainerConfig{
		Hostname:     i.Hostname,
		Domainname:   i.Domainname,
		Image:        i.Name,
		Cmd:          i.Args,
		Memory:       int(i.Memory) * 1024 * 1024,
		Env:          env,
		CpuShares:    int(i.Cpus * 100.0 / e.Cpus),
		ExposedPorts: make(map[string]struct{}),
		Volumes:      vols,
	}

	links := []string{}
	for k, v := range i.Links {
		links = append(links, fmt.Sprintf("%s:%s", k, v))
	}
	hostConfig := &dockerclient.HostConfig{
		PublishAllPorts: i.Publish,
		PortBindings:    make(map[string][]dockerclient.PortBinding),
		Links:           links,
		Binds:           binds,
		RestartPolicy: dockerclient.RestartPolicy{
			Name:              i.RestartPolicy.Name,
			MaximumRetryCount: i.RestartPolicy.MaximumRetryCount,
		},
		NetworkMode: i.NetworkMode,
	}

	for _, b := range i.BindPorts {
		key := fmt.Sprintf("%d/%s", b.ContainerPort, b.Proto)
		config.ExposedPorts[key] = struct{}{}

		hostConfig.PortBindings[key] = []dockerclient.PortBinding{
			{
				HostIp:   b.HostIp,
				HostPort: fmt.Sprint(b.Port),
			},
		}
	}

	if pullImage {
		if err := e.Pull(i.Name); err != nil {
			return err
		}
	}

	if c.ID, err = client.CreateContainer(config, c.Name); err != nil {
		return err
	}

	if err := client.StartContainer(c.ID, hostConfig); err != nil {
		return err
	}

	return e.updatePortInformation(c)
}

func (e *Engine) ListImages() ([]string, error) {
	images, err := e.client.ListImages()
	if err != nil {
		return nil, err
	}

	out := []string{}

	for _, i := range images {
		for _, t := range i.RepoTags {
			out = append(out, t)
		}
	}

	return out, nil
}

func (e *Engine) updatePortInformation(c *Container) error {
	info, err := e.client.InspectContainer(c.ID)
	if err != nil {
		return err
	}

	return parsePortInformation(info, c)
}

func (e *Engine) ListContainers(all bool) ([]*Container, error) {
	out := []*Container{}

	c, err := e.client.ListContainers(all, true)
	if err != nil {
		return nil, err
	}

	for _, ci := range c {
		cc, err := FromDockerContainer(ci.Id, ci.Image, e)
		if err != nil {
			return nil, err
		}

		out = append(out, cc)
	}

	return out, nil
}

func (e *Engine) Logs(container *Container, stdout bool, stderr bool) (io.ReadCloser, error) {
	logopts := &dockerclient.LogOptions{
		Stdout:     stdout,
		Stderr:     stderr,
		Timestamps: false,
		Follow:     false,
		Tail:       0,
	}
	return e.client.ContainerLogs(container.ID, logopts)
}

func (e *Engine) Kill(container *Container, sig int) error {
	return e.client.KillContainer(container.ID)
}

func (e *Engine) Stop(container *Container) error {
	return e.client.StopContainer(container.ID, 8)
}

func (e *Engine) Restart(container *Container, timeout int) error {
	return e.client.RestartContainer(container.ID, timeout)
}

func (e *Engine) Remove(container *Container) error {
	return e.client.RemoveContainer(container.ID, true)
}

func (e *Engine) Events(h EventHandler) error {
	if e.eventHandler != nil {
		return fmt.Errorf("event handler already set")
	}
	e.eventHandler = h

	e.client.StartMonitorEvents(e.handler)

	return nil
}

func (e *Engine) String() string {
	return fmt.Sprintf("engine %s addr %s", e.ID, e.Addr)
}

func (e *Engine) handler(ev *dockerclient.Event, args ...interface{}) {
	event := &Event{
		Engine: e,
		Type:   ev.Status,
		Time:   time.Unix(int64(ev.Time), 0),
	}

	container, err := FromDockerContainer(ev.Id, ev.From, e)
	if err != nil {
		// TODO: un fuck this shit, fuckin handler
		return
	}

	event.Container = container

	e.eventHandler.Handle(event)
}
<commit_msg>ListContainers API now includes filter parameter that we don't need for the moment<commit_after>package citadel

import (
	"crypto/tls"
	"fmt"
	"io"
	"strings"
	"time"

	"github.com/samalba/dockerclient"
)

type Engine struct {
	ID     string   `json:"id,omitempty"`
	Addr   string   `json:"addr,omitempty"`
	Cpus   float64  `json:"cpus,omitempty"`
	Memory float64  `json:"memory,omitempty"`
	Labels []string `json:"labels,omitempty"`

	client       *dockerclient.DockerClient
	eventHandler EventHandler
}

func (e *Engine) Connect(config *tls.Config) error {
	c, err := dockerclient.NewDockerClient(e.Addr, config)
	if err != nil {
		return err
	}

	e.client = c

	return nil
}

func (e *Engine) SetClient(c *dockerclient.DockerClient) {
	e.client = c
}

// IsConnected returns true if the engine is connected to a remote docker API
func (e *Engine) IsConnected() bool {
	return e.client != nil
}

func (e *Engine) Pull(image string) error {
	imageInfo := ParseImageName(image)
	if err := e.client.PullImage(imageInfo.Name, imageInfo.Tag); err != nil {
		return err
	}
	return nil
}

func (e *Engine) Start(c *Container, pullImage bool) error {
	var (
		err    error
		env    = []string{}
		client = e.client
		i      = c.Image
	)
	c.Engine = e

	for k, v := range i.Environment {
		env = append(env, fmt.Sprintf("%s=%s", k, v))
	}

	env = append(env,
		fmt.Sprintf("_citadel_type=%s", i.Type),
		fmt.Sprintf("_citadel_labels=%s", strings.Join(i.Labels, ",")),
	)

	vols := make(map[string]struct{})
	binds := []string{}
	for _, v := range i.Volumes {
		if strings.Index(v, ":") > -1 {
			cv := strings.Split(v, ":")
			binds = append(binds, v)
			v = cv[1]
		}
		vols[v] = struct{}{}
	}
	config := &dockerclient.ContainerConfig{
		Hostname:     i.Hostname,
		Domainname:   i.Domainname,
		Image:        i.Name,
		Cmd:          i.Args,
		Memory:       int(i.Memory) * 1024 * 1024,
		Env:          env,
		CpuShares:    int(i.Cpus * 100.0 / e.Cpus),
		ExposedPorts: make(map[string]struct{}),
		Volumes:      vols,
	}

	links := []string{}
	for k, v := range i.Links {
		links = append(links, fmt.Sprintf("%s:%s", k, v))
	}
	hostConfig := &dockerclient.HostConfig{
		PublishAllPorts: i.Publish,
		PortBindings:    make(map[string][]dockerclient.PortBinding),
		Links:           links,
		Binds:           binds,
		RestartPolicy: dockerclient.RestartPolicy{
			Name:              i.RestartPolicy.Name,
			MaximumRetryCount: i.RestartPolicy.MaximumRetryCount,
		},
		NetworkMode: i.NetworkMode,
	}

	for _, b := range i.BindPorts {
		key := fmt.Sprintf("%d/%s", b.ContainerPort, b.Proto)
		config.ExposedPorts[key] = struct{}{}

		hostConfig.PortBindings[key] = []dockerclient.PortBinding{
			{
				HostIp:   b.HostIp,
				HostPort: fmt.Sprint(b.Port),
			},
		}
	}

	if pullImage {
		if err := e.Pull(i.Name); err != nil {
			return err
		}
	}

	if c.ID, err = client.CreateContainer(config, c.Name); err != nil {
		return err
	}

	if err := client.StartContainer(c.ID, hostConfig); err != nil {
		return err
	}

	return e.updatePortInformation(c)
}

func (e *Engine) ListImages() ([]string, error) {
	images, err := e.client.ListImages()
	if err != nil {
		return nil, err
	}

	out := []string{}

	for _, i := range images {
		for _, t := range i.RepoTags {
			out = append(out, t)
		}
	}

	return out, nil
}

func (e *Engine) updatePortInformation(c *Container) error {
	info, err := e.client.InspectContainer(c.ID)
	if err != nil {
		return err
	}

	return parsePortInformation(info, c)
}

func (e *Engine) ListContainers(all bool) ([]*Container, error) {
	out := []*Container{}

	c, err := e.client.ListContainers(all, true, "")
	if err != nil {
		return nil, err
	}

	for _, ci := range c {
		cc, err := FromDockerContainer(ci.Id, ci.Image, e)
		if err != nil {
			return nil, err
		}

		out = append(out, cc)
	}

	return out, nil
}

func (e *Engine) Logs(container *Container, stdout bool, stderr bool) (io.ReadCloser, error) {
	logopts := &dockerclient.LogOptions{
		Stdout:     stdout,
		Stderr:     stderr,
		Timestamps: false,
		Follow:     false,
		Tail:       0,
	}
	return e.client.ContainerLogs(container.ID, logopts)
}

func (e *Engine) Kill(container *Container, sig int) error {
	return e.client.KillContainer(container.ID)
}

func (e *Engine) Stop(container *Container) error {
	return e.client.StopContainer(container.ID, 8)
}

func (e *Engine) Restart(container *Container, timeout int) error {
	return e.client.RestartContainer(container.ID, timeout)
}

func (e *Engine) Remove(container *Container) error {
	return e.client.RemoveContainer(container.ID, true)
}

func (e *Engine) Events(h EventHandler) error {
	if e.eventHandler != nil {
		return fmt.Errorf("event handler already set")
	}
	e.eventHandler = h

	e.client.StartMonitorEvents(e.handler)

	return nil
}

func (e *Engine) String() string {
	return fmt.Sprintf("engine %s addr %s", e.ID, e.Addr)
}

func (e *Engine) handler(ev *dockerclient.Event, args ...interface{}) {
	event := &Event{
		Engine: e,
		Type:   ev.Status,
		Time:   time.Unix(int64(ev.Time), 0),
	}

	container, err := FromDockerContainer(ev.Id, ev.From, e)
	if err != nil {
		// TODO: un fuck this shit, fuckin handler
		return
	}

	event.Container = container

	e.eventHandler.Handle(event)
}
<|endoftext|>


<commit_before>package main

func main() {
}
<commit_msg>Throw in some initial basic code so we have something to work with<commit_after>package main

import (
	"log"

	"pault.ag/go/debian/control"
	"pault.ag/go/debian/dependency"
	"pault.ag/go/resolver"
)

func main() {
	log.SetFlags(log.Lshortfile)

	// TODO configurable path?  perhaps allow for an optional *.dsc instead?
	con, err := control.ParseControlFile("debian/control")
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}

	// TODO configurable or something
	suite := "unstable"
	arch := "amd64"
	index, err := resolver.GetBinaryIndex(
		"http://httpredir.debian.org/debian",
		suite,
		"main",
		arch,
	)
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}

	allCan := true
	allPossi := append(
		con.Source.BuildDepends.GetAllPossibilities(),
		con.Source.BuildDependsIndep.GetAllPossibilities()...,
	)

	depArch, err := dependency.ParseArch("any")
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}

	allBins := []control.BinaryIndex{}
	for _, possi := range allPossi {
		can, why, bins := index.ExplainSatisfies(*depArch, possi)
		if !can {
			log.Printf("%s: %s\n", possi.Name, why)
			allCan = false
		} else {
			// TODO more smarts for which dep out of bins to use
			allBins = append(allBins, bins[0])
		}
	}

	if !allCan {
		log.Fatalf("Unsatisfied possi; exiting.\n")
	}
}
<|endoftext|>


<commit_before>package ghosttohugo

import (
	"bytes"
	"fmt"

	jww "github.com/spf13/jwalterweatherman"
)

func cardBookmark(payload interface{}) string {
	m, ok := payload.(map[string]interface{})
	if !ok {
		jww.ERROR.Println("cardBookmark: payload not correct type")
		return ""
	}

	meta, ok := m["metadata"]
	if !ok {
		jww.ERROR.Println("cardBookmark: payload does not contain metadata")
		return ""
	}

	metadata, ok := meta.(map[string]interface{})
	if !ok {
		jww.ERROR.Println("cardBookmark: payload metadata was not correct type")
		return ""
	}

	url, ok := metadata["url"]
	if !ok {
		jww.ERROR.Println("cardBookmark: missing url")
		return ""
	}
	title, ok := metadata["title"]
	if !ok {
		jww.ERROR.Println("cardBookmark: missing title")
		return ""
	}
	description, ok := metadata["description"]
	if !ok {
		jww.ERROR.Println("cardBookmark: missing description")
		return ""
	}

	var thumbnail, icon, author, publisher, caption string
	thumb, ok := metadata["thumbnail"]
	if ok && thumb != nil {
		jww.TRACE.Println("cardBookmark: found thumbnail")
		thumbnail = stripContentFolder(thumb.(string))
	}
	iconinfo, ok := metadata["icon"]
	if ok && iconinfo != nil {
		jww.TRACE.Println("cardBookmark: found icon")
		icon = stripContentFolder(iconinfo.(string))
	}
	authorinfo, ok := metadata["author"]
	if ok && authorinfo != nil {
		jww.TRACE.Println("cardBookmark: found author")
		author = authorinfo.(string)
	}
	publisherinfo, ok := metadata["publisher"]
	if ok && publisherinfo != nil {
		jww.TRACE.Println("cardBookmark: found publisher")
		publisher = publisherinfo.(string)
	}
	capt, ok := m["caption"]
	if ok && capt != nil {
		jww.TRACE.Println("cardBookmark: found caption")
		caption = capt.(string)
	}

	return fmt.Sprintf(
		"{{< bookmark url=%q title=%q description=%q icon=%q"+
			" author=%q publisher=%q thumbnail=%q caption=%q >}}",
		url.(string),
		title.(string),
		description.(string),
		icon,
		author,
		publisher,
		thumbnail,
		caption,
	)
}

func cardCode(payload interface{}) string {
	m, ok := payload.(map[string]interface{})
	if !ok {
		jww.ERROR.Println("cardCode: payload not correct type")
		return ""
	}

	code, ok := m["code"]
	if !ok {
		jww.ERROR.Println("cardCode: missing code")
		return ""
	}

	var buf bytes.Buffer

	buf.WriteString("```")
	if lang, ok := m["language"]; ok {
		buf.WriteString(lang.(string))
	}
	buf.WriteString("\n")
	buf.WriteString(code.(string))
	buf.WriteString("\n```\n")

	return buf.String()
}

func cardEmbed(payload interface{}) string {
	m, ok := payload.(map[string]interface{})
	if !ok {
		jww.ERROR.Println("cardEmbed: payload not correct type")
		return ""
	}

	html, ok := m["html"]
	if !ok {
		jww.ERROR.Println("cardEmbed: missing html")
		return ""
	}

	return html.(string)
}
func cardGallery(payload interface{}) string {
	m, ok := payload.(map[string]interface{})
	if !ok {
		jww.ERROR.Println("cardGallery: payload not correct type")
		return ""
	}
	images, ok := m["images"]
	if !ok {
		jww.ERROR.Println("cardGallery: missing images")
		return ""
	}

	var buf bytes.Buffer
	buf.WriteString("{{< gallery")
	if caption, ok := m["caption"]; ok {
		buf.WriteString(fmt.Sprintf(" caption=\"%s\"", caption.(string)))
	}
	buf.WriteString(" >}}\n")

	for _, img := range images.([]interface{}) {
		image, ok := img.(map[string]interface{})
		if !ok {
			continue
		}
		buf.WriteString("{{< galleryImg ")
		buf.WriteString(fmt.Sprintf(" src=%q", stripContentFolder(image["src"].(string))))
		buf.WriteString(fmt.Sprintf(" width=\"%.0f\"", image["width"].(float64)))
		buf.WriteString(fmt.Sprintf(" height=\"%.0f\"", image["height"].(float64)))
		if alt, ok := image["alt"]; ok {
			buf.WriteString(fmt.Sprintf(" alt=%q", alt.(string)))
		}
		if title, ok := image["title"]; ok {
			buf.WriteString(fmt.Sprintf(" title=%q", title.(string)))
		}
		buf.WriteString(" >}}")
	}

	buf.WriteString("{{< /gallery >}}")

	return buf.String()
}

func cardHR(payload interface{}) string {
	return "---\n"
}

func cardHTML(payload interface{}) string {
	m, ok := payload.(map[string]interface{})
	if !ok {
		jww.ERROR.Println("cardHTML: payload not correct type")
		return ""
	}

	if html, ok := m["html"]; ok {
		return html.(string)
	}

	return ""
}

func cardImage(payload interface{}) string {
	m, ok := payload.(map[string]interface{})
	if !ok {
		jww.ERROR.Println("cardImage: payload not correct type")
		return ""
	}

	src, ok := m["src"]
	if !ok {
		jww.ERROR.Println("cardImage: missing src")
		return ""
	}

	if caption, ok := m["caption"]; ok {
		return fmt.Sprintf(
			"{{< figure src=\"%s\" caption=\"%s\" >}}\n",
			stripContentFolder(src.(string)),
			caption,
		)
	}

	return fmt.Sprintf(
		"{{< figure src=\"%s\" >}}\n",
		stripContentFolder(src.(string)),
	)
}

func cardMarkdown(payload interface{}) string {
	m, ok := payload.(map[string]interface{})
	if !ok {
		jww.ERROR.Println("cardMarkdown: payload not correct type")
		return ""
	}
	if markdown, ok := m["markdown"]; ok {
		return fmt.Sprintf("%s\n", markdown.(string))
	}
	return ""
}
<commit_msg>resolve nil description bug (#37)<commit_after>package ghosttohugo

import (
	"bytes"
	"fmt"

	jww "github.com/spf13/jwalterweatherman"
)

func cardBookmark(payload interface{}) string {
	m, ok := payload.(map[string]interface{})
	if !ok {
		jww.ERROR.Println("cardBookmark: payload not correct type")
		return ""
	}

	meta, ok := m["metadata"]
	if !ok {
		jww.ERROR.Println("cardBookmark: payload does not contain metadata")
		return ""
	}

	metadata, ok := meta.(map[string]interface{})
	if !ok {
		jww.ERROR.Println("cardBookmark: payload metadata was not correct type")
		return ""
	}

	url, ok := metadata["url"]
	if !ok {
		jww.ERROR.Println("cardBookmark: missing url")
		return ""
	}
	title, ok := metadata["title"]
	if !ok {
		jww.ERROR.Println("cardBookmark: missing title")
		return ""
	}
	description, ok := metadata["description"]
	if !ok || description == nil {
		jww.ERROR.Println("cardBookmark: missing description")
		return ""
	}

	var thumbnail, icon, author, publisher, caption string
	thumb, ok := metadata["thumbnail"]
	if ok && thumb != nil {
		jww.TRACE.Println("cardBookmark: found thumbnail")
		thumbnail = stripContentFolder(thumb.(string))
	}
	iconinfo, ok := metadata["icon"]
	if ok && iconinfo != nil {
		jww.TRACE.Println("cardBookmark: found icon")
		icon = stripContentFolder(iconinfo.(string))
	}
	authorinfo, ok := metadata["author"]
	if ok && authorinfo != nil {
		jww.TRACE.Println("cardBookmark: found author")
		author = authorinfo.(string)
	}
	publisherinfo, ok := metadata["publisher"]
	if ok && publisherinfo != nil {
		jww.TRACE.Println("cardBookmark: found publisher")
		publisher = publisherinfo.(string)
	}
	capt, ok := m["caption"]
	if ok && capt != nil {
		jww.TRACE.Println("cardBookmark: found caption")
		caption = capt.(string)
	}

	return fmt.Sprintf(
		"{{< bookmark url=%q title=%q description=%q icon=%q"+
			" author=%q publisher=%q thumbnail=%q caption=%q >}}",
		url.(string),
		title.(string),
		description.(string),
		icon,
		author,
		publisher,
		thumbnail,
		caption,
	)
}

func cardCode(payload interface{}) string {
	m, ok := payload.(map[string]interface{})
	if !ok {
		jww.ERROR.Println("cardCode: payload not correct type")
		return ""
	}

	code, ok := m["code"]
	if !ok {
		jww.ERROR.Println("cardCode: missing code")
		return ""
	}

	var buf bytes.Buffer

	buf.WriteString("```")
	if lang, ok := m["language"]; ok {
		buf.WriteString(lang.(string))
	}
	buf.WriteString("\n")
	buf.WriteString(code.(string))
	buf.WriteString("\n```\n")

	return buf.String()
}

func cardEmbed(payload interface{}) string {
	m, ok := payload.(map[string]interface{})
	if !ok {
		jww.ERROR.Println("cardEmbed: payload not correct type")
		return ""
	}

	html, ok := m["html"]
	if !ok {
		jww.ERROR.Println("cardEmbed: missing html")
		return ""
	}

	return html.(string)
}
func cardGallery(payload interface{}) string {
	m, ok := payload.(map[string]interface{})
	if !ok {
		jww.ERROR.Println("cardGallery: payload not correct type")
		return ""
	}
	images, ok := m["images"]
	if !ok {
		jww.ERROR.Println("cardGallery: missing images")
		return ""
	}

	var buf bytes.Buffer
	buf.WriteString("{{< gallery")
	if caption, ok := m["caption"]; ok {
		buf.WriteString(fmt.Sprintf(" caption=\"%s\"", caption.(string)))
	}
	buf.WriteString(" >}}\n")

	for _, img := range images.([]interface{}) {
		image, ok := img.(map[string]interface{})
		if !ok {
			continue
		}
		buf.WriteString("{{< galleryImg ")
		buf.WriteString(fmt.Sprintf(" src=%q", stripContentFolder(image["src"].(string))))
		buf.WriteString(fmt.Sprintf(" width=\"%.0f\"", image["width"].(float64)))
		buf.WriteString(fmt.Sprintf(" height=\"%.0f\"", image["height"].(float64)))
		if alt, ok := image["alt"]; ok {
			buf.WriteString(fmt.Sprintf(" alt=%q", alt.(string)))
		}
		if title, ok := image["title"]; ok {
			buf.WriteString(fmt.Sprintf(" title=%q", title.(string)))
		}
		buf.WriteString(" >}}")
	}

	buf.WriteString("{{< /gallery >}}")

	return buf.String()
}

func cardHR(payload interface{}) string {
	return "---\n"
}

func cardHTML(payload interface{}) string {
	m, ok := payload.(map[string]interface{})
	if !ok {
		jww.ERROR.Println("cardHTML: payload not correct type")
		return ""
	}

	if html, ok := m["html"]; ok {
		return html.(string)
	}

	return ""
}

func cardImage(payload interface{}) string {
	m, ok := payload.(map[string]interface{})
	if !ok {
		jww.ERROR.Println("cardImage: payload not correct type")
		return ""
	}

	src, ok := m["src"]
	if !ok {
		jww.ERROR.Println("cardImage: missing src")
		return ""
	}

	if caption, ok := m["caption"]; ok {
		return fmt.Sprintf(
			"{{< figure src=\"%s\" caption=\"%s\" >}}\n",
			stripContentFolder(src.(string)),
			caption,
		)
	}

	return fmt.Sprintf(
		"{{< figure src=\"%s\" >}}\n",
		stripContentFolder(src.(string)),
	)
}

func cardMarkdown(payload interface{}) string {
	m, ok := payload.(map[string]interface{})
	if !ok {
		jww.ERROR.Println("cardMarkdown: payload not correct type")
		return ""
	}
	if markdown, ok := m["markdown"]; ok {
		return fmt.Sprintf("%s\n", markdown.(string))
	}
	return ""
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/koding/kite"
	"fmt"
	"os"
	"sync"
	"log"
	"github.com/bitly/go-nsq"
	"runtime"
	"time"
)

var GameState [][]uint8
var XSize uint8
var YSize uint8

func main() {
	fmt.Println("Starting Controller")
	XSize = 25
	YSize = 25
	reset()
	runtime.GOMAXPROCS(2)
	wg := &sync.WaitGroup{}
	wg.Add(2)
	go startKite()
	go startMessaging()
	wg.Wait()
}

func startKite() {
	fmt.Println("Controller starting kite")
	k := kite.New("controller", "1.0.0")
	k.Config.Port = 6001
	k.Config.DisableAuthentication = true
	k.HandleFunc("hello", hello)
	k.Run()
}

func startMessaging() {
	fmt.Println("Controller starting NSQ")
	config := nsq.NewConfig()

  	w, _ := nsq.NewProducer(os.Getenv("MESSAGING_SERVICE_HOST") + ":4150", config)

	for {
		tick(w)
		time.Sleep(time.Second * 1)
	}
	w.Stop()

}

func tick(w *nsq.Producer) {
	evaluate()
	err := w.Publish("tick", []byte("test"))
	if err != nil {
		log.Panic("Could not connect")
	}

}

func hello(r *kite.Request) (interface{}, error) {

	fmt.Println("Controller got hello")

	// You can return anything as result, as long as it is JSON marshalable.
	return nil, nil
}

func update(r *kite.Request) (interface{}, error) {
        fmt.Println("Controller received state update")
        return nil, nil
}

func getState(r *kite.Request) (interface{}, error) {
        fmt.Println("Controller received state request")
        return GameState, nil
}

func reset() {
	fmt.Println("Resetting Game State")
	// Allocate the top-level slice.
	GameState := make([][]uint8, YSize) // One row per unit of y.
	// Loop over the rows, allocating the slice for each row.
	for i := range GameState {
		GameState[i] = make([]uint8, XSize)
		for j := range GameState[i] {
			GameState[i][j] = 1
		}
	}
}

func evaluate() {
	fmt.Println("Evaluting Game State")
	for i := range GameState {
		for j := range GameState[i] {
			fmt.Print(GameState[i][j])
		}
		fmt.Println()
	}
}

<commit_msg>adding<commit_after>package main

import (
	"github.com/koding/kite"
	"fmt"
	"os"
	"sync"
	"log"
	"github.com/bitly/go-nsq"
	"runtime"
	"time"
)

var GameState [][]uint8
var XSize uint8
var YSize uint8

func main() {
	fmt.Println("Starting Controller")
	XSize = 25
	YSize = 25
	reset()
	evaluate()
	runtime.GOMAXPROCS(2)
	wg := &sync.WaitGroup{}
	wg.Add(2)
	go startKite()
	go startMessaging()
	wg.Wait()
}

func startKite() {
	fmt.Println("Controller starting kite")
	k := kite.New("controller", "1.0.0")
	k.Config.Port = 6001
	k.Config.DisableAuthentication = true
	k.HandleFunc("hello", hello)
	k.Run()
}

func startMessaging() {
	fmt.Println("Controller starting NSQ")
	config := nsq.NewConfig()

  	w, _ := nsq.NewProducer(os.Getenv("MESSAGING_SERVICE_HOST") + ":4150", config)

	for {
		tick(w)
		time.Sleep(time.Second * 1)
	}
	w.Stop()

}

func tick(w *nsq.Producer) {
	evaluate()
	err := w.Publish("tick", []byte("test"))
	if err != nil {
		log.Panic("Could not connect")
	}

}

func hello(r *kite.Request) (interface{}, error) {

	fmt.Println("Controller got hello")

	// You can return anything as result, as long as it is JSON marshalable.
	return nil, nil
}

func update(r *kite.Request) (interface{}, error) {
        fmt.Println("Controller received state update")
        return nil, nil
}

func getState(r *kite.Request) (interface{}, error) {
        fmt.Println("Controller received state request")
        return GameState, nil
}

func reset() {
	fmt.Println("Resetting Game State")
	// Allocate the top-level slice.
	GameState := make([][]uint8, YSize) // One row per unit of y.
	// Loop over the rows, allocating the slice for each row.
	for i := range GameState {
		GameState[i] = make([]uint8, XSize)
		for j := range GameState[i] {
			GameState[i][j] = 1
		}
	}
}

func evaluate() {
	fmt.Println("Evaluting Game State")
	for i := range GameState {
		for j := range GameState[i] {
			fmt.Print(GameState[i][j])
		}
		fmt.Println()
	}
}

<|endoftext|>


<commit_before>// Copyright 2017 ibelie, Chen Jie, Joungtao. All rights reserved.
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.

package rpc

import (
	"bytes"
	"fmt"
	"sort"

	"io/ioutil"
)

var (
	ENTITY_PKG = map[string]string{
		"github.com/ibelie/microserver": "",
		"github.com/ibelie/ruid":        "",
		"github.com/ibelie/tygo":        "",
	}
)

const (
	ENTITY_CODE = `

type Entity struct {
	tygo.Tygo
	ruid.RUID
	Key        ruid.RUID
	Type       string
	cachedSize int
}

func (e *Entity) MaxFieldNum() int {
	return 1
}

func (e *Entity) ByteSize() (size int) {
	if e != nil {
		e.cachedSize = tygo.SizeVarint(e.RUID) + tygo.SizeVarint(e.Key) + tygo.SizeVarint(microserver.EntityType(e.Type))
		size = 1 + tygo.SizeVarint(e.cachedSize) + e.cachedSize
		e.SetCachedSize(size)
	}
	return
}

func (e *Entity) Serialize(output *tygo.ProtoBuf) {
	if e != nil {
		output.WriteBytes(10) // tag: 10 MAKE_TAG(1, WireBytes=2)
		output.WriteVarint(e.cachedSize)
		output.WriteVarint(e.RUID)
		output.WriteVarint(e.Key)
		output.WriteVarint(microserver.EntityType(e.Type))
	}
}

func (e *Entity) Deserialize(input *tygo.ProtoBuf) (err error) {
	for !input.ExpectEnd() {
		var tag int
		var cutoff bool
		if tag, cutoff, err = input.ReadTag(127); err == nil && cutoff && tag == 10 { // MAKE_TAG(1, WireBytes=2)
			if x, e := input.ReadBuf(); e == nil {
				tmpi := &tygo.ProtoBuf{Buffer: x}
				if x, e := tmpi.ReadVarint(); e == nil {
					e.RUID = ruid.RUID(x)
				} else {
					err = e
				}
				if x, e := tmpi.ReadVarint(); e == nil {
					e.Key = ruid.RUID(x)
				} else {
					err = e
				}
				if x, e := tmpi.ReadVarint(); e == nil {
					e.Type = microserver.EntityName(x)
				} else {
					err = e
				}
			} else {
				err = e
			}
			return
		} else if err = input.SkipField(tag); err != nil {
			return
		}
	}
	return
}
`
)

func Entity(path string) {
	pkgname, depends := Extract(path)
	fmt.Println(depends)

	var head bytes.Buffer
	var body bytes.Buffer
	head.Write([]byte(fmt.Sprintf(`// Generated by ibelie-rpc.  DO NOT EDIT!

package %s
`, pkgname)))
	body.Write([]byte(ENTITY_CODE))

	pkgs := ENTITY_PKG
	var sortedPkg []string
	for path, _ := range pkgs {
		sortedPkg = append(sortedPkg, path)
	}
	sort.Strings(sortedPkg)
	for _, path := range sortedPkg {
		head.Write([]byte(fmt.Sprintf(`
import %s"%s"`, pkgs[path], path)))
	}

	head.Write(body.Bytes())
	ioutil.WriteFile(SRC_PATH+path+"/entity.rpc.go", head.Bytes(), 0666)
}
<commit_msg>modify entity injector<commit_after>// Copyright 2017 ibelie, Chen Jie, Joungtao. All rights reserved.
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.

package rpc

import (
	"bytes"
	"fmt"
	"sort"

	"io/ioutil"
)

var (
	ENTITY_PKG = map[string]string{
		"github.com/ibelie/ruid": "",
		"github.com/ibelie/tygo": "",
	}
)

const (
	ENTITY_CODE = `

type Entity struct {
	tygo.Tygo
	ruid.RUID
	Key  ruid.RUID
	Type uint64
	cachedSize int
}

func (e *Entity) MaxFieldNum() int {
	return 1
}

func (e *Entity) ByteSize() (size int) {
	if e != nil {
		e.cachedSize = tygo.SizeVarint(e.RUID) + tygo.SizeVarint(e.Key) + tygo.SizeVarint(e.Type)
		size = 1 + tygo.SizeVarint(e.cachedSize) + e.cachedSize
		e.SetCachedSize(size)
	}
	return
}

func (e *Entity) Serialize(output *tygo.ProtoBuf) {
	if e != nil {
		output.WriteBytes(10) // tag: 10 MAKE_TAG(1, WireBytes=2)
		output.WriteVarint(e.cachedSize)
		output.WriteVarint(e.RUID)
		output.WriteVarint(e.Key)
		output.WriteVarint(e.Type)
	}
}

func (e *Entity) Deserialize(input *tygo.ProtoBuf) (err error) {
	for !input.ExpectEnd() {
		var tag int
		var cutoff bool
		if tag, cutoff, err = input.ReadTag(127); err == nil && cutoff && tag == 10 { // MAKE_TAG(1, WireBytes=2)
			var buffer []byte
			if buffer, err = input.ReadBuf(); err == nil {
				var i, k uint64
				tmpi := &tygo.ProtoBuf{Buffer: buffer}
				if i, err = tmpi.ReadVarint(); err != nil {
					return
				} else if k, err = tmpi.ReadVarint(); err != nil {
					return
				} else if e.Type, err = tmpi.ReadVarint(); err != nil {
					return
				}
				e.RUID = i
				e.Key  = k
			}
			return
		} else if err = input.SkipField(tag); err != nil {
			return
		}
	}
	return
}
`
)

func Entity(path string) {
	pkgname, depends := Extract(path)
	fmt.Println(depends)

	var head bytes.Buffer
	var body bytes.Buffer
	head.Write([]byte(fmt.Sprintf(`// Generated by ibelie-rpc.  DO NOT EDIT!

package %s
`, pkgname)))
	body.Write([]byte(ENTITY_CODE))

	pkgs := ENTITY_PKG
	var sortedPkg []string
	for path, _ := range pkgs {
		sortedPkg = append(sortedPkg, path)
	}
	sort.Strings(sortedPkg)
	for _, path := range sortedPkg {
		head.Write([]byte(fmt.Sprintf(`
import %s"%s"`, pkgs[path], path)))
	}

	head.Write(body.Bytes())
	ioutil.WriteFile(SRC_PATH+path+"/entity.rpc.go", head.Bytes(), 0666)
}
<|endoftext|>


<commit_before>package goparse

import (
	"go/ast"
	"go/parser"
	"go/token"
	"reflect"
	"strings"
)

// StructDesc contains description of parsed struct
type StructDesc struct {
	Name  string
	Field []struct {
		Name string
		Type string
		Tags []string
	}
}

// GetFileStructs returns structs descriptions from parsed go file
func GetFileStructs(filename string, prefix string, tag string) ([]StructDesc, error) {
	result := make([]StructDesc, 0, 5)

	fset := token.NewFileSet()

	f, err := parser.ParseFile(fset, filename, nil, 0)
	if nil != err {
		return result, err
	}

	for i := range f.Decls {
		if g, ok := f.Decls[i].(*ast.GenDecl); ok {
			for _, s := range g.Specs {
				if ts, ok := s.(*ast.TypeSpec); ok {
					if "" == prefix || strings.HasPrefix(ts.Name.String(), prefix) {
						if tt, ok := ts.Type.(*ast.StructType); ok {
							newStruct := StructDesc{Name: ts.Name.String(), Field: make([]struct {
								Name string
								Type string
								Tags []string
							}, 0, len(tt.Fields.List))}
							for _, field := range tt.Fields.List {
								newField := struct {
									Name string
									Type string
									Tags []string
								}{}
								if len(field.Names) < 1 {
									continue
								}
								newField.Name = field.Names[0].Name
								switch e := field.Type.(type) {
								case *ast.Ident:
									newField.Type = e.Name
								case *ast.ArrayType:
									if e2, ok := e.Elt.(*ast.Ident); ok {
										newField.Type = "[]" + e2.Name
									}
								case *ast.StarExpr:
									if e2, ok := e.X.(*ast.Ident); ok {
										newField.Type = "*" + e2.Name
									}
								}
								if nil != field.Tag {
									newField.Tags = strings.Split(reflect.StructTag(strings.Trim(field.Tag.Value, "`")).Get(tag), ",")
								}
								newStruct.Field = append(newStruct.Field, newField)
							}
							result = append(result, newStruct)
						}
					}
				}
			}
		}
	}

	return result, nil
}
<commit_msg>Support for "recursive" types<commit_after>package goparse

import (
	"go/ast"
	"go/parser"
	"go/token"
	"reflect"
	"strings"
)

// StructDesc contains description of parsed struct
type StructDesc struct {
	Name  string
	Field []struct {
		Name string
		Type string
		Tags []string
	}
}

func getTypeName(t ast.Expr) string {
	switch e := t.(type) {
	case *ast.Ident:
		return e.Name
	case *ast.ArrayType:
		return "[]" + getTypeName(e.Elt)
	case *ast.StarExpr:
		return "*" + getTypeName(e.X)
	}
	return "unknown"
}

// GetFileStructs returns structs descriptions from parsed go file
func GetFileStructs(filename string, prefix string, tag string) ([]StructDesc, error) {
	result := make([]StructDesc, 0, 5)

	fset := token.NewFileSet()

	f, err := parser.ParseFile(fset, filename, nil, 0)
	if nil != err {
		return result, err
	}

	for i := range f.Decls {
		if g, ok := f.Decls[i].(*ast.GenDecl); ok {
			for _, s := range g.Specs {
				if ts, ok := s.(*ast.TypeSpec); ok {
					if "" == prefix || strings.HasPrefix(ts.Name.String(), prefix) {
						if tt, ok := ts.Type.(*ast.StructType); ok {
							newStruct := StructDesc{Name: ts.Name.String(), Field: make([]struct {
								Name string
								Type string
								Tags []string
							}, 0, len(tt.Fields.List))}
							for _, field := range tt.Fields.List {
								newField := struct {
									Name string
									Type string
									Tags []string
								}{}
								if len(field.Names) < 1 {
									continue
								}
								newField.Name = field.Names[0].Name
								newField.Type = getTypeName(field.Type)
								if nil != field.Tag {
									newField.Tags = strings.Split(reflect.StructTag(strings.Trim(field.Tag.Value, "`")).Get(tag), ",")
								}
								newStruct.Field = append(newStruct.Field, newField)
							}
							result = append(result, newStruct)
						}
					}
				}
			}
		}
	}

	return result, nil
}
<|endoftext|>


<commit_before>package gopcap

import (
	"bytes"
	"io"
	"time"
)

// checkMagicNum checks the first four bytes of a pcap file, searching for the magic number
// and checking the byte order. Returns three values: whether the file is a pcap file, whether
// the byte order needs flipping, and any error that was encountered. If error is returned,
// the other values are invalid.
func checkMagicNum(src io.Reader) (bool, bool, error) {
	// These magic numbers form the header of a pcap file.
	magic := []byte{0xa1, 0xb2, 0xc3, 0xd4}
	magic_reverse := []byte{0xd4, 0xc3, 0xb2, 0xa1}

	buffer := make([]byte, 4)
	read_count, err := src.Read(buffer)

	if err != nil {
		return false, false, err
	}
	if read_count != 4 {
		return false, false, InsufficientLength
	}

	if bytes.Compare(buffer, magic) == 0 {
		return true, false, nil
	} else if bytes.Compare(buffer, magic_reverse) == 0 {
		return true, true, nil
	}

	return false, false, NotAPcapFile
}

// parsePacket parses a full packet out of the pcap file. It returns an error if any problems were
// encountered.
func parsePacket(pkt *Packet, src io.Reader, flipped bool) error {
	err := populatePacketHeader(pkt, src, flipped)

	if err != nil {
		return err
	}

	data := make([]byte, pkt.IncludedLen)
	readlen, err := src.Read(data)
	pkt.Data = data

	if uint32(readlen) != pkt.IncludedLen {
		err = UnexpectedEOF
	}

	return err
}

// populateFileHeader reads the next 20 bytes out of the .pcap file and uses it to populate the
// PcapFile structure.
func populateFileHeader(file *PcapFile, src io.Reader, flipped bool) error {
	buffer := make([]byte, 20)
	read_count, err := src.Read(buffer)

	if err != nil {
		return err
	} else if read_count != 20 {
		return InsufficientLength
	}

	// First two bytes are the major version number.
	file.MajorVersion = getUint16(buffer[0:2], flipped)

	// Next two are the minor version number.
	file.MinorVersion = getUint16(buffer[2:4], flipped)

	// GMT to local correction, in seconds east of UTC.
	file.TZCorrection = getInt32(buffer[4:8], flipped)

	// Next is the number of significant figures in the timestamps. Almost always zero.
	file.SigFigs = getUint32(buffer[8:12], flipped)

	// Now the maximum length of the captured packet data.
	file.MaxLen = getUint32(buffer[12:16], flipped)

	// And the link type.
	file.LinkType = Link(getUint32(buffer[16:20], flipped))

	return nil
}

// populatePacketHeader reads the next 16 bytes out of the file and builds it into a
// packet header.
func populatePacketHeader(packet *Packet, src io.Reader, flipped bool) error {
	buffer := make([]byte, 16)
	read_count, err := src.Read(buffer)

	if read_count != 16 {
		return InsufficientLength
	}

	// First is a pair of fields that build up the timestamp.
	ts_seconds := getUint32(buffer[0:4], flipped)
	ts_millis := getUint32(buffer[4:8], flipped)
	packet.Timestamp = (time.Duration(ts_seconds) * time.Second) + (time.Duration(ts_millis) * time.Millisecond)

	// Next is the length of the data segment.
	packet.IncludedLen = getUint32(buffer[8:12], flipped)

	// Then the original length of the packet.
	packet.ActualLen = getUint32(buffer[12:16], flipped)

	return err
}
<commit_msg>Handle EOF better.<commit_after>package gopcap

import (
	"bytes"
	"io"
	"time"
)

// checkMagicNum checks the first four bytes of a pcap file, searching for the magic number
// and checking the byte order. Returns three values: whether the file is a pcap file, whether
// the byte order needs flipping, and any error that was encountered. If error is returned,
// the other values are invalid.
func checkMagicNum(src io.Reader) (bool, bool, error) {
	// These magic numbers form the header of a pcap file.
	magic := []byte{0xa1, 0xb2, 0xc3, 0xd4}
	magic_reverse := []byte{0xd4, 0xc3, 0xb2, 0xa1}

	buffer := make([]byte, 4)
	read_count, err := src.Read(buffer)

	if read_count != 4 {
		return false, false, InsufficientLength
	}
	if (err != nil) && (err != io.EOF) {
		return false, false, err
	}

	if bytes.Compare(buffer, magic) == 0 {
		return true, false, nil
	} else if bytes.Compare(buffer, magic_reverse) == 0 {
		return true, true, nil
	}

	return false, false, NotAPcapFile
}

// parsePacket parses a full packet out of the pcap file. It returns an error if any problems were
// encountered.
func parsePacket(pkt *Packet, src io.Reader, flipped bool) error {
	err := populatePacketHeader(pkt, src, flipped)

	if err != nil {
		return err
	}

	data := make([]byte, pkt.IncludedLen)
	readlen, err := src.Read(data)
	pkt.Data = data

	if uint32(readlen) != pkt.IncludedLen {
		err = UnexpectedEOF
	}

	return err
}

// populateFileHeader reads the next 20 bytes out of the .pcap file and uses it to populate the
// PcapFile structure.
func populateFileHeader(file *PcapFile, src io.Reader, flipped bool) error {
	buffer := make([]byte, 20)
	read_count, err := src.Read(buffer)

	if err != nil {
		return err
	} else if read_count != 20 {
		return InsufficientLength
	}

	// First two bytes are the major version number.
	file.MajorVersion = getUint16(buffer[0:2], flipped)

	// Next two are the minor version number.
	file.MinorVersion = getUint16(buffer[2:4], flipped)

	// GMT to local correction, in seconds east of UTC.
	file.TZCorrection = getInt32(buffer[4:8], flipped)

	// Next is the number of significant figures in the timestamps. Almost always zero.
	file.SigFigs = getUint32(buffer[8:12], flipped)

	// Now the maximum length of the captured packet data.
	file.MaxLen = getUint32(buffer[12:16], flipped)

	// And the link type.
	file.LinkType = Link(getUint32(buffer[16:20], flipped))

	return nil
}

// populatePacketHeader reads the next 16 bytes out of the file and builds it into a
// packet header.
func populatePacketHeader(packet *Packet, src io.Reader, flipped bool) error {
	buffer := make([]byte, 16)
	read_count, err := src.Read(buffer)

	if read_count != 16 {
		return InsufficientLength
	}

	// First is a pair of fields that build up the timestamp.
	ts_seconds := getUint32(buffer[0:4], flipped)
	ts_millis := getUint32(buffer[4:8], flipped)
	packet.Timestamp = (time.Duration(ts_seconds) * time.Second) + (time.Duration(ts_millis) * time.Millisecond)

	// Next is the length of the data segment.
	packet.IncludedLen = getUint32(buffer[8:12], flipped)

	// Then the original length of the packet.
	packet.ActualLen = getUint32(buffer[12:16], flipped)

	return err
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"os"
	"path"
	"strconv"
	"time"
	//	"github.com/davecgh/go-spew/spew"
)

//RawImage contains data we got from API that needs to be modified before further usage
type RawImage struct {
	Imgid          int    `json:"id_number"`
	URL            string `json:"image"`
	Score          int    `json:"score"`
	OriginalFormat string `json:"original_format"`
	Faves          int    `json:"faves"`
}

//Image contains data needed to filter fetch and save image
type Image struct {
	Imgid    int
	URL      string
	Filename string
	Score    int
	Faves    int
}

//Search returns to us array of searched images...
type Search struct {
	Images []RawImage `json:"search"`
}

func (imgdata *Image) isDeepEqual(b Image) bool {
	if b.Imgid == imgdata.Imgid &&
		b.URL == imgdata.URL &&
		b.Filename == imgdata.Filename &&
		b.Score == imgdata.Score &&
		b.Faves == imgdata.Faves {
		return true
	}
	return false
}

//ImageCh is a channel of image data. You can put images into channel by parsing
//Derpibooru API by id(s) or  by tags and you can download images that are already
//in channel
type ImageCh chan Image

//Push gets unmarchalled JSON info, massages it and plugs it into channel so it
//would be processed in other places
func trim(dat RawImage) Image {

	tfn := strconv.Itoa(dat.Imgid) + "." + dat.OriginalFormat
	return Image{
		Imgid:    dat.Imgid,
		Filename: tfn,
		URL:      prefix + "/" + path.Dir(dat.URL) + "/" + tfn,
		Score:    dat.Score,
		Faves:    dat.Faves,
	}
}

//ParseImg gets image IDs, fetches information about those images from Derpibooru and pushes them into the channel.
func (imgchan ImageCh) ParseImg(ids []int, key string) {

	for _, imgid := range ids {
		source := prefix + "//derpibooru.org/images/" + strconv.Itoa(imgid) + ".json"
		if key != "" {
			source = source + "?key=" + key
		}

		lInfo("Getting image info at:", source)

		body, err := getRemoteJSON(source)
		if err != nil {
			lErr(err)
			continue
		}
		var dat RawImage
		if err := json.Unmarshal(body, &dat); //transforming json into native map

		err != nil {
			lErr(err)
			continue
		}

		imgchan <- trim(dat)
	}

	close(imgchan) //closing channel, we are done here

	return
}

//DlImg reads image data from channel and downloads specified images to disc
func (imgchan ImageCh) downloadImages(opts *Config) {

	lInfo("Worker started; reading channel") //nice notification that we are not forgotten
	var n, size int
	for imgdata := range imgchan {

		if imgdata.Filename == "" {
			lErr("Empty filename. Oops?") //something somewhere had gone wrong, not a cause to die, going to the next image
			continue
		}

		lInfo("Saving as", imgdata.Filename)

		size += imgdata.saveImage(opts)
		n++
	}
	lInfof("Downloaded %d images, for a total of %s", n, fmtbytes(float64(size)))
}

func (imgdata Image) saveImage(opts *Config) (size int) { // To not hold all the files open when there is no need. All pointers to files are in the scope of this function.

	output, err := os.Create(opts.ImageDir + string(os.PathSeparator) + imgdata.Filename) //And now, THE FILE!
	if err != nil {
		lErr("Error when creating file for image: ", strconv.Itoa(imgdata.Imgid))
		lErr(err) //Either we got no permisson or no space, end of line
		return
	}
	defer func() {
		err = output.Close() //Not forgetting to deal with it after completing download
		if err != nil {
			lFatal("Could  not close downloaded file")
		}
	}()

	start := time.Now() //timing download time. We can't begin it sooner, not sure if we can begin it later

	body, header, err := getImage(imgdata.URL)

	if err != nil {
		lErr("Error when getting image: ", strconv.Itoa(imgdata.Imgid))
		lErr(err)
		return
	}

	size, err = output.Write(body) //
	if err != nil {
		lErr("Unable to write image on disk, id: ", strconv.Itoa(imgdata.Imgid))
		lErr(err)
		return
	}
	timed := time.Since(start).Seconds()

	lInfof("Downloaded %d bytes in %.2fs, speed %s/s\n", size, timed, fmtbytes(float64(size)/timed))

	sizestring, prs := header["Content-Length"]
	if !prs {
		lErr("Filesize not provided")
		return
	}

	expsize, err := strconv.Atoi(sizestring[0])
	if err != nil {
		lErr("Unable to get expected filesize")
		return
	}
	if expsize != size {
		lErr("Unable to download full image")
		return 0
	}
	return
}

//ParseTag gets image tags, fetches information about all images it could from Derpibooru and pushes them into the channel.
func (imgchan ImageCh) ParseTag(opts *TagOpts, key string) {

	//Unlike main, I don't see how I could separate bits out to decrease complexity
	source := prefix + "//derpibooru.org/search.json?q=" + opts.Tag //yay hardwiring url strings!

	if key != "" {
		source = source + "&key=" + key
	}

	lInfo("Searching as", source)

	for i := opts.StartPage; opts.StopPage == 0 || i <= opts.StopPage; i++ {
		lInfo("Searching page", i)

		body, err := getRemoteJSON(source + "&page=" + strconv.Itoa(i))
		if err != nil {
			lErr("Error while json from page ", i)
			lErr(err)
			continue
		}

		var dats Search                   //Because we got array incoming instead of single object, we using a slive of maps!
		err = json.Unmarshal(body, &dats) //transforming json into native view

		if err != nil {
			lErr("Error while parsing search page", i)
			lErr(err)
			if serr, ok := err.(*json.SyntaxError); ok { //In case crap was still given, we are looking at it.
				lErr("Occurred at offset: ", serr.Offset)
			}
			continue

		}

		if len(dats.Images) == 0 {
			lInfo("Pages are all over") //Does not mean that process is over.
			break
		} //exit due to finishing all pages

		for _, dat := range dats.Images {
			imgchan <- trim(dat)
		}

	}

	close(imgchan)
}
<commit_msg>Enabled no-clobber by default Closes #17 Also removed some unneeded returns after non-fatal errors<commit_after>package main

import (
	"encoding/json"
	"os"
	"path"
	"strconv"
	"time"
	//	"github.com/davecgh/go-spew/spew"
)

//RawImage contains data we got from API that needs to be modified before further usage
type RawImage struct {
	Imgid          int    `json:"id_number"`
	URL            string `json:"image"`
	Score          int    `json:"score"`
	OriginalFormat string `json:"original_format"`
	Faves          int    `json:"faves"`
}

//Image contains data needed to filter fetch and save image
type Image struct {
	Imgid    int
	URL      string
	Filename string
	Score    int
	Faves    int
}

//Search returns to us array of searched images...
type Search struct {
	Images []RawImage `json:"search"`
}

func (imgdata *Image) isDeepEqual(b Image) bool {
	if b.Imgid == imgdata.Imgid &&
		b.URL == imgdata.URL &&
		b.Filename == imgdata.Filename &&
		b.Score == imgdata.Score &&
		b.Faves == imgdata.Faves {
		return true
	}
	return false
}

//ImageCh is a channel of image data. You can put images into channel by parsing
//Derpibooru API by id(s) or  by tags and you can download images that are already
//in channel
type ImageCh chan Image

//Push gets unmarchalled JSON info, massages it and plugs it into channel so it
//would be processed in other places
func trim(dat RawImage) Image {

	tfn := strconv.Itoa(dat.Imgid) + "." + dat.OriginalFormat
	return Image{
		Imgid:    dat.Imgid,
		Filename: tfn,
		URL:      prefix + "/" + path.Dir(dat.URL) + "/" + tfn,
		Score:    dat.Score,
		Faves:    dat.Faves,
	}
}

//ParseImg gets image IDs, fetches information about those images from Derpibooru and pushes them into the channel.
func (imgchan ImageCh) ParseImg(ids []int, key string) {

	for _, imgid := range ids {
		source := prefix + "//derpibooru.org/images/" + strconv.Itoa(imgid) + ".json"
		if key != "" {
			source = source + "?key=" + key
		}

		lInfo("Getting image info at:", source)

		body, err := getRemoteJSON(source)
		if err != nil {
			lErr(err)
			continue
		}
		var dat RawImage
		if err := json.Unmarshal(body, &dat); //transforming json into native map

		err != nil {
			lErr(err)
			continue
		}

		imgchan <- trim(dat)
	}

	close(imgchan) //closing channel, we are done here

	return
}

//DlImg reads image data from channel and downloads specified images to disc
func (imgchan ImageCh) downloadImages(opts *Config) {

	lInfo("Worker started; reading channel") //nice notification that we are not forgotten
	var n, size int
	for imgdata := range imgchan {

		if imgdata.Filename == "" {
			lErr("Empty filename. Oops?") //something somewhere had gone wrong, not a cause to die, going to the next image
			continue
		}

		lInfo("Saving as", imgdata.Filename)

		size += imgdata.saveImage(opts)
		n++
	}
	lInfof("Downloaded %d images, for a total of %s", n, fmtbytes(float64(size)))
}

func (imgdata Image) saveImage(opts *Config) (size int) { // To not hold all the files open when there is no need. All pointers to files are in the scope of this function.

	filepath := opts.ImageDir + string(os.PathSeparator) + imgdata.Filename

	output, _ := os.Open(filepath) //And now, THE FILE! New, truncated, ready to write

	fstat, err := output.Stat()
	if err != nil {
		lErr("Can't get file stats")
	}
	output.Close()

	start := time.Now() //timing download time. We can't begin it sooner, not sure if we can begin it later

	body, header, err := getImage(imgdata.URL)

	sizestring, prs := header["Content-Length"]
	if !prs {
		lErr("Filesize not provided")
	}
	expsize, err := strconv.Atoi(sizestring[0])
	if err != nil {
		lErr("Unable to get expected filesize")
	}
	lInfo(expsize, fstat.Size())
	if int64(expsize) == fstat.Size() {
		lInfo("Skipping: no-clobber")
		return 0
	}

	if err != nil {
		lErr("Error when getting image: ", strconv.Itoa(imgdata.Imgid))
		lErr(err)
		return
	}

	output, err = os.Create(filepath) //And now, THE FILE! New, truncated, ready to write
	if err != nil {
		lErr("Error when creating file for image: ", strconv.Itoa(imgdata.Imgid))
		lErr(err) //Either we got no permisson or no space, end of line
		return
	}
	defer func() {
		err = output.Close() //Not forgetting to deal with it after completing download
		if err != nil {
			lFatal("Could  not close downloaded file")
		}
	}()

	size, err = output.Write(body) //
	if err != nil {
		lErr("Unable to write image on disk, id: ", strconv.Itoa(imgdata.Imgid))
		lErr(err)
		return
	}
	timed := time.Since(start).Seconds()

	lInfof("Downloaded %d bytes in %.2fs, speed %s/s\n", size, timed, fmtbytes(float64(size)/timed))

	if expsize != size {
		lErr("Unable to download full image")
		return 0
	}
	return
}

//ParseTag gets image tags, fetches information about all images it could from Derpibooru and pushes them into the channel.
func (imgchan ImageCh) ParseTag(opts *TagOpts, key string) {

	//Unlike main, I don't see how I could separate bits out to decrease complexity
	source := prefix + "//derpibooru.org/search.json?q=" + opts.Tag //yay hardwiring url strings!

	if key != "" {
		source = source + "&key=" + key
	}

	lInfo("Searching as", source)

	for i := opts.StartPage; opts.StopPage == 0 || i <= opts.StopPage; i++ {
		lInfo("Searching page", i)

		body, err := getRemoteJSON(source + "&page=" + strconv.Itoa(i))
		if err != nil {
			lErr("Error while json from page ", i)
			lErr(err)
			continue
		}

		var dats Search                   //Because we got array incoming instead of single object, we using a slive of maps!
		err = json.Unmarshal(body, &dats) //transforming json into native view

		if err != nil {
			lErr("Error while parsing search page", i)
			lErr(err)
			if serr, ok := err.(*json.SyntaxError); ok { //In case crap was still given, we are looking at it.
				lErr("Occurred at offset: ", serr.Offset)
			}
			continue

		}

		if len(dats.Images) == 0 {
			lInfo("Pages are all over") //Does not mean that process is over.
			break
		} //exit due to finishing all pages

		for _, dat := range dats.Images {
			imgchan <- trim(dat)
		}

	}

	close(imgchan)
}
<|endoftext|>