<commit_before>package radius

import (
	"context"
	"errors"
	"net"
	"sync"
	"sync/atomic"
)

type packetResponseWriter struct {
	// listener that received the packet
	conn net.PacketConn
	addr net.Addr
}

func (r *packetResponseWriter) Write(packet *Packet) error {
	raw, err := packet.Encode()
	if err != nil {
		return err
	}
	if _, err := r.conn.WriteTo(raw, r.addr); err != nil {
		return err
	}
	return nil
}

// PacketServer listens for RADIUS requests on a packet-based protocols (e.g.
// UDP).
type PacketServer struct {
	// The address on which the server listens. Defaults to :1812.
	Addr string
	// The network on which the server listens. Defaults to udp.
	Network      string
	SecretSource SecretSource
	Handler      Handler

	// Skip incoming packet authenticity validation.
	// This should only be set to true for debugging purposes.
	InsecureSkipVerify bool

	mu           sync.Mutex
	shuttingDown bool
	ctx          context.Context
	ctxDone      context.CancelFunc
	running      chan struct{}
	listeners    map[net.PacketConn]int
	activeCount  int32
}

// TODO: logger on PacketServer

// Serve accepts incoming connections on conn.
func (s *PacketServer) Serve(conn net.PacketConn) error {
	if s.Handler == nil {
		return errors.New("radius: nil Handler")
	}
	if s.SecretSource == nil {
		return errors.New("radius: nil SecretSource")
	}

	s.mu.Lock()
	if s.shuttingDown {
		s.mu.Unlock()
		return ErrServerShutdown
	}
	var ctx context.Context
	if s.ctx == nil {
		s.ctx, s.ctxDone = context.WithCancel(context.Background())
		ctx = s.ctx
	}
	if s.running == nil {
		s.running = make(chan struct{})
	}
	if s.listeners == nil {
		s.listeners = make(map[net.PacketConn]int)
	}
	s.listeners[conn]++
	s.mu.Unlock()

	type activeKey struct {
		IP         string
		Identifier byte
	}

	var (
		activeLock sync.Mutex
		active     = map[activeKey]struct{}{}
	)

	atomic.AddInt32(&s.activeCount, 1)
	defer func() {
		s.mu.Lock()
		s.listeners[conn]--
		if s.listeners[conn] == 0 {
			delete(s.listeners, conn)
		}
		s.mu.Unlock()

		if atomic.AddInt32(&s.activeCount, -1) == 0 {
			s.mu.Lock()
			s.shuttingDown = false
			close(s.running)
			s.running = nil
			s.ctx = nil
			s.mu.Unlock()
		}
	}()

	for {
		var buff [MaxPacketLength]byte
		n, remoteAddr, err := conn.ReadFrom(buff[:])
		if err != nil {
			s.mu.Lock()
			if s.shuttingDown {
				s.mu.Unlock()
				return nil
			}
			s.mu.Unlock()

			if ne, ok := err.(net.Error); ok && !ne.Temporary() {
				return err
			}
			// TODO: log error?
			continue
		}

		secret, err := s.SecretSource.RADIUSSecret(ctx, remoteAddr)
		if err != nil {
			// TODO: log?
			continue
		}
		if len(secret) == 0 {
			continue
		}

		if !s.InsecureSkipVerify && !IsAuthenticRequest(buff[:n], secret) {
			// TODO: log?
			continue
		}

		packet, err := Parse(buff[:n], secret)
		if err != nil {
			// TODO: error logger
			continue
		}

		atomic.AddInt32(&s.activeCount, 1)
		go func(packet *Packet, remoteAddr net.Addr) {
			key := activeKey{
				IP:         remoteAddr.String(),
				Identifier: packet.Identifier,
			}
			activeLock.Lock()
			if _, ok := active[key]; ok {
				activeLock.Unlock()
				return
			}
			active[key] = struct{}{}
			activeLock.Unlock()

			response := packetResponseWriter{
				conn: conn,
				addr: remoteAddr,
			}

			defer func() {
				activeLock.Lock()
				delete(active, key)
				activeLock.Unlock()

				if atomic.AddInt32(&s.activeCount, -1) == 0 {
					s.mu.Lock()
					s.shuttingDown = false
					close(s.running)
					s.running = nil
					s.ctx = nil
					s.mu.Unlock()
				}
			}()

			request := Request{
				LocalAddr:  conn.LocalAddr(),
				RemoteAddr: remoteAddr,
				Packet:     packet,
				ctx:        ctx,
			}

			s.Handler.ServeRADIUS(&response, &request)
		}(packet, remoteAddr)
	}
}

// ListenAndServe starts a RADIUS server on the address given in s.
func (s *PacketServer) ListenAndServe() error {
	if s.Handler == nil {
		return errors.New("radius: nil Handler")
	}
	if s.SecretSource == nil {
		return errors.New("radius: nil SecretSource")
	}

	addrStr := ":1812"
	if s.Addr != "" {
		addrStr = s.Addr
	}

	network := "udp"
	if s.Network != "" {
		network = s.Network
	}

	pc, err := net.ListenPacket(network, addrStr)
	if err != nil {
		return err
	}
	defer pc.Close()
	return s.Serve(pc)
}

// Shutdown gracefully stops the server. It first closes all listeners (which
// stops accepting new packets) and then waits for running handlers to complete.
//
// Shutdown returns after all handlers have completed, or when ctx is canceled.
// The PacketServer is ready for re-use once the function returns nil.
func (s *PacketServer) Shutdown(ctx context.Context) error {
	s.mu.Lock()

	if len(s.listeners) == 0 {
		s.mu.Unlock()
		return nil
	}

	if !s.shuttingDown {
		s.shuttingDown = true
		s.ctxDone()
		for listener := range s.listeners {
			listener.Close()
		}
	}

	ch := s.running
	s.mu.Unlock()

	select {
	case <-ch:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}
<commit_msg>move SecretSource.RADIUSSecret to inside handler goroutine<commit_after>package radius

import (
	"context"
	"errors"
	"net"
	"sync"
	"sync/atomic"
)

type packetResponseWriter struct {
	// listener that received the packet
	conn net.PacketConn
	addr net.Addr
}

func (r *packetResponseWriter) Write(packet *Packet) error {
	raw, err := packet.Encode()
	if err != nil {
		return err
	}
	if _, err := r.conn.WriteTo(raw, r.addr); err != nil {
		return err
	}
	return nil
}

// PacketServer listens for RADIUS requests on a packet-based protocols (e.g.
// UDP).
type PacketServer struct {
	// The address on which the server listens. Defaults to :1812.
	Addr string
	// The network on which the server listens. Defaults to udp.
	Network      string
	SecretSource SecretSource
	Handler      Handler

	// Skip incoming packet authenticity validation.
	// This should only be set to true for debugging purposes.
	InsecureSkipVerify bool

	mu           sync.Mutex
	shuttingDown bool
	ctx          context.Context
	ctxDone      context.CancelFunc
	running      chan struct{}
	listeners    map[net.PacketConn]int
	activeCount  int32
}

// TODO: logger on PacketServer

// Serve accepts incoming connections on conn.
func (s *PacketServer) Serve(conn net.PacketConn) error {
	if s.Handler == nil {
		return errors.New("radius: nil Handler")
	}
	if s.SecretSource == nil {
		return errors.New("radius: nil SecretSource")
	}

	s.mu.Lock()
	if s.shuttingDown {
		s.mu.Unlock()
		return ErrServerShutdown
	}
	var ctx context.Context
	if s.ctx == nil {
		s.ctx, s.ctxDone = context.WithCancel(context.Background())
		ctx = s.ctx
	}
	if s.running == nil {
		s.running = make(chan struct{})
	}
	if s.listeners == nil {
		s.listeners = make(map[net.PacketConn]int)
	}
	s.listeners[conn]++
	s.mu.Unlock()

	type activeKey struct {
		IP         string
		Identifier byte
	}

	var (
		activeLock sync.Mutex
		active     = map[activeKey]struct{}{}
	)

	atomic.AddInt32(&s.activeCount, 1)
	defer func() {
		s.mu.Lock()
		s.listeners[conn]--
		if s.listeners[conn] == 0 {
			delete(s.listeners, conn)
		}
		s.mu.Unlock()

		if atomic.AddInt32(&s.activeCount, -1) == 0 {
			s.mu.Lock()
			s.shuttingDown = false
			close(s.running)
			s.running = nil
			s.ctx = nil
			s.mu.Unlock()
		}
	}()

	for {
		var buff [MaxPacketLength]byte
		n, remoteAddr, err := conn.ReadFrom(buff[:])
		if err != nil {
			s.mu.Lock()
			if s.shuttingDown {
				s.mu.Unlock()
				return nil
			}
			s.mu.Unlock()

			if ne, ok := err.(net.Error); ok && !ne.Temporary() {
				return err
			}
			// TODO: log error?
			continue
		}

		buffCopy := make([]byte, n)
		copy(buffCopy, buff[:n])

		atomic.AddInt32(&s.activeCount, 1)
		go func(buff []byte, remoteAddr net.Addr) {
			secret, err := s.SecretSource.RADIUSSecret(ctx, remoteAddr)
			if err != nil {
				// TODO: log only if server is not shutting down?
				return
			}
			if len(secret) == 0 {
				return
			}

			if !s.InsecureSkipVerify && !IsAuthenticRequest(buff, secret) {
				// TODO: log?
				return
			}

			packet, err := Parse(buff, secret)
			if err != nil {
				// TODO: error logger
				return
			}

			key := activeKey{
				IP:         remoteAddr.String(),
				Identifier: packet.Identifier,
			}
			activeLock.Lock()
			if _, ok := active[key]; ok {
				activeLock.Unlock()
				return
			}
			active[key] = struct{}{}
			activeLock.Unlock()

			response := packetResponseWriter{
				conn: conn,
				addr: remoteAddr,
			}

			defer func() {
				activeLock.Lock()
				delete(active, key)
				activeLock.Unlock()

				if atomic.AddInt32(&s.activeCount, -1) == 0 {
					s.mu.Lock()
					s.shuttingDown = false
					close(s.running)
					s.running = nil
					s.ctx = nil
					s.mu.Unlock()
				}
			}()

			request := Request{
				LocalAddr:  conn.LocalAddr(),
				RemoteAddr: remoteAddr,
				Packet:     packet,
				ctx:        ctx,
			}

			s.Handler.ServeRADIUS(&response, &request)
		}(buffCopy, remoteAddr)
	}
}

// ListenAndServe starts a RADIUS server on the address given in s.
func (s *PacketServer) ListenAndServe() error {
	if s.Handler == nil {
		return errors.New("radius: nil Handler")
	}
	if s.SecretSource == nil {
		return errors.New("radius: nil SecretSource")
	}

	addrStr := ":1812"
	if s.Addr != "" {
		addrStr = s.Addr
	}

	network := "udp"
	if s.Network != "" {
		network = s.Network
	}

	pc, err := net.ListenPacket(network, addrStr)
	if err != nil {
		return err
	}
	defer pc.Close()
	return s.Serve(pc)
}

// Shutdown gracefully stops the server. It first closes all listeners (which
// stops accepting new packets) and then waits for running handlers to complete.
//
// Shutdown returns after all handlers have completed, or when ctx is canceled.
// The PacketServer is ready for re-use once the function returns nil.
func (s *PacketServer) Shutdown(ctx context.Context) error {
	s.mu.Lock()

	if len(s.listeners) == 0 {
		s.mu.Unlock()
		return nil
	}

	if !s.shuttingDown {
		s.shuttingDown = true
		s.ctxDone()
		for listener := range s.listeners {
			listener.Close()
		}
	}

	ch := s.running
	s.mu.Unlock()

	select {
	case <-ch:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}
<|endoftext|>


<commit_before>package db

import (
	"fmt"
	"io"
	"strings"
	"sync"
	"time"

	turtle "github.com/gtfierro/hod/goraptor"
	"github.com/gtfierro/hod/query"

	"github.com/blevesearch/bleve"
	"github.com/coocood/freecache"
	"github.com/mitghi/btree"
	"github.com/pkg/errors"
)

func (db *DB) RunQuery(q query.Query) (QueryResult, error) {
	fullQueryStart := time.Now()

	// "clean" the query by expanding out the prefixes
	// make sure to first do the Filters, then the Or clauses
	for idx, filter := range q.Where.Filters {
		q.Where.Filters[idx] = db.expandFilter(filter)
	}
	for idx, orclause := range q.Where.Ors {
		q.Where.Ors[idx] = db.expandOrClauseFilters(orclause)
	}

	// we flatten the OR clauses to get the array of queries we are going
	// to run and then merge
	orTerms := query.FlattenOrClauseList(q.Where.Ors)
	oldFilters := q.Where.Filters

	// check query hash
	var queryhash []byte
	if db.queryCacheEnabled {
		queryhash = q.Hash(orTerms)
		if ans, err := db.queryCache.Get(queryhash); err == nil {
			var res QueryResult
			if _, err := res.UnmarshalMsg(ans); err != nil {
				log.Error(errors.Wrap(err, "Could not fetch query from cache. Running..."))
			} else {
				// successful!
				res.Elapsed = time.Since(fullQueryStart)
				return res, nil
			}
		} else if err != nil && err == freecache.ErrNotFound {
			log.Notice("Could not fetch query from cache")
		} else if err != nil {
			log.Error(errors.Wrap(err, "Could not access query cache"))
		}
	}

	unionedRows := btree.New(BTREE_DEGREE, "")
	defer cleanResultRows(unionedRows)

	// if we have terms that are part of a set of OR statements, then we run
	// parallel queries for each fully-elaborated "branch" or the OR statement,
	// and then merge the results together at the end
	if len(orTerms) > 0 {
		var rowLock sync.Mutex
		var wg sync.WaitGroup
		var queryErr error
		wg.Add(len(orTerms))
		for _, orTerm := range orTerms {
			tmpQuery := q.Copy()
			// augment with the filters
			tmpQuery.Where.Filters = make([]query.Filter, len(oldFilters)+len(orTerm))
			copy(tmpQuery.Where.Filters, oldFilters)
			copy(tmpQuery.Where.Filters[len(oldFilters):], orTerm)
			go func(q query.Query) {
				results, err := db.getQueryResults(q)
				rowLock.Lock()
				if err != nil {
					queryErr = err
				} else {
					for _, row := range results {
						unionedRows.ReplaceOrInsert(row)
					}
				}
				rowLock.Unlock()
				wg.Done()
			}(tmpQuery)
		}
		wg.Wait()
		if queryErr != nil {
			return QueryResult{}, queryErr
		}
	} else {
		results, err := db.getQueryResults(q)
		if err != nil {
			return QueryResult{}, err
		}
		for _, row := range results {
			unionedRows.ReplaceOrInsert(row)
		}
	}
	if db.showQueryLatencies {
		log.Noticef("Full Query took %s", time.Since(fullQueryStart))
	}

	var result = newQueryResult()
	result.selectVars = q.Select.Variables
	result.Elapsed = time.Since(fullQueryStart)

	if q.Select.Count {
		// return the count of results
		result.Count = unionedRows.Len()
	} else {
		// return the rows
		i := unionedRows.DeleteMax()
		for i != nil {
			row := i.(*ResultRow)
			m := make(ResultMap)
			for idx, vname := range q.Select.Variables {
				m[vname.Var.String()] = row.row[idx]
			}
			result.Rows = append(result.Rows, m)
			i = unionedRows.DeleteMax()
		}
		result.Count = len(result.Rows)
	}

	if db.queryCacheEnabled {
		// set this in the cache
		marshalled, err := result.MarshalMsg(nil)
		if err != nil {
			log.Error(errors.Wrap(err, "Could not marshal results"))
		}
		if err := db.queryCache.Set(queryhash, marshalled, -1); err != nil {
			log.Error(errors.Wrap(err, "Could not cache results"))
		}
	}

	return result, nil
}

// takes a query and returns a DOT representation to visualize
// the construction of the query
func (db *DB) QueryToDOT(querystring io.Reader) (string, error) {
	q, err := query.Parse(querystring)
	if err != nil {
		return "", err
	}

	dot := ""
	dot += "digraph G {\n"
	dot += "ratio=\"auto\"\n"
	dot += "rankdir=\"LR\"\n"
	dot += "size=\"7.5,10\"\n"

	if len(q.Where.Ors) > 0 {
		orTerms := query.FlattenOrClauseList(q.Where.Ors)
		oldFilters := q.Where.Filters
		for _, orTerm := range orTerms {
			filters := append(oldFilters, orTerm...)
			for _, filter := range filters {
				var parts []string
				for _, p := range filter.Path {
					parts = append(parts, fmt.Sprintf("%s%s", p.Predicate, p.Pattern))
				}
				line := fmt.Sprintf("\"%s\" -> \"%s\" [label=\"%s\"];\n", filter.Subject, filter.Object, strings.Join(parts, "/"))
				if !strings.Contains(dot, line) {
					dot += line
				}

			}
		}
	} else {
		for _, filter := range q.Where.Filters {
			var parts []string
			for _, p := range filter.Path {
				parts = append(parts, fmt.Sprintf("%s%s", p.Predicate, p.Pattern))
			}
			line := fmt.Sprintf("\"%s\" -> \"%s\" [label=\"%s\"];\n", filter.Subject, filter.Object, strings.Join(parts, "/"))
			if !strings.Contains(dot, line) {
				dot += line
			}
		}
	}
	for _, sv := range q.Select.Variables {
		dot += fmt.Sprintf("\"%s\" [fillcolor=#e57373]\n", sv.Var)
	}
	dot += "}"
	return dot, nil
}

// executes a query and returns a DOT string of the classes involved
func (db *DB) QueryToClassDOT(querystring io.Reader) (string, error) {
	q, err := query.Parse(querystring)
	if err != nil {
		return "", err
	}
	// create DOT template string
	dot := ""
	dot += "digraph G {\n"

	// get rdf:type predicate hash as a string
	typeURI := turtle.ParseURI("rdf:type")
	typeURI.Namespace = db.namespaces[typeURI.Namespace]
	typeKey, err := db.GetHash(typeURI)
	if err != nil {
		return "", err
	}
	typeKeyString := typeKey.String()

	getClass := func(ent *Entity) (classes []turtle.URI, err error) {
		_classes := ent.OutEdges[typeKeyString]
		for _, class := range _classes {
			classes = append(classes, db.MustGetURI(class))
		}
		return
	}

	getEdges := func(ent *Entity) (predicates, objects []turtle.URI, reterr error) {
		var predKey Key
		for predKeyString, objectList := range ent.OutEdges {
			predKey.FromSlice([]byte(predKeyString))
			predURI, err := db.GetURI(predKey)
			if err != nil {
				reterr = err
				return
			}
			for _, objectKey := range objectList {
				objectEnt, err := db.GetEntityFromHash(objectKey)
				if err != nil {
					reterr = err
					return
				}
				objectClasses, err := getClass(objectEnt)
				if err != nil {
					reterr = err
					return
				}
				for _, class := range objectClasses {
					if predURI.Value != "type" && class.Value != "Class" {
						predicates = append(predicates, predURI)
						objects = append(objects, class)
					}
				}

			}
		}
		return
	}

	result, err := db.RunQuery(q)
	if err != nil {
		return "", err
	}
	for _, row := range result.Rows {
		for _, uri := range row {
			ent, err := db.GetEntity(uri)
			if err != nil {
				return "", err
			}
			classList, err := getClass(ent)
			if err != nil {
				return "", err
			}
			preds, objs, err := getEdges(ent)
			if err != nil {
				return "", err
			}
			// add class as node to graph
			for _, class := range classList {
				line := fmt.Sprintf("\"%s\" [fillcolor=\"#4caf50\"];\n", db.Abbreviate(class))
				if !strings.Contains(dot, line) {
					dot += line
				}
				for i := 0; i < len(preds); i++ {
					line := fmt.Sprintf("\"%s\" -> \"%s\" [label=\"%s\"];\n", db.Abbreviate(class), db.Abbreviate(objs[i]), db.Abbreviate(preds[i]))
					if !strings.Contains(dot, line) {
						dot += line
					}
				}
			}

		}
	}

	dot += "}"

	return dot, nil
}

func (db *DB) Abbreviate(uri turtle.URI) string {
	for abbv, ns := range db.namespaces {
		if abbv != "" && ns == uri.Namespace {
			return abbv + ":" + uri.Value
		}
	}
	return uri.Value
}

// Searches all of the values in the database; basic wildcard search
func (db *DB) Search(q string, n int) ([]string, error) {
	var res []string

	fmt.Println("Displaying", n, "results")
	query := bleve.NewMatchQuery(q)
	search := bleve.NewSearchRequestOptions(query, n, 0, false)
	searchResults, err := db.textidx.Search(search)
	if err != nil {
		fmt.Println(err)
		return res, err
	}
	for _, doc := range searchResults.Hits {
		res = append(res, db.Abbreviate(turtle.ParseURI(doc.ID)))
	}
	return res, nil
}
<commit_msg>remove concurrent or-clause execution for now<commit_after>package db

import (
	"fmt"
	"io"
	"strings"
	"sync"
	"time"

	turtle "github.com/gtfierro/hod/goraptor"
	"github.com/gtfierro/hod/query"

	"github.com/blevesearch/bleve"
	"github.com/coocood/freecache"
	"github.com/mitghi/btree"
	"github.com/pkg/errors"
)

func (db *DB) RunQuery(q query.Query) (QueryResult, error) {
	fullQueryStart := time.Now()

	// "clean" the query by expanding out the prefixes
	// make sure to first do the Filters, then the Or clauses
	for idx, filter := range q.Where.Filters {
		q.Where.Filters[idx] = db.expandFilter(filter)
	}
	for idx, orclause := range q.Where.Ors {
		q.Where.Ors[idx] = db.expandOrClauseFilters(orclause)
	}

	// we flatten the OR clauses to get the array of queries we are going
	// to run and then merge
	orTerms := query.FlattenOrClauseList(q.Where.Ors)
	oldFilters := q.Where.Filters

	// check query hash
	var queryhash []byte
	if db.queryCacheEnabled {
		queryhash = q.Hash(orTerms)
		if ans, err := db.queryCache.Get(queryhash); err == nil {
			var res QueryResult
			if _, err := res.UnmarshalMsg(ans); err != nil {
				log.Error(errors.Wrap(err, "Could not fetch query from cache. Running..."))
			} else {
				// successful!
				res.Elapsed = time.Since(fullQueryStart)
				return res, nil
			}
		} else if err != nil && err == freecache.ErrNotFound {
			log.Notice("Could not fetch query from cache")
		} else if err != nil {
			log.Error(errors.Wrap(err, "Could not access query cache"))
		}
	}

	unionedRows := btree.New(BTREE_DEGREE, "")
	defer cleanResultRows(unionedRows)

	// if we have terms that are part of a set of OR statements, then we run
	// parallel queries for each fully-elaborated "branch" or the OR statement,
	// and then merge the results together at the end
	if len(orTerms) > 0 {
		var rowLock sync.Mutex
		var wg sync.WaitGroup
		var queryErr error
		wg.Add(len(orTerms))
		for _, orTerm := range orTerms {
			tmpQuery := q.Copy()
			// augment with the filters
			tmpQuery.Where.Filters = make([]query.Filter, len(oldFilters)+len(orTerm))
			copy(tmpQuery.Where.Filters, oldFilters)
			copy(tmpQuery.Where.Filters[len(oldFilters):], orTerm)

			//	go func(q query.Query) {
			results, err := db.getQueryResults(tmpQuery)
			rowLock.Lock()
			if err != nil {
				queryErr = err
			} else {
				for _, row := range results {
					unionedRows.ReplaceOrInsert(row)
				}
			}
			rowLock.Unlock()
			wg.Done()
			//}(tmpQuery)
		}
		wg.Wait()
		if queryErr != nil {
			return QueryResult{}, queryErr
		}
	} else {
		results, err := db.getQueryResults(q)
		if err != nil {
			return QueryResult{}, err
		}
		for _, row := range results {
			unionedRows.ReplaceOrInsert(row)
		}
	}
	if db.showQueryLatencies {
		log.Noticef("Full Query took %s", time.Since(fullQueryStart))
	}

	var result = newQueryResult()
	result.selectVars = q.Select.Variables
	result.Elapsed = time.Since(fullQueryStart)

	if q.Select.Count {
		// return the count of results
		result.Count = unionedRows.Len()
	} else {
		// return the rows
		i := unionedRows.DeleteMax()
		for i != nil {
			row := i.(*ResultRow)
			m := make(ResultMap)
			for idx, vname := range q.Select.Variables {
				m[vname.Var.String()] = row.row[idx]
			}
			result.Rows = append(result.Rows, m)
			i = unionedRows.DeleteMax()
		}
		result.Count = len(result.Rows)
	}

	if db.queryCacheEnabled {
		// set this in the cache
		marshalled, err := result.MarshalMsg(nil)
		if err != nil {
			log.Error(errors.Wrap(err, "Could not marshal results"))
		}
		if err := db.queryCache.Set(queryhash, marshalled, -1); err != nil {
			log.Error(errors.Wrap(err, "Could not cache results"))
		}
	}

	return result, nil
}

// takes a query and returns a DOT representation to visualize
// the construction of the query
func (db *DB) QueryToDOT(querystring io.Reader) (string, error) {
	q, err := query.Parse(querystring)
	if err != nil {
		return "", err
	}

	dot := ""
	dot += "digraph G {\n"
	dot += "ratio=\"auto\"\n"
	dot += "rankdir=\"LR\"\n"
	dot += "size=\"7.5,10\"\n"

	if len(q.Where.Ors) > 0 {
		orTerms := query.FlattenOrClauseList(q.Where.Ors)
		oldFilters := q.Where.Filters
		for _, orTerm := range orTerms {
			filters := append(oldFilters, orTerm...)
			for _, filter := range filters {
				var parts []string
				for _, p := range filter.Path {
					parts = append(parts, fmt.Sprintf("%s%s", p.Predicate, p.Pattern))
				}
				line := fmt.Sprintf("\"%s\" -> \"%s\" [label=\"%s\"];\n", filter.Subject, filter.Object, strings.Join(parts, "/"))
				if !strings.Contains(dot, line) {
					dot += line
				}

			}
		}
	} else {
		for _, filter := range q.Where.Filters {
			var parts []string
			for _, p := range filter.Path {
				parts = append(parts, fmt.Sprintf("%s%s", p.Predicate, p.Pattern))
			}
			line := fmt.Sprintf("\"%s\" -> \"%s\" [label=\"%s\"];\n", filter.Subject, filter.Object, strings.Join(parts, "/"))
			if !strings.Contains(dot, line) {
				dot += line
			}
		}
	}
	for _, sv := range q.Select.Variables {
		dot += fmt.Sprintf("\"%s\" [fillcolor=#e57373]\n", sv.Var)
	}
	dot += "}"
	return dot, nil
}

// executes a query and returns a DOT string of the classes involved
func (db *DB) QueryToClassDOT(querystring io.Reader) (string, error) {
	q, err := query.Parse(querystring)
	if err != nil {
		return "", err
	}
	// create DOT template string
	dot := ""
	dot += "digraph G {\n"

	// get rdf:type predicate hash as a string
	typeURI := turtle.ParseURI("rdf:type")
	typeURI.Namespace = db.namespaces[typeURI.Namespace]
	typeKey, err := db.GetHash(typeURI)
	if err != nil {
		return "", err
	}
	typeKeyString := typeKey.String()

	getClass := func(ent *Entity) (classes []turtle.URI, err error) {
		_classes := ent.OutEdges[typeKeyString]
		for _, class := range _classes {
			classes = append(classes, db.MustGetURI(class))
		}
		return
	}

	getEdges := func(ent *Entity) (predicates, objects []turtle.URI, reterr error) {
		var predKey Key
		for predKeyString, objectList := range ent.OutEdges {
			predKey.FromSlice([]byte(predKeyString))
			predURI, err := db.GetURI(predKey)
			if err != nil {
				reterr = err
				return
			}
			for _, objectKey := range objectList {
				objectEnt, err := db.GetEntityFromHash(objectKey)
				if err != nil {
					reterr = err
					return
				}
				objectClasses, err := getClass(objectEnt)
				if err != nil {
					reterr = err
					return
				}
				for _, class := range objectClasses {
					if predURI.Value != "type" && class.Value != "Class" {
						predicates = append(predicates, predURI)
						objects = append(objects, class)
					}
				}

			}
		}
		return
	}

	result, err := db.RunQuery(q)
	if err != nil {
		return "", err
	}
	for _, row := range result.Rows {
		for _, uri := range row {
			ent, err := db.GetEntity(uri)
			if err != nil {
				return "", err
			}
			classList, err := getClass(ent)
			if err != nil {
				return "", err
			}
			preds, objs, err := getEdges(ent)
			if err != nil {
				return "", err
			}
			// add class as node to graph
			for _, class := range classList {
				line := fmt.Sprintf("\"%s\" [fillcolor=\"#4caf50\"];\n", db.Abbreviate(class))
				if !strings.Contains(dot, line) {
					dot += line
				}
				for i := 0; i < len(preds); i++ {
					line := fmt.Sprintf("\"%s\" -> \"%s\" [label=\"%s\"];\n", db.Abbreviate(class), db.Abbreviate(objs[i]), db.Abbreviate(preds[i]))
					if !strings.Contains(dot, line) {
						dot += line
					}
				}
			}

		}
	}

	dot += "}"

	return dot, nil
}

func (db *DB) Abbreviate(uri turtle.URI) string {
	for abbv, ns := range db.namespaces {
		if abbv != "" && ns == uri.Namespace {
			return abbv + ":" + uri.Value
		}
	}
	return uri.Value
}

// Searches all of the values in the database; basic wildcard search
func (db *DB) Search(q string, n int) ([]string, error) {
	var res []string

	fmt.Println("Displaying", n, "results")
	query := bleve.NewMatchQuery(q)
	search := bleve.NewSearchRequestOptions(query, n, 0, false)
	searchResults, err := db.textidx.Search(search)
	if err != nil {
		fmt.Println(err)
		return res, err
	}
	for _, doc := range searchResults.Hits {
		res = append(res, db.Abbreviate(turtle.ParseURI(doc.ID)))
	}
	return res, nil
}
<|endoftext|>


<commit_before>package db

// Level DB wrapper over multiple service data store.
// Each key has a prefix that combines service name as well as type of stored data.
// The following format is used for item keys:
// somename:m:itemid
// This format is used for payload keys:
// somename:p:payload

import (
	"bytes"
	"firempq/common"
	"firempq/conf"
	"sync"
	"time"

	"github.com/jmhodges/levigo"
	"github.com/op/go-logging"
)

// Default LevelDB read options.
var defaultReadOptions = levigo.NewReadOptions()

// Default LevelDB write options.
var defaultWriteOptions = levigo.NewWriteOptions()

var log = logging.MustGetLogger("ldb")

// ItemIterator built on top of LevelDB.
// It takes into account service name to limit the amount of selected data.
type ItemIterator struct {
	iter   *levigo.Iterator
	prefix []byte // Prefix for look ups.
	Key    []byte // Currently selected key. Valid only if the iterator is valid.
	Value  []byte // Currently selected value. Valid only if the iterator is valid.
}

//
func makeItemIterator(iter *levigo.Iterator, prefix []byte) *ItemIterator {
	iter.Seek(prefix)
	return &ItemIterator{iter, prefix, nil, nil}
}

// Next switches to the next element.
func (mi *ItemIterator) Next() {
	mi.iter.Next()
}

// Valid returns true if the current value is OK, otherwise false.
// for iter.Valid() {
//    mykey := iter.Key
//    myvalue := iter.Value
//    ......
//    iter.Next()
//}
func (mi *ItemIterator) Valid() bool {
	valid := mi.iter.Valid()
	if valid {
		k := mi.iter.Key()
		// Strip key prefix. If prefix doesn't match the length of the slice will remain the same.
		normKey := bytes.TrimPrefix(k, mi.prefix)
		if len(normKey) == len(k) {
			return false
		}
		mi.Key = normKey
		mi.Value = mi.iter.Value()
		return true
	}
	return false
}

// Close closes iterator. Iterator must be closed!
func (mi *ItemIterator) Close() {
	mi.iter.Close()
}

// DataStorage A high level structure on top of LevelDB.
// It caches item storing them into database
// as multiple large batches later.
type DataStorage struct {
	db              *levigo.DB        // Pointer the the instance of level db.
	dbName          string            // LevelDB database name.
	itemCache       map[string][]byte // Active cache for item metadata.
	payloadCache    map[string]string // Active cache for item payload.
	tmpPayloadCache map[string]string // Temporary map of cached payloads while it is in process of flushing into DB.
	cacheLock       sync.Mutex        // Used for caches access.
	flushLock       sync.Mutex        // Used to prevent double flush.
	closed          bool
}

// NewDataStorage is a constructor of DataStorage.
func NewDataStorage(dbName string) (*DataStorage, error) {
	ds := DataStorage{
		dbName:          dbName,
		itemCache:       make(map[string][]byte),
		payloadCache:    make(map[string]string),
		tmpPayloadCache: make(map[string]string),
		closed:          false,
	}
	opts := levigo.NewOptions()
	opts.SetCreateIfMissing(true)
	opts.SetWriteBufferSize(10 * 1024 * 1024)
	opts.SetCompression(levigo.SnappyCompression)
	db, err := levigo.Open(dbName, opts)
	if err != nil {
		return nil, err
	}
	ds.db = db
	go ds.periodicCacheFlush()
	return &ds, nil
}

func (ds *DataStorage) periodicCacheFlush() {
	for !ds.closed {
		ds.flushLock.Lock()
		if !ds.closed {
			ds.FlushCache()
		}
		ds.flushLock.Unlock()
		time.Sleep(conf.CFG.DbFlushInterval)
	}
}

// Item payload Id.
func makePayloadID(svcName, id string) string {
	return svcName + "\x01" + id
}

// Item Id.
func makeItemID(svcName, id string) string {
	return svcName + "\x02" + id
}

// StoreItemWithPayload stores item and provide payload.
func (ds *DataStorage) StoreItemWithPayload(svcName string, item common.IItemMetaData, payload string) {
	itemBody, err := item.Marshal()
	if err != nil {
		log.Error(err.Error())
		return
	}

	itemID := makeItemID(svcName, item.GetId())
	payloadID := makePayloadID(svcName, item.GetId())
	ds.cacheLock.Lock()
	ds.itemCache[itemID] = itemBody
	ds.payloadCache[payloadID] = payload
	ds.cacheLock.Unlock()
}

// StoreItem stores item onlt without paylaod.
func (ds *DataStorage) StoreItem(svcName string, item common.IItemMetaData) {
	itemBody, err := item.Marshal()
	if err != nil {
		log.Error(err.Error())
		return
	}
	itemID := makeItemID(svcName, item.GetId())

	ds.cacheLock.Lock()
	ds.itemCache[itemID] = itemBody
	ds.cacheLock.Unlock()
}

// DeleteItem deletes item metadata and payload, affects cache only until flushed.
func (ds *DataStorage) DeleteItem(svcName string, itemID string) {
	id := makeItemID(svcName, itemID)
	payloadID := makePayloadID(svcName, itemID)
	ds.cacheLock.Lock()
	ds.itemCache[id] = nil
	ds.payloadCache[payloadID] = ""
	ds.cacheLock.Unlock()
}

// DeleteServiceData deletes all service data such as service metadata, items and payloads.
func (ds *DataStorage) DeleteServiceData(svcName string) {

	counter := 0
	iter := ds.IterServiceItems(svcName)
	wb := levigo.NewWriteBatch()

	for iter.Valid() {
		if counter < 1000 {
			wb.Delete(iter.Key)
			counter++
		} else {
			counter = 0
			ds.db.Write(defaultWriteOptions, wb)
		}
	}

	wb.Delete([]byte(serviceMetadataPrefix + svcName))
	wb.Delete([]byte(serviceConfigPrefix + svcName))

	ds.db.Write(defaultWriteOptions, wb)
}

// GetPayload returns item payload. Three places are checked:
// 1. Top level cache.
// 2. Temp cache while data is getting flushed into db.
// 3. If not found in cache, will mane a DB lookup.
func (ds *DataStorage) GetPayload(svcName string, itemID string) string {
	payloadID := makePayloadID(svcName, itemID)
	ds.cacheLock.Lock()
	payload, ok := ds.payloadCache[payloadID]
	if ok {
		ds.cacheLock.Unlock()
		return payload
	}
	payload, ok = ds.tmpPayloadCache[payloadID]
	if ok {
		ds.cacheLock.Unlock()
		return payload
	}
	ds.cacheLock.Unlock()
	value, _ := ds.db.Get(defaultReadOptions, []byte(payloadID))
	return string(value)
}

// FlushCache flushes all cache into database.
func (ds *DataStorage) FlushCache() {
	if ds.closed {
		return
	}
	ds.cacheLock.Lock()
	tmpItemCache := ds.itemCache
	ds.tmpPayloadCache = ds.payloadCache
	ds.itemCache = make(map[string][]byte)
	ds.payloadCache = make(map[string]string)
	ds.cacheLock.Unlock()

	wb := levigo.NewWriteBatch()
	for k, v := range tmpItemCache {
		key := []byte(k)
		if v == nil {
			wb.Delete(key)
		} else {
			wb.Put(key, v)
		}
	}

	for k, v := range ds.tmpPayloadCache {
		key := []byte(k)
		if v == "" {
			wb.Delete(key)
		} else {
			wb.Put(key, []byte(v))
		}
	}

	ds.db.Write(defaultWriteOptions, wb)

	ds.cacheLock.Lock()
	ds.tmpPayloadCache = make(map[string]string)
	ds.cacheLock.Unlock()
}

// IterServiceItems returns new over all service item metadata.
// Service name used as a prefix to file all service items.
func (ds *DataStorage) IterServiceItems(svcName string) *ItemIterator {
	iter := ds.db.NewIterator(defaultReadOptions)
	prefix := []byte(makeItemID(svcName, ""))
	return makeItemIterator(iter, prefix)
}

const serviceMetadataPrefix = ":meta:"
const serviceConfigPrefix = ":conf:"

// GetAllServiceMeta returns all available services.
func (ds *DataStorage) GetAllServiceMeta() []*common.ServiceMetaInfo {
	qmiList := make([]*common.ServiceMetaInfo, 0, 1000)
	qtPrefix := []byte(serviceMetadataPrefix)
	iter := ds.db.NewIterator(defaultReadOptions)
	iter.Seek(qtPrefix)
	for iter.Valid() {
		svcKey := iter.Key()
		svcName := bytes.TrimPrefix(svcKey, qtPrefix)

		if len(svcKey) == len(svcName) {
			break
		}
		smi, err := common.ServiceInfoFromBinary(iter.Value())
		if err != nil {
			log.Error("Coudn't read service meta data because of: %s", err.Error())
			iter.Next()
			continue
		}
		if smi.Disabled {
			log.Error("Service is disabled. Skipping: %s", smi.Name)
			iter.Next()
			continue
		}
		qmiList = append(qmiList, smi)
		iter.Next()
	}
	return qmiList
}

// SaveServiceMeta stores service metadata into database.
func (ds *DataStorage) SaveServiceMeta(qmi *common.ServiceMetaInfo) {
	key := serviceMetadataPrefix + qmi.Name
	ds.db.Put(defaultWriteOptions, []byte(key), qmi.ToBinary())
}

func makeSettingsKey(svcName string) []byte {
	return []byte(serviceConfigPrefix + svcName)
}

// GetServiceConfig reads service config bases on service name.
// Caller should provide correct settings structure to read binary data.
func (ds *DataStorage) LoadServiceConfig(conf common.Marshalable, svcName string) error {
	key := makeSettingsKey(svcName)
	data, _ := ds.db.Get(defaultReadOptions, key)
	if data == nil {
		return common.InvalidRequest("No service settings found: " + svcName)
	}
	err := conf.Unmarshal(data)
	return err
}

// SaveServiceConfig saves service config into database.
func (ds *DataStorage) SaveServiceConfig(svcName string, conf common.Marshalable) {
	key := makeSettingsKey(svcName)
	data, err := conf.Marshal()
	if err == nil {
		err = ds.db.Put(defaultWriteOptions, key, data)
	}
	if err != nil {
		log.Error("Failed to save config: %s", err.Error())
	}
}

// Close flushes data on disk and closes database.
func (ds *DataStorage) Close() {
	ds.flushLock.Lock()
	defer ds.flushLock.Unlock()
	if !ds.closed {
		ds.closed = true
		ds.FlushCache()
		//ds.db.CompactRange(levigo.Range{nil, nil})
		ds.db.Close()
	} else {
		log.Error("Attempt to close database more than once!")
	}
}
<commit_msg>Added millisecond multiplicator to avoid constant database flush calls.<commit_after>package db

// Level DB wrapper over multiple service data store.
// Each key has a prefix that combines service name as well as type of stored data.
// The following format is used for item keys:
// somename:m:itemid
// This format is used for payload keys:
// somename:p:payload

import (
	"bytes"
	"firempq/common"
	"firempq/conf"
	"sync"
	"time"

	"github.com/jmhodges/levigo"
	"github.com/op/go-logging"
)

// Default LevelDB read options.
var defaultReadOptions = levigo.NewReadOptions()

// Default LevelDB write options.
var defaultWriteOptions = levigo.NewWriteOptions()

var log = logging.MustGetLogger("ldb")

// ItemIterator built on top of LevelDB.
// It takes into account service name to limit the amount of selected data.
type ItemIterator struct {
	iter   *levigo.Iterator
	prefix []byte // Prefix for look ups.
	Key    []byte // Currently selected key. Valid only if the iterator is valid.
	Value  []byte // Currently selected value. Valid only if the iterator is valid.
}

//
func makeItemIterator(iter *levigo.Iterator, prefix []byte) *ItemIterator {
	iter.Seek(prefix)
	return &ItemIterator{iter, prefix, nil, nil}
}

// Next switches to the next element.
func (mi *ItemIterator) Next() {
	mi.iter.Next()
}

// Valid returns true if the current value is OK, otherwise false.
// for iter.Valid() {
//    mykey := iter.Key
//    myvalue := iter.Value
//    ......
//    iter.Next()
//}
func (mi *ItemIterator) Valid() bool {
	valid := mi.iter.Valid()
	if valid {
		k := mi.iter.Key()
		// Strip key prefix. If prefix doesn't match the length of the slice will remain the same.
		normKey := bytes.TrimPrefix(k, mi.prefix)
		if len(normKey) == len(k) {
			return false
		}
		mi.Key = normKey
		mi.Value = mi.iter.Value()
		return true
	}
	return false
}

// Close closes iterator. Iterator must be closed!
func (mi *ItemIterator) Close() {
	mi.iter.Close()
}

// DataStorage A high level structure on top of LevelDB.
// It caches item storing them into database
// as multiple large batches later.
type DataStorage struct {
	db              *levigo.DB        // Pointer the the instance of level db.
	dbName          string            // LevelDB database name.
	itemCache       map[string][]byte // Active cache for item metadata.
	payloadCache    map[string]string // Active cache for item payload.
	tmpPayloadCache map[string]string // Temporary map of cached payloads while it is in process of flushing into DB.
	cacheLock       sync.Mutex        // Used for caches access.
	flushLock       sync.Mutex        // Used to prevent double flush.
	closed          bool
}

// NewDataStorage is a constructor of DataStorage.
func NewDataStorage(dbName string) (*DataStorage, error) {
	ds := DataStorage{
		dbName:          dbName,
		itemCache:       make(map[string][]byte),
		payloadCache:    make(map[string]string),
		tmpPayloadCache: make(map[string]string),
		closed:          false,
	}
	opts := levigo.NewOptions()
	opts.SetCreateIfMissing(true)
	opts.SetWriteBufferSize(10 * 1024 * 1024)
	opts.SetCompression(levigo.SnappyCompression)
	db, err := levigo.Open(dbName, opts)
	if err != nil {
		return nil, err
	}
	ds.db = db
	go ds.periodicCacheFlush()
	return &ds, nil
}

func (ds *DataStorage) periodicCacheFlush() {
	for !ds.closed {
		ds.flushLock.Lock()
		if !ds.closed {
			ds.FlushCache()
		}
		ds.flushLock.Unlock()
		time.Sleep(conf.CFG.DbFlushInterval * time.Millisecond)
	}
}

// Item payload Id.
func makePayloadID(svcName, id string) string {
	return svcName + "\x01" + id
}

// Item Id.
func makeItemID(svcName, id string) string {
	return svcName + "\x02" + id
}

// StoreItemWithPayload stores item and provide payload.
func (ds *DataStorage) StoreItemWithPayload(svcName string, item common.IItemMetaData, payload string) {
	itemBody, err := item.Marshal()
	if err != nil {
		log.Error(err.Error())
		return
	}

	itemID := makeItemID(svcName, item.GetId())
	payloadID := makePayloadID(svcName, item.GetId())
	ds.cacheLock.Lock()
	ds.itemCache[itemID] = itemBody
	ds.payloadCache[payloadID] = payload
	ds.cacheLock.Unlock()
}

// StoreItem stores item onlt without paylaod.
func (ds *DataStorage) StoreItem(svcName string, item common.IItemMetaData) {
	itemBody, err := item.Marshal()
	if err != nil {
		log.Error(err.Error())
		return
	}
	itemID := makeItemID(svcName, item.GetId())

	ds.cacheLock.Lock()
	ds.itemCache[itemID] = itemBody
	ds.cacheLock.Unlock()
}

// DeleteItem deletes item metadata and payload, affects cache only until flushed.
func (ds *DataStorage) DeleteItem(svcName string, itemID string) {
	id := makeItemID(svcName, itemID)
	payloadID := makePayloadID(svcName, itemID)
	ds.cacheLock.Lock()
	ds.itemCache[id] = nil
	ds.payloadCache[payloadID] = ""
	ds.cacheLock.Unlock()
}

// DeleteServiceData deletes all service data such as service metadata, items and payloads.
func (ds *DataStorage) DeleteServiceData(svcName string) {

	counter := 0
	iter := ds.IterServiceItems(svcName)
	wb := levigo.NewWriteBatch()

	for iter.Valid() {
		if counter < 1000 {
			wb.Delete(iter.Key)
			counter++
		} else {
			counter = 0
			ds.db.Write(defaultWriteOptions, wb)
		}
	}

	wb.Delete([]byte(serviceMetadataPrefix + svcName))
	wb.Delete([]byte(serviceConfigPrefix + svcName))

	ds.db.Write(defaultWriteOptions, wb)
}

// GetPayload returns item payload. Three places are checked:
// 1. Top level cache.
// 2. Temp cache while data is getting flushed into db.
// 3. If not found in cache, will mane a DB lookup.
func (ds *DataStorage) GetPayload(svcName string, itemID string) string {
	payloadID := makePayloadID(svcName, itemID)
	ds.cacheLock.Lock()
	payload, ok := ds.payloadCache[payloadID]
	if ok {
		ds.cacheLock.Unlock()
		return payload
	}
	payload, ok = ds.tmpPayloadCache[payloadID]
	if ok {
		ds.cacheLock.Unlock()
		return payload
	}
	ds.cacheLock.Unlock()
	value, _ := ds.db.Get(defaultReadOptions, []byte(payloadID))
	return string(value)
}

// FlushCache flushes all cache into database.
func (ds *DataStorage) FlushCache() {
	if ds.closed {
		return
	}
	ds.cacheLock.Lock()
	tmpItemCache := ds.itemCache
	ds.tmpPayloadCache = ds.payloadCache
	ds.itemCache = make(map[string][]byte)
	ds.payloadCache = make(map[string]string)
	ds.cacheLock.Unlock()

	wb := levigo.NewWriteBatch()
	for k, v := range tmpItemCache {
		key := []byte(k)
		if v == nil {
			wb.Delete(key)
		} else {
			wb.Put(key, v)
		}
	}

	for k, v := range ds.tmpPayloadCache {
		key := []byte(k)
		if v == "" {
			wb.Delete(key)
		} else {
			wb.Put(key, []byte(v))
		}
	}

	ds.db.Write(defaultWriteOptions, wb)

	ds.cacheLock.Lock()
	ds.tmpPayloadCache = make(map[string]string)
	ds.cacheLock.Unlock()
}

// IterServiceItems returns new over all service item metadata.
// Service name used as a prefix to file all service items.
func (ds *DataStorage) IterServiceItems(svcName string) *ItemIterator {
	iter := ds.db.NewIterator(defaultReadOptions)
	prefix := []byte(makeItemID(svcName, ""))
	return makeItemIterator(iter, prefix)
}

const serviceMetadataPrefix = ":meta:"
const serviceConfigPrefix = ":conf:"

// GetAllServiceMeta returns all available services.
func (ds *DataStorage) GetAllServiceMeta() []*common.ServiceMetaInfo {
	qmiList := make([]*common.ServiceMetaInfo, 0, 1000)
	qtPrefix := []byte(serviceMetadataPrefix)
	iter := ds.db.NewIterator(defaultReadOptions)
	iter.Seek(qtPrefix)
	for iter.Valid() {
		svcKey := iter.Key()
		svcName := bytes.TrimPrefix(svcKey, qtPrefix)

		if len(svcKey) == len(svcName) {
			break
		}
		smi, err := common.ServiceInfoFromBinary(iter.Value())
		if err != nil {
			log.Error("Coudn't read service meta data because of: %s", err.Error())
			iter.Next()
			continue
		}
		if smi.Disabled {
			log.Error("Service is disabled. Skipping: %s", smi.Name)
			iter.Next()
			continue
		}
		qmiList = append(qmiList, smi)
		iter.Next()
	}
	return qmiList
}

// SaveServiceMeta stores service metadata into database.
func (ds *DataStorage) SaveServiceMeta(qmi *common.ServiceMetaInfo) {
	key := serviceMetadataPrefix + qmi.Name
	ds.db.Put(defaultWriteOptions, []byte(key), qmi.ToBinary())
}

func makeSettingsKey(svcName string) []byte {
	return []byte(serviceConfigPrefix + svcName)
}

// GetServiceConfig reads service config bases on service name.
// Caller should provide correct settings structure to read binary data.
func (ds *DataStorage) LoadServiceConfig(conf common.Marshalable, svcName string) error {
	key := makeSettingsKey(svcName)
	data, _ := ds.db.Get(defaultReadOptions, key)
	if data == nil {
		return common.InvalidRequest("No service settings found: " + svcName)
	}
	err := conf.Unmarshal(data)
	return err
}

// SaveServiceConfig saves service config into database.
func (ds *DataStorage) SaveServiceConfig(svcName string, conf common.Marshalable) {
	key := makeSettingsKey(svcName)
	data, err := conf.Marshal()
	if err == nil {
		err = ds.db.Put(defaultWriteOptions, key, data)
	}
	if err != nil {
		log.Error("Failed to save config: %s", err.Error())
	}
}

// Close flushes data on disk and closes database.
func (ds *DataStorage) Close() {
	ds.flushLock.Lock()
	defer ds.flushLock.Unlock()
	if !ds.closed {
		ds.closed = true
		ds.FlushCache()
		//ds.db.CompactRange(levigo.Range{nil, nil})
		ds.db.Close()
	} else {
		log.Error("Attempt to close database more than once!")
	}
}
<|endoftext|>


<commit_before>package httplog

import (
	"fmt"
	"net/http"
	"strings"
	"time"
)

// for debugging request headers
func LogHeaders(r *http.Request) {
	fmt.Printf("HEADERS:\n")
	for k, v := range r.Header {
		fmt.Printf("\t%s\n", k)
		for i, _ := range v {
			fmt.Printf("\t\t%s\n", v[i])
		}
	}
}

/* kindof a common log type output */
func LogRequest(r *http.Request, statusCode int) {
	var addr string
	var user_agent string

	user_agent = ""
	addr = RealIP(r)

	for k, v := range r.Header {
		if k == "User-Agent" {
			user_agent = strings.Join(v, " ")
		}
		if k == "X-Forwarded-For" {
			addr = strings.Join(v, " ")
		}
	}

	fmt.Printf("%s - - [%s] \"%s %s\" \"%s\" %d %d\n",
		addr,
		time.Now(),
		r.Method,
		r.URL.String(),
		user_agent,
		statusCode,
		r.ContentLength)
}

func RealIP(r *http.Request) (ip string) {
	ip = r.RemoteAddr

	port_pos := strings.LastIndex(ip, ":")
	if port_pos != -1 {
		ip = ip[0:port_pos]
	}

	for k, v := range r.Header {
		if k == "X-Forwarded-For" {
			ip = strings.Join(v, " ")
		}
	}

	return ip
}
<commit_msg>adding a default favicon handler. no more 404's for that<commit_after>package httplog

import (
	"fmt"
	"net/http"
	"strings"
	"time"
)

var (
  /* This default icon is empty with a long lived cache */
	DefaultFavIcon FavIcon = defaultFavIcon{}
)

type defaultFavIcon struct {
}

func (dfi defaultFavIcon) Route(w http.ResponseWriter, r *http.Request) {
	LogRequest(r, 200)
	w.Header().Set("Cache-Control", "max-age=315360000")
}

/* simple interface for a favicon */
type FavIcon interface {
	Route(w http.ResponseWriter, r *http.Request)
}

// for debugging request headers
func LogHeaders(r *http.Request) {
	fmt.Printf("HEADERS:\n")
	for k, v := range r.Header {
		fmt.Printf("\t%s\n", k)
		for i, _ := range v {
			fmt.Printf("\t\t%s\n", v[i])
		}
	}
}

/* kindof a common log type output */
func LogRequest(r *http.Request, statusCode int) {
	var addr string
	var user_agent string

	user_agent = ""
	addr = RealIP(r)

	for k, v := range r.Header {
		if k == "User-Agent" {
			user_agent = strings.Join(v, " ")
		}
		if k == "X-Forwarded-For" {
			addr = strings.Join(v, " ")
		}
	}

	fmt.Printf("%s - - [%s] \"%s %s\" \"%s\" %d %d\n",
		addr,
		time.Now(),
		r.Method,
		r.URL.String(),
		user_agent,
		statusCode,
		r.ContentLength)
}

func RealIP(r *http.Request) (ip string) {
	ip = r.RemoteAddr

	port_pos := strings.LastIndex(ip, ":")
	if port_pos != -1 {
		ip = ip[0:port_pos]
	}

	for k, v := range r.Header {
		if k == "X-Forwarded-For" {
			ip = strings.Join(v, " ")
		}
	}

	return ip
}
<|endoftext|>


<commit_before>package rpcman

import (
    "log"
    "errors"
    "encoding/json"
    "time"
    zmq "github.com/alecthomas/gozmq"

)
type RPCMan struct {
    Context *zmq.Context
    ServAddr string
}

type Request struct {
    Method string
    Args interface{}
}

type Response struct {
    Response interface{}
    Status int
}

func Init(addr string) RPCMan {
    context, _ := zmq.NewContext()
    return RPCMan{context,addr}
}

func (rpc RPCMan) Close() {
    defer rpc.Context.Close()
}

func (rpc RPCMan) Call(method string, args ...interface{}) (interface{}, error){

    msg := Request{method, args}

    enc, err := json.Marshal(msg)
    if err != nil {
        log.Println("json error", err)
        return -1, err
    }

    socket, err := rpc.Context.NewSocket(zmq.REQ)
    if err != nil {
        log.Println("could not set up socket to rpcman",err)
        return -1, err
    }

    // set timeout to 1s
    duration, _ := time.ParseDuration("1s")
    socket.SetRcvTimeout(duration)

    err = socket.Connect(rpc.ServAddr)
    if err != nil {
        log.Println("Could not connect to socket",err)
        return -1, err
    }

    socket.Send(enc,0)
    //rpc.Socket.Send(enc, 0)
    resp := new(Response)

    //reply, err := rpc.Socket.Recv(0)
    reply, err := socket.Recv(0)
    if err != nil {
        log.Println("RPC recv failed:", err)
        return -1, err
    }

    err = json.Unmarshal(reply,resp)

    if err != nil {
        log.Println("json error", err)
        return -1, err
    }

    if resp.Status != 0 {
        return -1, errors.New("RPC Method not found:"+method)
    }

    socket.Close()

    return resp.Response, nil

}

<commit_msg>Now using a single socket.<commit_after>package rpcman

import (
	"encoding/json"
	"errors"
	"log"
	"sync"
	"time"

	zmq "github.com/alecthomas/gozmq"
)

type RPCMan struct {
	Context  *zmq.Context
	ServAddr string
	Socket   *zmq.Socket
	mutex    *sync.Mutex
}

type Request struct {
	Method string
	Args   interface{}
}

type Response struct {
	Response interface{}
	Status   int
}

func Init(addr string) (*RPCMan, error) {
	context, _ := zmq.NewContext()
	major, minor, patch := zmq.Version()

	socket, err := context.NewSocket(zmq.REQ)
	if err != nil {
		log.Println("could not set up socket to rpcman", err)
		return nil, err
	}
	err = socket.Connect(addr)
	if err != nil {
		log.Println("Could not connect to socket", err)
		return nil, err
	}

	// set timeout to 1s
	duration, _ := time.ParseDuration("1s")
	socket.SetRcvTimeout(duration)

	mutex := &sync.Mutex{}
	//cond := sync.NewCond(sync.Mutex{})

	rpc := RPCMan{
		Context:  context,
		ServAddr: addr,
		Socket:   socket,
		mutex:    mutex}

	return &rpc, nil

}

func (rpc RPCMan) Close() {
	rpc.Context.Close()
}

func (rpc RPCMan) Call(method string, args ...interface{}) (interface{}, error) {

	// exclusive access to socket
	//rpc.mutex.Lock()
	//defer rpc.mutex.Unlock()

	msg := Request{method, args}

	enc, err := json.Marshal(msg)
	if err != nil {
		log.Println("json error", err)
		return -1, err
	}

	rpc.Socket.Send(enc, 0)

	//rpc.Socket.Send(enc, 0)
	resp := new(Response)

	//reply, err := rpc.Socket.Recv(0)
	reply, err := rpc.Socket.Recv(0)
	if err != nil {
		log.Println("RPC recv failed:", err, method)
		return -1, err
	}

	err = json.Unmarshal(reply, resp)

	if err != nil {
		log.Println("json error", err)
		return -1, err
	}

	if resp.Status != 0 {
		return -1, errors.New("RPC Method not found:" + method)
	}

	// socket.Close()

	return resp.Response, nil

}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"regexp"
	"sort"
	"sync"

	"github.com/blang/semver"
	"github.com/google/go-github/github"
)

var (
	updateAssetRe = regexp.MustCompile(`^update_(darwin|windows|linux)_(arm|386|amd64)\.?.*$`)

	emptyVersion semver.Version
)

// Arch holds architecture names.
var Arch = struct {
	X64 string
	X86 string
	ARM string
}{
	"amd64",
	"386",
	"arm",
}

// OS holds operating system names.
var OS = struct {
	Windows string
	Linux   string
	Darwin  string
}{
	"windows",
	"linux",
	"darwin",
}

// Release struct represents a single github release.
type Release struct {
	id      int
	URL     string
	Version semver.Version
	Assets  []Asset
}

type releasesByID []Release

// Asset struct represents a file included as part of a Release.
type Asset struct {
	id        int
	v         semver.Version
	Name      string
	URL       string
	LocalFile string
	Checksum  string
	Signature string
	AssetInfo
}

// AssetInfo struct holds OS and Arch information of an asset.
type AssetInfo struct {
	OS   string
	Arch string
}

// ReleaseManager struct defines a repository to pull releases from.
type ReleaseManager struct {
	client          *github.Client
	owner           string
	repo            string
	updateAssetsMap map[string]map[string]map[string]*Asset
	latestAssetsMap map[string]map[string]*Asset
	mu              *sync.RWMutex
}

func (a releasesByID) Len() int {
	return len(a)
}

func (a releasesByID) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a releasesByID) Less(i, j int) bool {
	return a[i].id < a[j].id
}

// NewReleaseManager creates a wrapper of github.Client.
func NewReleaseManager(owner string, repo string) *ReleaseManager {

	ghc := &ReleaseManager{
		client:          github.NewClient(nil),
		owner:           owner,
		repo:            repo,
		mu:              new(sync.RWMutex),
		updateAssetsMap: make(map[string]map[string]map[string]*Asset),
		latestAssetsMap: make(map[string]map[string]*Asset),
	}

	return ghc
}

// getReleases queries github for all product releases.
func (g *ReleaseManager) getReleases() ([]Release, error) {
	var releases []Release

	for page := 1; true; page++ {
		opt := &github.ListOptions{Page: page}

		rels, _, err := g.client.Repositories.ListReleases(g.owner, g.repo, opt)

		if err != nil {
			return nil, err
		}

		if len(rels) == 0 {
			break
		}

		releases = make([]Release, 0, len(rels))

		for i := range rels {
			version := *rels[i].TagName
			v, err := semver.Parse(version)
			if err != nil {
				log.Debugf("Release %q is not semantically versioned (%q). Skipping.", version, err)
				continue
			}
			rel := Release{
				id:      *rels[i].ID,
				URL:     *rels[i].ZipballURL,
				Version: v,
			}
			rel.Assets = make([]Asset, 0, len(rels[i].Assets))
			for _, asset := range rels[i].Assets {
				rel.Assets = append(rel.Assets, Asset{
					id:   *asset.ID,
					Name: *asset.Name,
					URL:  *asset.BrowserDownloadURL,
				})
			}
			log.Debugf("Release %q has %d assets...", version, len(rel.Assets))
			releases = append(releases, rel)
		}
	}

	sort.Sort(sort.Reverse(releasesByID(releases)))

	return releases, nil
}

// UpdateAssetsMap will pull published releases, scan for compatible
// update-only binaries and will add them to the updateAssetsMap.
func (g *ReleaseManager) UpdateAssetsMap() (err error) {

	var rs []Release

	log.Debugf("Getting releases...")
	if rs, err = g.getReleases(); err != nil {
		return err
	}

	// Resetting file hashes.
	fileHashMapMu.Lock()
	fileHashMap = map[string]string{}
	fileHashMapMu.Unlock()

	log.Debugf("Getting assets...")
	for i := range rs {
		log.Debugf("Getting assets for release %q...", rs[i].Version)
		for j := range rs[i].Assets {
			log.Debugf("Found %q.", rs[i].Assets[j].Name)
			// Does this asset represent a binary update?
			if isUpdateAsset(rs[i].Assets[j].Name) {
				log.Debugf("%q is an auto-update asset.", rs[i].Assets[j].Name)
				asset := rs[i].Assets[j]
				asset.v = rs[i].Version
				info, err := getAssetInfo(asset.Name)
				if err != nil {
					return fmt.Errorf("Could not get asset info: %q", err)
				}
				if err = g.pushAsset(info.OS, info.Arch, &asset); err != nil {
					return fmt.Errorf("Could not push asset: %q", err)
				}
			} else {
				log.Debugf("%q is not an auto-update asset. Skipping.", rs[i].Assets[j].Name)
			}
		}
	}

	return nil
}

func (g *ReleaseManager) getProductUpdate(os string, arch string) (asset *Asset, err error) {
	g.mu.RLock()
	defer g.mu.RUnlock()

	if g.latestAssetsMap == nil {
		return nil, fmt.Errorf("No updates available.")
	}

	if g.latestAssetsMap[os] == nil {
		return nil, fmt.Errorf("No such OS.")
	}

	if g.latestAssetsMap[os][arch] == nil {
		return nil, fmt.Errorf("No such Arch.")
	}

	return g.latestAssetsMap[os][arch], nil
}

func (g *ReleaseManager) lookupAssetWithChecksum(os string, arch string, checksum string) (asset *Asset, err error) {
	g.mu.RLock()
	defer g.mu.RUnlock()

	if g.updateAssetsMap == nil {
		return nil, fmt.Errorf("No updates available.")
	}

	if g.updateAssetsMap[os] == nil {
		return nil, fmt.Errorf("No such OS.")
	}

	if g.updateAssetsMap[os][arch] == nil {
		return nil, fmt.Errorf("No such Arch.")
	}

	for _, a := range g.updateAssetsMap[os][arch] {
		if a.Checksum == checksum {
			return a, nil
		}
	}

	return nil, fmt.Errorf("Could not find a matching checksum in assets list.")
}

func (g *ReleaseManager) lookupAssetWithVersion(os string, arch string, version string) (asset *Asset, err error) {
	g.mu.RLock()
	defer g.mu.RUnlock()

	if g.updateAssetsMap == nil {
		return nil, fmt.Errorf("No updates available.")
	}

	if g.updateAssetsMap[os] == nil {
		return nil, fmt.Errorf("No such OS.")
	}

	if g.updateAssetsMap[os][arch] == nil {
		return nil, fmt.Errorf("No such Arch.")
	}

	for _, a := range g.updateAssetsMap[os][arch] {
		if a.v.String() == version {
			return a, nil
		}
	}

	return nil, fmt.Errorf("Could not find a matching version in assets list.")
}

func (g *ReleaseManager) pushAsset(os string, arch string, asset *Asset) (err error) {
	g.mu.Lock()
	defer g.mu.Unlock()

	version := asset.v

	asset.OS = os
	asset.Arch = arch

	if version.EQ(emptyVersion) {
		return fmt.Errorf("Missing asset version.")
	}

	var localfile string
	if localfile, err = downloadAsset(asset.URL); err != nil {
		return err
	}

	if asset.Checksum, err = checksumForFile(localfile); err != nil {
		return err
	}

	if asset.Signature, err = signatureForFile(localfile); err != nil {
		return err
	}

	// Pushing version.
	if g.updateAssetsMap[os] == nil {
		g.updateAssetsMap[os] = make(map[string]map[string]*Asset)
	}
	if g.updateAssetsMap[os][arch] == nil {
		g.updateAssetsMap[os][arch] = make(map[string]*Asset)
	}
	g.updateAssetsMap[os][arch][version.String()] = asset

	// Setting latest version.
	if g.latestAssetsMap[os] == nil {
		g.latestAssetsMap[os] = make(map[string]*Asset)
	}

	// Only considering non-manoto versions for the latestAssetsMap
	if !buildStringContainsManoto(asset.v) {
		if g.latestAssetsMap[os][arch] == nil {
			g.latestAssetsMap[os][arch] = asset
		} else {
			// Compare against already set version.
			if asset.v.GT(g.latestAssetsMap[os][arch].v) {
				g.latestAssetsMap[os][arch] = asset
			}
		}
	}

	return nil
}

func getAssetInfo(s string) (*AssetInfo, error) {
	matches := updateAssetRe.FindStringSubmatch(s)
	if len(matches) >= 3 {
		if matches[1] != OS.Windows && matches[1] != OS.Linux && matches[1] != OS.Darwin {
			return nil, fmt.Errorf("Unknown OS: \"%s\".", matches[1])
		}
		if matches[2] != Arch.X64 && matches[2] != Arch.X86 && matches[2] != Arch.ARM {
			return nil, fmt.Errorf("Unknown architecture \"%s\".", matches[2])
		}
		info := &AssetInfo{
			OS:   matches[1],
			Arch: matches[2],
		}
		return info, nil
	}
	return nil, fmt.Errorf("Could not find asset info.")
}

func isUpdateAsset(s string) bool {
	return updateAssetRe.MatchString(s)
}
<commit_msg>add android support<commit_after>package server

import (
	"fmt"
	"regexp"
	"sort"
	"sync"

	"github.com/blang/semver"
	"github.com/google/go-github/github"
)

var (
	updateAssetRe = regexp.MustCompile(`^update_(darwin|windows|linux|android)_(arm|386|amd64)\.?.*$`)

	emptyVersion semver.Version
)

// Arch holds architecture names.
var Arch = struct {
	X64 string
	X86 string
	ARM string
}{
	"amd64",
	"386",
	"arm",
}

// OS holds operating system names.
var OS = struct {
	Windows string
	Linux   string
	Darwin  string
	Android string
}{
	"windows",
	"linux",
	"darwin",
	"android"
}

// Release struct represents a single github release.
type Release struct {
	id      int
	URL     string
	Version semver.Version
	Assets  []Asset
}

type releasesByID []Release

// Asset struct represents a file included as part of a Release.
type Asset struct {
	id        int
	v         semver.Version
	Name      string
	URL       string
	LocalFile string
	Checksum  string
	Signature string
	AssetInfo
}

// AssetInfo struct holds OS and Arch information of an asset.
type AssetInfo struct {
	OS   string
	Arch string
}

// ReleaseManager struct defines a repository to pull releases from.
type ReleaseManager struct {
	client          *github.Client
	owner           string
	repo            string
	updateAssetsMap map[string]map[string]map[string]*Asset
	latestAssetsMap map[string]map[string]*Asset
	mu              *sync.RWMutex
}

func (a releasesByID) Len() int {
	return len(a)
}

func (a releasesByID) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a releasesByID) Less(i, j int) bool {
	return a[i].id < a[j].id
}

// NewReleaseManager creates a wrapper of github.Client.
func NewReleaseManager(owner string, repo string) *ReleaseManager {

	ghc := &ReleaseManager{
		client:          github.NewClient(nil),
		owner:           owner,
		repo:            repo,
		mu:              new(sync.RWMutex),
		updateAssetsMap: make(map[string]map[string]map[string]*Asset),
		latestAssetsMap: make(map[string]map[string]*Asset),
	}

	return ghc
}

// getReleases queries github for all product releases.
func (g *ReleaseManager) getReleases() ([]Release, error) {
	var releases []Release

	for page := 1; true; page++ {
		opt := &github.ListOptions{Page: page}

		rels, _, err := g.client.Repositories.ListReleases(g.owner, g.repo, opt)

		if err != nil {
			return nil, err
		}

		if len(rels) == 0 {
			break
		}

		releases = make([]Release, 0, len(rels))

		for i := range rels {
			version := *rels[i].TagName
			v, err := semver.Parse(version)
			if err != nil {
				log.Debugf("Release %q is not semantically versioned (%q). Skipping.", version, err)
				continue
			}
			rel := Release{
				id:      *rels[i].ID,
				URL:     *rels[i].ZipballURL,
				Version: v,
			}
			rel.Assets = make([]Asset, 0, len(rels[i].Assets))
			for _, asset := range rels[i].Assets {
				rel.Assets = append(rel.Assets, Asset{
					id:   *asset.ID,
					Name: *asset.Name,
					URL:  *asset.BrowserDownloadURL,
				})
			}
			log.Debugf("Release %q has %d assets...", version, len(rel.Assets))
			releases = append(releases, rel)
		}
	}

	sort.Sort(sort.Reverse(releasesByID(releases)))

	return releases, nil
}

// UpdateAssetsMap will pull published releases, scan for compatible
// update-only binaries and will add them to the updateAssetsMap.
func (g *ReleaseManager) UpdateAssetsMap() (err error) {

	var rs []Release

	log.Debugf("Getting releases...")
	if rs, err = g.getReleases(); err != nil {
		return err
	}

	// Resetting file hashes.
	fileHashMapMu.Lock()
	fileHashMap = map[string]string{}
	fileHashMapMu.Unlock()

	log.Debugf("Getting assets...")
	for i := range rs {
		log.Debugf("Getting assets for release %q...", rs[i].Version)
		for j := range rs[i].Assets {
			log.Debugf("Found %q.", rs[i].Assets[j].Name)
			// Does this asset represent a binary update?
			if isUpdateAsset(rs[i].Assets[j].Name) {
				log.Debugf("%q is an auto-update asset.", rs[i].Assets[j].Name)
				asset := rs[i].Assets[j]
				asset.v = rs[i].Version
				info, err := getAssetInfo(asset.Name)
				if err != nil {
					return fmt.Errorf("Could not get asset info: %q", err)
				}
				if err = g.pushAsset(info.OS, info.Arch, &asset); err != nil {
					return fmt.Errorf("Could not push asset: %q", err)
				}
			} else {
				log.Debugf("%q is not an auto-update asset. Skipping.", rs[i].Assets[j].Name)
			}
		}
	}

	return nil
}

func (g *ReleaseManager) getProductUpdate(os string, arch string) (asset *Asset, err error) {
	g.mu.RLock()
	defer g.mu.RUnlock()

	if g.latestAssetsMap == nil {
		return nil, fmt.Errorf("No updates available.")
	}

	if g.latestAssetsMap[os] == nil {
		return nil, fmt.Errorf("No such OS.")
	}

	if g.latestAssetsMap[os][arch] == nil {
		return nil, fmt.Errorf("No such Arch.")
	}

	return g.latestAssetsMap[os][arch], nil
}

func (g *ReleaseManager) lookupAssetWithChecksum(os string, arch string, checksum string) (asset *Asset, err error) {
	g.mu.RLock()
	defer g.mu.RUnlock()

	if g.updateAssetsMap == nil {
		return nil, fmt.Errorf("No updates available.")
	}

	if g.updateAssetsMap[os] == nil {
		return nil, fmt.Errorf("No such OS.")
	}

	if g.updateAssetsMap[os][arch] == nil {
		return nil, fmt.Errorf("No such Arch.")
	}

	for _, a := range g.updateAssetsMap[os][arch] {
		if a.Checksum == checksum {
			return a, nil
		}
	}

	return nil, fmt.Errorf("Could not find a matching checksum in assets list.")
}

func (g *ReleaseManager) lookupAssetWithVersion(os string, arch string, version string) (asset *Asset, err error) {
	g.mu.RLock()
	defer g.mu.RUnlock()

	if g.updateAssetsMap == nil {
		return nil, fmt.Errorf("No updates available.")
	}

	if g.updateAssetsMap[os] == nil {
		return nil, fmt.Errorf("No such OS.")
	}

	if g.updateAssetsMap[os][arch] == nil {
		return nil, fmt.Errorf("No such Arch.")
	}

	for _, a := range g.updateAssetsMap[os][arch] {
		if a.v.String() == version {
			return a, nil
		}
	}

	return nil, fmt.Errorf("Could not find a matching version in assets list.")
}

func (g *ReleaseManager) pushAsset(os string, arch string, asset *Asset) (err error) {
	g.mu.Lock()
	defer g.mu.Unlock()

	version := asset.v

	asset.OS = os
	asset.Arch = arch

	if version.EQ(emptyVersion) {
		return fmt.Errorf("Missing asset version.")
	}

	var localfile string
	if localfile, err = downloadAsset(asset.URL); err != nil {
		return err
	}

	if asset.Checksum, err = checksumForFile(localfile); err != nil {
		return err
	}

	if asset.Signature, err = signatureForFile(localfile); err != nil {
		return err
	}

	// Pushing version.
	if g.updateAssetsMap[os] == nil {
		g.updateAssetsMap[os] = make(map[string]map[string]*Asset)
	}
	if g.updateAssetsMap[os][arch] == nil {
		g.updateAssetsMap[os][arch] = make(map[string]*Asset)
	}
	g.updateAssetsMap[os][arch][version.String()] = asset

	// Setting latest version.
	if g.latestAssetsMap[os] == nil {
		g.latestAssetsMap[os] = make(map[string]*Asset)
	}

	// Only considering non-manoto versions for the latestAssetsMap
	if !buildStringContainsManoto(asset.v) {
		if g.latestAssetsMap[os][arch] == nil {
			g.latestAssetsMap[os][arch] = asset
		} else {
			// Compare against already set version.
			if asset.v.GT(g.latestAssetsMap[os][arch].v) {
				g.latestAssetsMap[os][arch] = asset
			}
		}
	}

	return nil
}

func getAssetInfo(s string) (*AssetInfo, error) {
	matches := updateAssetRe.FindStringSubmatch(s)
	if len(matches) >= 3 {
		if matches[1] != OS.Windows && matches[1] != OS.Linux && matches[1] != OS.Darwin {
			return nil, fmt.Errorf("Unknown OS: \"%s\".", matches[1])
		}
		if matches[2] != Arch.X64 && matches[2] != Arch.X86 && matches[2] != Arch.ARM {
			return nil, fmt.Errorf("Unknown architecture \"%s\".", matches[2])
		}
		info := &AssetInfo{
			OS:   matches[1],
			Arch: matches[2],
		}
		return info, nil
	}
	return nil, fmt.Errorf("Could not find asset info.")
}

func isUpdateAsset(s string) bool {
	return updateAssetRe.MatchString(s)
}
<|endoftext|>


<commit_before>// Copyright 2016 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build js

package vorbis

import (
	"errors"
	"io"
	"io/ioutil"

	"github.com/gopherjs/gopherjs/js"
	"github.com/hajimehoshi/ebiten/audio"
)

// TODO: This just uses decodeAudioData can treat audio files other than Ogg/Vorbis.
// TODO: This doesn't work on iOS which doesn't have Ogg/Vorbis decoder.

type Stream struct {
	leftData   []float32
	rightData  []float32
	posInBytes int
}

func (s *Stream) Read(b []byte) (int, error) {
	l := len(s.leftData)*4 - s.posInBytes
	if l > len(b) {
		l = len(b)
	}
	l = l / 4 * 4
	for i := 0; i < l/4; i++ {
		il := int16(s.leftData[s.posInBytes/4+i] * (1 << 15))
		ir := int16(s.rightData[s.posInBytes/4+i] * (1 << 15))
		b[4*i] = uint8(il)
		b[4*i+1] = uint8(il >> 8)
		b[4*i+2] = uint8(ir)
		b[4*i+3] = uint8(ir >> 8)
	}
	s.posInBytes += l
	if s.posInBytes == len(s.leftData)*4 {
		return l, io.EOF
	}
	return l, nil
}

func (s *Stream) Seek(offset int64, whence int) (int64, error) {
	next := int64(0)
	switch whence {
	case 0:
		next = offset
	case 1:
		next = int64(s.posInBytes) + offset
	case 2:
		next = int64(len(s.leftData)*4) + offset
	}
	s.posInBytes = int(next)
	return next, nil
}

func (s *Stream) Close() error {
	return nil
}

func (s *Stream) Size() int64 {
	return int64(len(s.leftData) * 4)
}

func Decode(context *audio.Context, src audio.ReadSeekCloser) (*Stream, error) {
	b, err := ioutil.ReadAll(src)
	if err != nil {
		return nil, err
	}
	if err := src.Close(); err != nil {
		return nil, err
	}
	s := &Stream{}
	ch := make(chan error)

	klass := js.Global.Get("OfflineAudioContext")
	if klass == js.Undefined {
		klass = js.Global.Get("webkitOfflineAudioContext")
	}
	if klass == js.Undefined {
		return nil, errors.New("vorbis: OfflineAudioContext is not available")
	}
	// TODO: 1 is a correct second argument?
	// At least, 1 works on Chrome and Firefox but not Safari.
	oc := klass.New(2, 1, context.SampleRate())
	oc.Call("decodeAudioData", js.NewArrayBuffer(b), func(buf *js.Object) {
		s.leftData = buf.Call("getChannelData", 0).Interface().([]float32)
		s.rightData = buf.Call("getChannelData", 1).Interface().([]float32)
		close(ch)
	}, func() {
		ch <- errors.New("vorbis: decodeAudioData failed (Safari might not be able to decode Ogg/Vorbis).")
		close(ch)
	})
	if err := <-ch; err != nil {
		return nil, err
	}
	return s, nil
}
<commit_msg>audio/vorbis: Use consts<commit_after>// Copyright 2016 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build js

package vorbis

import (
	"errors"
	"io"
	"io/ioutil"

	"github.com/gopherjs/gopherjs/js"
	"github.com/hajimehoshi/ebiten/audio"
)

// TODO: This just uses decodeAudioData can treat audio files other than Ogg/Vorbis.
// TODO: This doesn't work on iOS which doesn't have Ogg/Vorbis decoder.

type Stream struct {
	leftData   []float32
	rightData  []float32
	posInBytes int
}

func (s *Stream) Read(b []byte) (int, error) {
	l := len(s.leftData)*4 - s.posInBytes
	if l > len(b) {
		l = len(b)
	}
	l = l / 4 * 4
	for i := 0; i < l/4; i++ {
		il := int16(s.leftData[s.posInBytes/4+i] * (1 << 15))
		ir := int16(s.rightData[s.posInBytes/4+i] * (1 << 15))
		b[4*i] = uint8(il)
		b[4*i+1] = uint8(il >> 8)
		b[4*i+2] = uint8(ir)
		b[4*i+3] = uint8(ir >> 8)
	}
	s.posInBytes += l
	if s.posInBytes == len(s.leftData)*4 {
		return l, io.EOF
	}
	return l, nil
}

func (s *Stream) Seek(offset int64, whence int) (int64, error) {
	next := int64(0)
	switch whence {
	case io.SeekStart:
		next = offset
	case io.SeekCurrent:
		next = int64(s.posInBytes) + offset
	case io.SeekEnd:
		next = int64(len(s.leftData)*4) + offset
	}
	s.posInBytes = int(next)
	return next, nil
}

func (s *Stream) Close() error {
	return nil
}

func (s *Stream) Size() int64 {
	return int64(len(s.leftData) * 4)
}

func Decode(context *audio.Context, src audio.ReadSeekCloser) (*Stream, error) {
	b, err := ioutil.ReadAll(src)
	if err != nil {
		return nil, err
	}
	if err := src.Close(); err != nil {
		return nil, err
	}
	s := &Stream{}
	ch := make(chan error)

	klass := js.Global.Get("OfflineAudioContext")
	if klass == js.Undefined {
		klass = js.Global.Get("webkitOfflineAudioContext")
	}
	if klass == js.Undefined {
		return nil, errors.New("vorbis: OfflineAudioContext is not available")
	}
	// TODO: 1 is a correct second argument?
	// At least, 1 works on Chrome and Firefox but not Safari.
	oc := klass.New(2, 1, context.SampleRate())
	oc.Call("decodeAudioData", js.NewArrayBuffer(b), func(buf *js.Object) {
		s.leftData = buf.Call("getChannelData", 0).Interface().([]float32)
		s.rightData = buf.Call("getChannelData", 1).Interface().([]float32)
		close(ch)
	}, func() {
		ch <- errors.New("vorbis: decodeAudioData failed (Safari might not be able to decode Ogg/Vorbis).")
		close(ch)
	})
	if err := <-ch; err != nil {
		return nil, err
	}
	return s, nil
}
<|endoftext|>


<commit_before>// TODO: Encode/Decode map[string]string

// Package csvstruct provides methods to decode a CSV file into a struct.
package csvstruct

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io"
	"reflect"
	"strconv"
)

// Decoder reads and decodes CSV rows from an input stream.
type Decoder interface {
	// DecodeNext populates v with the values from the next row in the
	// Decoder's Reader.
	//
	// On the first call to DecodeNext, the first row in the reader will be
	// used as the header row to map CSV fields to struct fields, and the
	// second row will be read to populate v.
	DecodeNext(v interface{}) error
}

// DecodeOpts specifies options to modify decoding behavior.
type DecoderOpts struct {
	Comma            rune // field delimiter (set to ',' by default)
	Comment          rune // comment character for start of line
	LazyQuotes       bool // allow lazy quotes
	TrimLeadingSpace bool // trim leading space
}

type decoder struct {
	r    csv.Reader
	hm   map[string]int
	opts DecoderOpts
}

// NewDecoder returns a Decoder that reads from r.
func NewDecoder(r io.Reader) Decoder {
	return NewDecoderOpts(r, DecoderOpts{})
}

func NewDecoderOpts(r io.Reader, opts DecoderOpts) Decoder {
	csvr := csv.NewReader(r)
	if opts.Comma != rune(0) {
		csvr.Comma = opts.Comma
	}
	if opts.Comment != rune(0) {
		csvr.Comment = opts.Comment
	}
	csvr.LazyQuotes = opts.LazyQuotes
	csvr.TrimLeadingSpace = opts.TrimLeadingSpace
	return &decoder{r: *csvr, opts: opts}
}

func (d *decoder) DecodeNext(v interface{}) error {
	line, err := d.read()
	if err != nil {
		return err
	}

	// v is nil, skip this line and proceed.
	if v == nil {
		return nil
	}

	rv := reflect.ValueOf(v)
	if rv.Kind() != reflect.Ptr {
		return errors.New("must be pointer")
	}
	rv = rv.Elem()

	switch rv.Type().Kind() {
	case reflect.Map:
		return d.decodeMap(v, line)
	case reflect.Struct:
		return d.decodeStruct(v, line)
	default:
		return errors.New("must be pointer to struct")
	}
}
func (d *decoder) decodeMap(v interface{}, line []string) error {
	rv := reflect.ValueOf(v)
	t := rv.Elem().Type()
	if t.Key().Kind() != reflect.String {
		return errors.New("map key must be string")
	}
	switch t.Elem().Kind() {
	case reflect.String:
		m := *(v.(*map[string]string))
		for hv, hidx := range d.hm {
			m[hv] = line[hidx]
		}
	// TODO: Support arbitrary map values by parsing string values
	case reflect.Interface:
		return errors.New("TODO")
	default:
		return fmt.Errorf("can't decode type %v", t.Elem().Kind())
	}
	return nil
}

func (d *decoder) decodeStruct(v interface{}, line []string) error {
	rv := reflect.ValueOf(v).Elem()
	t := rv.Type()
	for i := 0; i < t.NumField(); i++ {
		f := t.Field(i)
		if f.Anonymous {
			continue
		}
		n := f.Name
		if f.Tag.Get("csv") != "" {
			n = f.Tag.Get("csv")
			if n == "-" {
				continue
			}
		}
		idx, ok := d.hm[n]
		if !ok {
			// Unmapped header value
			continue
		}
		strv := line[idx]
		vf := rv.FieldByName(f.Name)
		if vf.CanSet() {
			switch vf.Kind() {
			case reflect.String:
				vf.SetString(strv)
			case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
				i, err := strconv.ParseInt(strv, 10, 64)
				if err != nil {
					return fmt.Errorf("error decoding: %v", err)
				}
				vf.SetInt(i)
			case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
				u, err := strconv.ParseUint(strv, 10, 64)
				if err != nil {
					return fmt.Errorf("error decoding: %v", err)
				}
				vf.SetUint(u)
			case reflect.Float64:
				f, err := strconv.ParseFloat(strv, 64)
				if err != nil {
					return fmt.Errorf("error decoding: %v", err)
				}
				vf.SetFloat(f)
			case reflect.Bool:
				b, err := strconv.ParseBool(strv)
				if err != nil {
					return fmt.Errorf("error decoding: %v", err)
				}
				vf.SetBool(b)
			default:
				return fmt.Errorf("can't decode type %v", vf.Type())
			}
		}
	}
	return nil
}

func (d *decoder) read() ([]string, error) {
	if d.hm == nil {
		// First run; read header row
		header, err := d.r.Read()
		if err != nil {
			return nil, fmt.Errorf("error reading headers: %v", err)
		}
		d.hm = reverse(header)
	}
	// Read data row into []string
	return d.r.Read()
}

func reverse(in []string) map[string]int {
	m := make(map[string]int, len(in))
	for i, v := range in {
		m[v] = i
	}
	return m
}
<commit_msg>remove TODO<commit_after>// Package csvstruct provides methods to decode a CSV file into a struct.
package csvstruct

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io"
	"reflect"
	"strconv"
)

// Decoder reads and decodes CSV rows from an input stream.
type Decoder interface {
	// DecodeNext populates v with the values from the next row in the
	// Decoder's Reader.
	//
	// On the first call to DecodeNext, the first row in the reader will be
	// used as the header row to map CSV fields to struct fields, and the
	// second row will be read to populate v.
	DecodeNext(v interface{}) error
}

// DecodeOpts specifies options to modify decoding behavior.
type DecoderOpts struct {
	Comma            rune // field delimiter (set to ',' by default)
	Comment          rune // comment character for start of line
	LazyQuotes       bool // allow lazy quotes
	TrimLeadingSpace bool // trim leading space
}

type decoder struct {
	r    csv.Reader
	hm   map[string]int
	opts DecoderOpts
}

// NewDecoder returns a Decoder that reads from r.
func NewDecoder(r io.Reader) Decoder {
	return NewDecoderOpts(r, DecoderOpts{})
}

func NewDecoderOpts(r io.Reader, opts DecoderOpts) Decoder {
	csvr := csv.NewReader(r)
	if opts.Comma != rune(0) {
		csvr.Comma = opts.Comma
	}
	if opts.Comment != rune(0) {
		csvr.Comment = opts.Comment
	}
	csvr.LazyQuotes = opts.LazyQuotes
	csvr.TrimLeadingSpace = opts.TrimLeadingSpace
	return &decoder{r: *csvr, opts: opts}
}

func (d *decoder) DecodeNext(v interface{}) error {
	line, err := d.read()
	if err != nil {
		return err
	}

	// v is nil, skip this line and proceed.
	if v == nil {
		return nil
	}

	rv := reflect.ValueOf(v)
	if rv.Kind() != reflect.Ptr {
		return errors.New("must be pointer")
	}
	rv = rv.Elem()

	switch rv.Type().Kind() {
	case reflect.Map:
		return d.decodeMap(v, line)
	case reflect.Struct:
		return d.decodeStruct(v, line)
	default:
		return errors.New("must be pointer to struct")
	}
}
func (d *decoder) decodeMap(v interface{}, line []string) error {
	rv := reflect.ValueOf(v)
	t := rv.Elem().Type()
	if t.Key().Kind() != reflect.String {
		return errors.New("map key must be string")
	}
	switch t.Elem().Kind() {
	case reflect.String:
		m := *(v.(*map[string]string))
		for hv, hidx := range d.hm {
			m[hv] = line[hidx]
		}
	// TODO: Support arbitrary map values by parsing string values
	case reflect.Interface:
		return errors.New("TODO")
	default:
		return fmt.Errorf("can't decode type %v", t.Elem().Kind())
	}
	return nil
}

func (d *decoder) decodeStruct(v interface{}, line []string) error {
	rv := reflect.ValueOf(v).Elem()
	t := rv.Type()
	for i := 0; i < t.NumField(); i++ {
		f := t.Field(i)
		if f.Anonymous {
			continue
		}
		n := f.Name
		if f.Tag.Get("csv") != "" {
			n = f.Tag.Get("csv")
			if n == "-" {
				continue
			}
		}
		idx, ok := d.hm[n]
		if !ok {
			// Unmapped header value
			continue
		}
		strv := line[idx]
		vf := rv.FieldByName(f.Name)
		if vf.CanSet() {
			switch vf.Kind() {
			case reflect.String:
				vf.SetString(strv)
			case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
				i, err := strconv.ParseInt(strv, 10, 64)
				if err != nil {
					return fmt.Errorf("error decoding: %v", err)
				}
				vf.SetInt(i)
			case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
				u, err := strconv.ParseUint(strv, 10, 64)
				if err != nil {
					return fmt.Errorf("error decoding: %v", err)
				}
				vf.SetUint(u)
			case reflect.Float64:
				f, err := strconv.ParseFloat(strv, 64)
				if err != nil {
					return fmt.Errorf("error decoding: %v", err)
				}
				vf.SetFloat(f)
			case reflect.Bool:
				b, err := strconv.ParseBool(strv)
				if err != nil {
					return fmt.Errorf("error decoding: %v", err)
				}
				vf.SetBool(b)
			default:
				return fmt.Errorf("can't decode type %v", vf.Type())
			}
		}
	}
	return nil
}

func (d *decoder) read() ([]string, error) {
	if d.hm == nil {
		// First run; read header row
		header, err := d.r.Read()
		if err != nil {
			return nil, fmt.Errorf("error reading headers: %v", err)
		}
		d.hm = reverse(header)
	}
	// Read data row into []string
	return d.r.Read()
}

func reverse(in []string) map[string]int {
	m := make(map[string]int, len(in))
	for i, v := range in {
		m[v] = i
	}
	return m
}
<|endoftext|>