<commit_before>package kiwi

// This file consists of Sink related structures and functions.
// Outputs accepts incoming log records from Loggers, check them with filters
// and write to output streams if checks passed.

/* Copyright (c) 2016-2019, Alexander I.Grafov <grafov@gmail.com>
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

* Neither the name of kvlog nor the names of its
  contributors may be used to endorse or promote products derived from
  this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ॐ तारे तुत्तारे तुरे स्व */

import (
	"io"
	"sync"
	"sync/atomic"
	"time"
)

// States of the sink.
const (
	sinkClosed int32 = iota - 1
	sinkStopped
	sinkActive
)

// Sinks accepts records through the chanels.
// Each sink has its own channel.
var collector struct {
	sync.RWMutex
	sinks []*Sink
	count int
}

type (
	// Sink used for filtering incoming log records from all logger instances
	// and decides how to filter them. Each output wraps its own io.Writer.
	// Sink methods are safe for concurrent usage.
	Sink struct {
		id     uint
		In     chan chain
		close  chan struct{}
		writer io.Writer
		format Formatter
		state  *int32

		sync.RWMutex
		positiveFilters map[string]Filter
		negativeFilters map[string]Filter
		hiddenKeys      map[string]bool
	}
	chain struct {
		wg    *sync.WaitGroup
		pairs []*Pair
	}
)

// SinkTo creates a new sink for an arbitrary number of loggers.
// There are any number of sinks may be created for saving incoming log
// records to different places.
// The sink requires explicit start with Start() before usage.
// That allows firstly setup filters before sink will really accept any records.
func SinkTo(w io.Writer, fn Formatter) *Sink {
	collector.RLock()
	for i, sink := range collector.sinks {
		if sink.writer == w {
			collector.sinks[i].format = fn
			collector.RUnlock()
			return collector.sinks[i]
		}
	}
	collector.RUnlock()
	var (
		state = sinkStopped
		sink  = &Sink{
			In:              make(chan chain, 16),
			close:           make(chan struct{}),
			format:          fn,
			state:           &state,
			writer:          w,
			positiveFilters: make(map[string]Filter),
			negativeFilters: make(map[string]Filter),
			hiddenKeys:      make(map[string]bool),
		}
	)
	collector.Lock()
	sink.id = uint(collector.count)
	collector.sinks = append(collector.sinks, sink)
	collector.count++
	collector.Unlock()
	go processSink(sink)
	return sink
}

// HasKey sets restriction for records output.
// Only the records WITH any of the keys will be passed to output.
func (s *Sink) HasKey(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			s.positiveFilters[key] = &keyFilter{}
			delete(s.negativeFilters, key)
		}
		s.Unlock()
	}
	return s
}

// HasNotKey sets restriction for records output.
// Only the records WITHOUT any of the keys will be passed to output.
func (s *Sink) HasNotKey(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			s.negativeFilters[key] = &keyFilter{}
			delete(s.positiveFilters, key)
		}
		s.Unlock()
	}
	return s
}

// HasValue sets restriction for records output.
// A record passed to output if the key equal one of any of the listed values.
func (s *Sink) HasValue(key string, vals ...string) *Sink {
	if len(vals) == 0 {
		return s.HasKey(key)
	}
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		s.positiveFilters[key] = &valsFilter{Vals: vals}
		delete(s.negativeFilters, key)
		s.Unlock()
	}
	return s
}

// HasNotValue sets restriction for records output.
func (s *Sink) HasNotValue(key string, vals ...string) *Sink {
	if len(vals) == 0 {
		return s.HasNotKey(key)
	}
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		s.negativeFilters[key] = &valsFilter{Vals: vals}
		delete(s.positiveFilters, key)
		s.Unlock()
	}
	return s
}

// Int64Range sets restriction for records output.
func (s *Sink) Int64Range(key string, from, to int64) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.negativeFilters, key)
		s.positiveFilters[key] = &int64RangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// Int64NotRange sets restriction for records output.
func (s *Sink) Int64NotRange(key string, from, to int64) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.positiveFilters, key)
		s.negativeFilters[key] = &int64RangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// Float64Range sets restriction for records output.
func (s *Sink) Float64Range(key string, from, to float64) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.negativeFilters, key)
		s.positiveFilters[key] = &float64RangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// Float64NotRange sets restriction for records output.
func (s *Sink) Float64NotRange(key string, from, to float64) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.positiveFilters, key)
		s.negativeFilters[key] = &float64RangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// TimeRange sets restriction for records output.
func (s *Sink) TimeRange(key string, from, to time.Time) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.negativeFilters, key)
		s.positiveFilters[key] = &timeRangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// TimeNotRange sets restriction for records output.
func (s *Sink) TimeNotRange(key string, from, to time.Time) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.positiveFilters, key)
		s.negativeFilters[key] = &timeRangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// WithFilter setup custom filtering function for values for a specific key.
// Custom filter should realize Filter interface. All custom filters treated
// as positive filters. So if the filter returns true then it will be passed.
func (s *Sink) WithFilter(key string, customFilter Filter) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.negativeFilters, key)
		s.positiveFilters[key] = customFilter
		s.Unlock()
	}
	return s
}

// Reset all filters for the keys for the output.
func (s *Sink) Reset(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			delete(s.positiveFilters, key)
			delete(s.negativeFilters, key)
		}
		s.Unlock()
	}
	return s
}

// Hide keys from the output. Other keys in record will be displayed
// but not hidden keys.
func (s *Sink) Hide(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			s.hiddenKeys[key] = true
		}
		s.Unlock()
	}
	return s
}

// Unhide previously hidden keys. They will be displayed in the output again.
func (s *Sink) Unhide(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			delete(s.hiddenKeys, key)
		}
		s.Unlock()
	}
	return s
}

// Stop stops writing to the output.
func (s *Sink) Stop() *Sink {
	atomic.StoreInt32(s.state, sinkStopped)
	return s
}

// Start writing to the output.
// After creation of a new sink it will paused and you need explicitly start it.
// It allows setup the filters before the sink will accepts any records.
func (s *Sink) Start() *Sink {
	atomic.StoreInt32(s.state, sinkActive)
	return s
}

// Close closes the sink. It flushes records for the sink before closing.
func (s *Sink) Close() {
	if atomic.LoadInt32(s.state) > sinkClosed {
		atomic.StoreInt32(s.state, sinkClosed)
		s.close <- struct{}{}
		collector.Lock()
		collector.count--
		collector.sinks = append(collector.sinks[0:s.id], collector.sinks[s.id+1:]...)
		collector.Unlock()
	}
}

func processSink(s *Sink) {
	var (
		record chain
		ok     bool
	)
	for {
		select {
		case record, ok = <-s.In:
			if !ok {
				return
			}
			if atomic.LoadInt32(s.state) < sinkActive {
				record.wg.Done()
				continue
			}
			s.RLock()
			var filter Filter
			for _, pair := range record.pairs {
				// Negative conditions have highest priority
				if filter, ok = s.negativeFilters[pair.Key]; ok {
					if filter.Check(pair.Key, pair.Val) {
						goto skipRecord
					}
				}
				// At last check for positive conditions
				if filter, ok = s.positiveFilters[pair.Key]; ok {
					if !filter.Check(pair.Key, pair.Val) {
						goto skipRecord
					}
				}
			}
			s.formatRecord(record.pairs)
		skipRecord:
			s.RUnlock()
			record.wg.Done()
		case <-s.close:
			s.Lock()
			s.positiveFilters = nil
			s.negativeFilters = nil
			s.hiddenKeys = nil
			s.Unlock()
			return
		}
	}
}

func (s *Sink) formatRecord(record []*Pair) {
	s.format.Begin()
	for _, pair := range record {
		if ok := s.hiddenKeys[pair.Key]; ok {
			continue
		}
		s.format.Pair(pair.Key, pair.Val, pair.Type)
	}
	s.writer.Write(s.format.Finish())
}

const flushTimeout = 3 * time.Second

func sinkRecord(rec []*Pair) {
	var wg sync.WaitGroup
	collector.RLock()
	for _, s := range collector.sinks {
		if atomic.LoadInt32(s.state) == sinkActive {
			wg.Add(1)
			s.In <- chain{&wg, rec}
		}
	}
	collector.RUnlock()
	var c = make(chan struct{})
	go func() {
		defer close(c)
		wg.Wait()
	}()
	select {
	case <-c:
		return
	case <-time.After(flushTimeout):
		return
	}
}
<commit_msg>Fix side effects of sink.Close() refs #4 and #5<commit_after>package kiwi

// This file consists of Sink related structures and functions.
// Outputs accepts incoming log records from Loggers, check them with filters
// and write to output streams if checks passed.

/* Copyright (c) 2016-2019, Alexander I.Grafov <grafov@gmail.com>
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

* Neither the name of kvlog nor the names of its
  contributors may be used to endorse or promote products derived from
  this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ॐ तारे तुत्तारे तुरे स्व */

import (
	"io"
	"sync"
	"sync/atomic"
	"time"
)

// States of the sink.
const (
	sinkClosed int32 = iota - 1
	sinkStopped
	sinkActive
)

// Sinks accepts records through the chanels.
// Each sink has its own channel.
var collector struct {
	sync.RWMutex
	sinks []*Sink
}

type (
	// Sink used for filtering incoming log records from all logger instances
	// and decides how to filter them. Each output wraps its own io.Writer.
	// Sink methods are safe for concurrent usage.
	Sink struct {
		In     chan chain
		close  chan struct{}
		writer io.Writer
		format Formatter
		state  *int32

		sync.RWMutex
		positiveFilters map[string]Filter
		negativeFilters map[string]Filter
		hiddenKeys      map[string]bool
	}
	chain struct {
		wg    *sync.WaitGroup
		pairs []*Pair
	}
)

// SinkTo creates a new sink for an arbitrary number of loggers.
// There are any number of sinks may be created for saving incoming log
// records to different places.
// The sink requires explicit start with Start() before usage.
// That allows firstly setup filters before sink will really accept any records.
func SinkTo(w io.Writer, fn Formatter) *Sink {
	collector.RLock()
	for i, sink := range collector.sinks {
		if sink.writer == w {
			collector.sinks[i].format = fn
			collector.RUnlock()
			return collector.sinks[i]
		}
	}
	collector.RUnlock()
	var (
		state = sinkStopped
		sink  = &Sink{
			In:              make(chan chain, 16),
			close:           make(chan struct{}),
			format:          fn,
			state:           &state,
			writer:          w,
			positiveFilters: make(map[string]Filter),
			negativeFilters: make(map[string]Filter),
			hiddenKeys:      make(map[string]bool),
		}
	)
	collector.Lock()
	collector.sinks = append(collector.sinks, sink)
	collector.Unlock()
	go processSink(sink)
	return sink
}

// HasKey sets restriction for records output.
// Only the records WITH any of the keys will be passed to output.
func (s *Sink) HasKey(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			s.positiveFilters[key] = &keyFilter{}
			delete(s.negativeFilters, key)
		}
		s.Unlock()
	}
	return s
}

// HasNotKey sets restriction for records output.
// Only the records WITHOUT any of the keys will be passed to output.
func (s *Sink) HasNotKey(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			s.negativeFilters[key] = &keyFilter{}
			delete(s.positiveFilters, key)
		}
		s.Unlock()
	}
	return s
}

// HasValue sets restriction for records output.
// A record passed to output if the key equal one of any of the listed values.
func (s *Sink) HasValue(key string, vals ...string) *Sink {
	if len(vals) == 0 {
		return s.HasKey(key)
	}
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		s.positiveFilters[key] = &valsFilter{Vals: vals}
		delete(s.negativeFilters, key)
		s.Unlock()
	}
	return s
}

// HasNotValue sets restriction for records output.
func (s *Sink) HasNotValue(key string, vals ...string) *Sink {
	if len(vals) == 0 {
		return s.HasNotKey(key)
	}
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		s.negativeFilters[key] = &valsFilter{Vals: vals}
		delete(s.positiveFilters, key)
		s.Unlock()
	}
	return s
}

// Int64Range sets restriction for records output.
func (s *Sink) Int64Range(key string, from, to int64) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.negativeFilters, key)
		s.positiveFilters[key] = &int64RangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// Int64NotRange sets restriction for records output.
func (s *Sink) Int64NotRange(key string, from, to int64) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.positiveFilters, key)
		s.negativeFilters[key] = &int64RangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// Float64Range sets restriction for records output.
func (s *Sink) Float64Range(key string, from, to float64) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.negativeFilters, key)
		s.positiveFilters[key] = &float64RangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// Float64NotRange sets restriction for records output.
func (s *Sink) Float64NotRange(key string, from, to float64) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.positiveFilters, key)
		s.negativeFilters[key] = &float64RangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// TimeRange sets restriction for records output.
func (s *Sink) TimeRange(key string, from, to time.Time) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.negativeFilters, key)
		s.positiveFilters[key] = &timeRangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// TimeNotRange sets restriction for records output.
func (s *Sink) TimeNotRange(key string, from, to time.Time) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.positiveFilters, key)
		s.negativeFilters[key] = &timeRangeFilter{From: from, To: to}
		s.Unlock()
	}
	return s
}

// WithFilter setup custom filtering function for values for a specific key.
// Custom filter should realize Filter interface. All custom filters treated
// as positive filters. So if the filter returns true then it will be passed.
func (s *Sink) WithFilter(key string, customFilter Filter) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		delete(s.negativeFilters, key)
		s.positiveFilters[key] = customFilter
		s.Unlock()
	}
	return s
}

// Reset all filters for the keys for the output.
func (s *Sink) Reset(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			delete(s.positiveFilters, key)
			delete(s.negativeFilters, key)
		}
		s.Unlock()
	}
	return s
}

// Hide keys from the output. Other keys in record will be displayed
// but not hidden keys.
func (s *Sink) Hide(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			s.hiddenKeys[key] = true
		}
		s.Unlock()
	}
	return s
}

// Unhide previously hidden keys. They will be displayed in the output again.
func (s *Sink) Unhide(keys ...string) *Sink {
	if atomic.LoadInt32(s.state) > sinkClosed {
		s.Lock()
		for _, key := range keys {
			delete(s.hiddenKeys, key)
		}
		s.Unlock()
	}
	return s
}

// Stop stops writing to the output.
func (s *Sink) Stop() *Sink {
	atomic.StoreInt32(s.state, sinkStopped)
	return s
}

// Start writing to the output.
// After creation of a new sink it will paused and you need explicitly start it.
// It allows setup the filters before the sink will accepts any records.
func (s *Sink) Start() *Sink {
	atomic.StoreInt32(s.state, sinkActive)
	return s
}

// Close closes the sink. It flushes records for the sink before closing.
func (s *Sink) Close() {
	if atomic.LoadInt32(s.state) > sinkClosed {
		atomic.StoreInt32(s.state, sinkClosed)
		s.close <- struct{}{}
		collector.Lock()
		for i, v := range collector.sinks {
			if s == v {
				collector.sinks[i] = collector.sinks[len(collector.sinks)-1]
				collector.sinks[len(collector.sinks)-1] = nil
				collector.sinks = collector.sinks[:len(collector.sinks)-1]
				break
			}
		}
		collector.Unlock()
	}
}

func processSink(s *Sink) {
	var (
		record chain
		ok     bool
	)
	for {
		select {
		case record, ok = <-s.In:
			if !ok {
				return
			}
			if atomic.LoadInt32(s.state) < sinkActive {
				record.wg.Done()
				continue
			}
			s.RLock()
			var filter Filter
			for _, pair := range record.pairs {
				// Negative conditions have highest priority
				if filter, ok = s.negativeFilters[pair.Key]; ok {
					if filter.Check(pair.Key, pair.Val) {
						goto skipRecord
					}
				}
				// At last check for positive conditions
				if filter, ok = s.positiveFilters[pair.Key]; ok {
					if !filter.Check(pair.Key, pair.Val) {
						goto skipRecord
					}
				}
			}
			s.formatRecord(record.pairs)
		skipRecord:
			s.RUnlock()
			record.wg.Done()
		case <-s.close:
			s.Lock()
			s.positiveFilters = nil
			s.negativeFilters = nil
			s.hiddenKeys = nil
			s.Unlock()
			return
		}
	}
}

func (s *Sink) formatRecord(record []*Pair) {
	s.format.Begin()
	for _, pair := range record {
		if ok := s.hiddenKeys[pair.Key]; ok {
			continue
		}
		s.format.Pair(pair.Key, pair.Val, pair.Type)
	}
	s.writer.Write(s.format.Finish())
}

const flushTimeout = 3 * time.Second

func sinkRecord(rec []*Pair) {
	var wg sync.WaitGroup
	collector.RLock()
	for _, s := range collector.sinks {
		if atomic.LoadInt32(s.state) == sinkActive {
			wg.Add(1)
			s.In <- chain{&wg, rec}
		}
	}
	collector.RUnlock()
	var c = make(chan struct{})
	go func() {
		defer close(c)
		wg.Wait()
	}()
	select {
	case <-c:
		return
	case <-time.After(flushTimeout):
		return
	}
}
<|endoftext|>


<commit_before>// Package smaz is an implementation of the smaz library
// (https://github.com/antirez/smaz) for compressing small strings.
package smaz

import (
	"bytes"
	"errors"

	"github.com/kjk/smaz/trie"
)

var codeStrings = []string{" ",
	"the", "e", "t", "a", "of", "o", "and", "i", "n", "s", "e ", "r", " th",
	" t", "in", "he", "th", "h", "he ", "to", "\r\n", "l", "s ", "d", " a", "an",
	"er", "c", " o", "d ", "on", " of", "re", "of ", "t ", ", ", "is", "u", "at",
	"   ", "n ", "or", "which", "f", "m", "as", "it", "that", "\n", "was", "en",
	"  ", " w", "es", " an", " i", "\r", "f ", "g", "p", "nd", " s", "nd ", "ed ",
	"w", "ed", "http://", "for", "te", "ing", "y ", "The", " c", "ti", "r ", "his",
	"st", " in", "ar", "nt", ",", " to", "y", "ng", " h", "with", "le", "al", "to ",
	"b", "ou", "be", "were", " b", "se", "o ", "ent", "ha", "ng ", "their", "\"",
	"hi", "from", " f", "in ", "de", "ion", "me", "v", ".", "ve", "all", "re ",
	"ri", "ro", "is ", "co", "f t", "are", "ea", ". ", "her", " m", "er ", " p",
	"es ", "by", "they", "di", "ra", "ic", "not", "s, ", "d t", "at ", "ce", "la",
	"h ", "ne", "as ", "tio", "on ", "n t", "io", "we", " a ", "om", ", a", "s o",
	"ur", "li", "ll", "ch", "had", "this", "e t", "g ", "e\r\n", " wh", "ere",
	" co", "e o", "a ", "us", " d", "ss", "\n\r\n", "\r\n\r", "=\"", " be", " e",
	"s a", "ma", "one", "t t", "or ", "but", "el", "so", "l ", "e s", "s,", "no",
	"ter", " wa", "iv", "ho", "e a", " r", "hat", "s t", "ns", "ch ", "wh", "tr",
	"ut", "/", "have", "ly ", "ta", " ha", " on", "tha", "-", " l", "ati", "en ",
	"pe", " re", "there", "ass", "si", " fo", "wa", "ec", "our", "who", "its", "z",
	"fo", "rs", ">", "ot", "un", "<", "im", "th ", "nc", "ate", "><", "ver", "ad",
	" we", "ly", "ee", " n", "id", " cl", "ac", "il", "</", "rt", " wi", "div",
	"e, ", " it", "whi", " ma", "ge", "x", "e c", "men", ".com",
}

var codes = make([][]byte, len(codeStrings))
var codeTrie = trie.New()

func init() {
	for i, code := range codeStrings {
		codes[i] = []byte(code)
		codeTrie.Put([]byte(code), i)
	}
}

func flushVerb(dst *[]byte, verbBuf *bytes.Buffer) {
	d := *dst
	// We can write a max of 255 continuous verbatim characters, because the
	// length of the continous verbatim section is represented by a single byte.
	for verbBuf.Len() > 0 {
		chunk := verbBuf.Next(255)
		if len(chunk) == 1 {
			// 254 is code for a single verbatim byte
			d = append(d, byte(254))
		} else {
			// 255 is code for a verbatim string. It is followed by a byte
			// containing the length of the string.
			d = append(d, byte(255))
			d = append(d, byte(len(chunk)))
		}
		d = append(d, chunk...)
	}
	verbBuf.Reset()
	*dst = d
}

// Compress compresses a byte slice and returns the compressed data.
func Compress(dst, input []byte) []byte {
	dst = dst[0:0]
	var verbBuf bytes.Buffer
	root := codeTrie.Root()

	for len(input) > 0 {
		prefixLen := 0
		code := 0
		node := root
		for i, c := range input {
			next, ok := node.Walk(c)
			if !ok {
				break
			}
			node = next
			if node.Terminal() {
				prefixLen = i + 1
				code = node.Val()
			}
		}

		if prefixLen > 0 {
			input = input[prefixLen:]
			flushVerb(&dst, &verbBuf)
			dst = append(dst, byte(code))
		} else {
			verbBuf.WriteByte(input[0])
			input = input[1:]
		}
	}
	flushVerb(&dst, &verbBuf)
	return dst
}

// ErrDecompression is returned when decompressing invalid smaz-encoded data.
var ErrDecompression = errors.New("Invalid or corrupted compressed data.")

// Decompress decompresses a smaz-compressed byte slice and return a new slice
// with the decompressed data. err is nil if and only if decompression fails
// for any reason (e.g., corrupted data).
func Decompress(compressed []byte) ([]byte, error) {
	decompressed := make([]byte, 0, len(compressed)) // Estimate initial size
	for len(compressed) > 0 {
		n := int(compressed[0])
		switch n {
		case 254: // Verbatim byte
			if len(compressed) < 2 {
				return nil, ErrDecompression
			}
			decompressed = append(decompressed, compressed[1])
			compressed = compressed[2:]
		case 255: // Verbatim string
			if len(compressed) < 2 {
				return nil, ErrDecompression
			}
			n = int(compressed[1])
			if len(compressed) < n+2 {
				return nil, ErrDecompression
			}
			decompressed = append(decompressed, compressed[2:n+2]...)
			compressed = compressed[n+2:]
		default: // Look up encoded value
			d := codes[n]
			decompressed = append(decompressed, d...)
			compressed = compressed[1:]
		}
	}

	return decompressed, nil
}
<commit_msg>speed up compression<commit_after>// Package smaz is an implementation of the smaz library
// (https://github.com/antirez/smaz) for compressing small strings.
package smaz

import (
	"errors"

	"github.com/kjk/smaz/trie"
)

var codeStrings = []string{" ",
	"the", "e", "t", "a", "of", "o", "and", "i", "n", "s", "e ", "r", " th",
	" t", "in", "he", "th", "h", "he ", "to", "\r\n", "l", "s ", "d", " a", "an",
	"er", "c", " o", "d ", "on", " of", "re", "of ", "t ", ", ", "is", "u", "at",
	"   ", "n ", "or", "which", "f", "m", "as", "it", "that", "\n", "was", "en",
	"  ", " w", "es", " an", " i", "\r", "f ", "g", "p", "nd", " s", "nd ", "ed ",
	"w", "ed", "http://", "for", "te", "ing", "y ", "The", " c", "ti", "r ", "his",
	"st", " in", "ar", "nt", ",", " to", "y", "ng", " h", "with", "le", "al", "to ",
	"b", "ou", "be", "were", " b", "se", "o ", "ent", "ha", "ng ", "their", "\"",
	"hi", "from", " f", "in ", "de", "ion", "me", "v", ".", "ve", "all", "re ",
	"ri", "ro", "is ", "co", "f t", "are", "ea", ". ", "her", " m", "er ", " p",
	"es ", "by", "they", "di", "ra", "ic", "not", "s, ", "d t", "at ", "ce", "la",
	"h ", "ne", "as ", "tio", "on ", "n t", "io", "we", " a ", "om", ", a", "s o",
	"ur", "li", "ll", "ch", "had", "this", "e t", "g ", "e\r\n", " wh", "ere",
	" co", "e o", "a ", "us", " d", "ss", "\n\r\n", "\r\n\r", "=\"", " be", " e",
	"s a", "ma", "one", "t t", "or ", "but", "el", "so", "l ", "e s", "s,", "no",
	"ter", " wa", "iv", "ho", "e a", " r", "hat", "s t", "ns", "ch ", "wh", "tr",
	"ut", "/", "have", "ly ", "ta", " ha", " on", "tha", "-", " l", "ati", "en ",
	"pe", " re", "there", "ass", "si", " fo", "wa", "ec", "our", "who", "its", "z",
	"fo", "rs", ">", "ot", "un", "<", "im", "th ", "nc", "ate", "><", "ver", "ad",
	" we", "ly", "ee", " n", "id", " cl", "ac", "il", "</", "rt", " wi", "div",
	"e, ", " it", "whi", " ma", "ge", "x", "e c", "men", ".com",
}

var codes = make([][]byte, len(codeStrings))
var codeTrie = trie.New()

func init() {
	for i, code := range codeStrings {
		codes[i] = []byte(code)
		codeTrie.Put([]byte(code), i)
	}
}

func next(d []byte, n int) ([]byte, []byte) {
	if n >= len(d) {
		return d, nil
	}
	return d[:n], d[n:]
}

func flushVerb(d, verbBuf []byte) ([]byte, []byte) {
	var chunk []byte
	// We can write a max of 255 continuous verbatim characters, because the
	// length of the continous verbatim section is represented by a single byte.
	for len(verbBuf) > 0 {
		chunk, verbBuf = next(verbBuf, 255)
		if len(chunk) == 1 {
			// 254 is code for a single verbatim byte
			d = append(d, byte(254))
		} else {
			// 255 is code for a verbatim string. It is followed by a byte
			// containing the length of the string.
			d = append(d, byte(255))
			d = append(d, byte(len(chunk)))
		}
		d = append(d, chunk...)
	}
	return d, verbBuf[0:0]
}

// Compress compresses a byte slice and returns the compressed data.
func Compress(dst, input []byte) []byte {
	dst = dst[0:0]
	var verbBuf []byte
	root := codeTrie.Root()

	for len(input) > 0 {
		prefixLen := 0
		code := 0
		node := root
		for i, c := range input {
			next, ok := node.Walk(c)
			if !ok {
				break
			}
			node = next
			if node.Terminal() {
				prefixLen = i + 1
				code = node.Val()
			}
		}

		if prefixLen > 0 {
			input = input[prefixLen:]
			dst, verbBuf = flushVerb(dst, verbBuf)
			dst = append(dst, byte(code))
		} else {
			verbBuf = append(verbBuf, input[0])
			input = input[1:]
		}
	}
	dst, _ = flushVerb(dst, verbBuf)
	return dst
}

// ErrDecompression is returned when decompressing invalid smaz-encoded data.
var ErrDecompression = errors.New("Invalid or corrupted compressed data.")

// Decompress decompresses a smaz-compressed byte slice and return a new slice
// with the decompressed data. err is nil if and only if decompression fails
// for any reason (e.g., corrupted data).
func Decompress(compressed []byte) ([]byte, error) {
	decompressed := make([]byte, 0, len(compressed)) // Estimate initial size
	for len(compressed) > 0 {
		n := int(compressed[0])
		switch n {
		case 254: // Verbatim byte
			if len(compressed) < 2 {
				return nil, ErrDecompression
			}
			decompressed = append(decompressed, compressed[1])
			compressed = compressed[2:]
		case 255: // Verbatim string
			if len(compressed) < 2 {
				return nil, ErrDecompression
			}
			n = int(compressed[1])
			if len(compressed) < n+2 {
				return nil, ErrDecompression
			}
			decompressed = append(decompressed, compressed[2:n+2]...)
			compressed = compressed[n+2:]
		default: // Look up encoded value
			d := codes[n]
			decompressed = append(decompressed, d...)
			compressed = compressed[1:]
		}
	}

	return decompressed, nil
}
<|endoftext|>


<commit_before>package kmath

import (
    "sort"
    "errors"
    "math"
)

// Gini returns the Gini coefficient. It is a measure of inequality in a
// frequency distribution. 0 indicates perfect equality, 1 maximum inequality.
//
// See https://en.wikipedia.org/wiki/Gini_coefficient
func Gini(in []float64) float64 {

    var i int
    N := len(in)

	// Get the total of the series
	c := 0.0
	for i=0;i<N;i++ {
		c += in[i]
	}

	// Create a vector Y with what part of the total is each input value.
	y := make([]float64, N)

	for i=0;i<N;i++ {
		y[i] = in[i] / c
	}

    // sort Y values in ascending order
	sort.Float64s(y)

	// Normalize the X
	x := 1.0 / float64(N)

	// Calculate areas below the diagonal
	gini := 0.0
	acc := 0.0
	pacc := 0.0

	for i := 0; i < N; i++ {
		acc += y[i]
		pacc += x
		gini += (pacc - acc) * x
	}

    // Normalize to 0...1 (area of triangle is 0.5 max)
	return 2 * gini
}

// Cover
func Cover(v []float64, lev float64) (float64,float64) {
    var i int
    N := len(v)

    Ad := 0.0
    Au := 0.0

    for i=0;i<N;i++ {
        if v[i]>=lev {
            Au += (v[i]-lev)
            Ad += lev
        } else {
            Ad += v[i]
        }
    }

    n := float64(N)
    return Ad/(lev*n), Au/(lev*n)
}

// Pearson returns the Pearson correlation coeficient of two arrays
func Pearson (a, b [] float64) (float64, error) {

    n := len(a)

    if n!=len(b) {
        return 0,errors.New("lengths differ")
    }

    ma := 0.0
    mb := 0.0

    for i:=0;i<n;i++ {
        ma += a[i]
        mb += b[i]
    }

    ma /= float64(n)
    mb /= float64(n)

    nu := 0.0
    d1 := 0.0
    d2 := 0.0

    var da, db float64

    for i:=0; i<n; i++ {
        da = a[i] - ma
        db = b[i] - mb
        nu += da*db
        d1 += da*da
        d2 += db*db
    }

    return nu / ( math.Sqrt(d1) * math.Sqrt(d2) ), nil
}<commit_msg>gini corrected<commit_after>package kmath

import (
    "sort"
    "errors"
    "math"
)

// Gini returns the Gini coefficient. It is a measure of inequality in a
// frequency distribution. 0 indicates perfect equality, 1 maximum inequality.
//
// See https://en.wikipedia.org/wiki/Gini_coefficient
func Gini(in []float64) float64 {

    var i int
    N := len(in)

	// Get the total of the series
	c := 0.0
	for i=0;i<N;i++ {
		c += in[i]
	}

	// Create a normalized vector Y
	y := make([]float64, N)

	for i=0;i<N;i++ {
		y[i] = in[i] / c
	}

    // sort Y values in ascending order
	sort.Float64s(y)

	// Normalize X
	x := 1.0 / float64(N)

	// Accumulate Y
	for i=0; i<N-1; i++ {
	    y[i+1] += y[i]
	}

	// Calculate areas below the diagonal (Brown formula)
	gini := 0.0

	for i := 0; i < N-1; i++ {
		gini += (y[i+1] + y[i]) * x
	}

	return 1 - gini
}

// Cover
func Cover(v []float64, lev float64) (float64,float64) {
    var i int
    N := len(v)

    Ad := 0.0
    Au := 0.0

    for i=0;i<N;i++ {
        if v[i]>=lev {
            Au += (v[i]-lev)
            Ad += lev
        } else {
            Ad += v[i]
        }
    }

    n := float64(N)
    return Ad/(lev*n), Au/(lev*n)
}

// Pearson returns the Pearson correlation coeficient of two arrays
func Pearson (a, b [] float64) (float64, error) {

    n := len(a)

    if n!=len(b) {
        return 0,errors.New("lengths differ")
    }

    ma := 0.0
    mb := 0.0

    for i:=0;i<n;i++ {
        ma += a[i]
        mb += b[i]
    }

    ma /= float64(n)
    mb /= float64(n)

    nu := 0.0
    d1 := 0.0
    d2 := 0.0

    var da, db float64

    for i:=0; i<n; i++ {
        da = a[i] - ma
        db = b[i] - mb
        nu += da*db
        d1 += da*da
        d2 += db*db
    }

    return nu / ( math.Sqrt(d1) * math.Sqrt(d2) ), nil
}<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/brotherlogic/godiscogs"
	"golang.org/x/net/context"

	pbd "github.com/brotherlogic/godiscogs"
)

import "github.com/golang/protobuf/proto"

import "strconv"

import "time"

import pb "github.com/brotherlogic/discogssyncer/server"

// GetRelease Gets the release and metadata for the release
func (syncer *Syncer) GetRelease(id int, folder int) (*pbd.Release, *pb.ReleaseMetadata) {
	releaseData, err := ioutil.ReadFile(syncer.saveLocation + "/" + strconv.Itoa(folder) + "/" + strconv.Itoa(id) + ".release")
	if err != nil {
		return nil, nil
	}

	metadataData, _ := ioutil.ReadFile(syncer.saveLocation + "/static-metadata/" + strconv.Itoa(id) + ".metadata")
	release := &pbd.Release{}
	metadata := &pb.ReleaseMetadata{}

	proto.Unmarshal(releaseData, release)
	proto.Unmarshal(metadataData, metadata)

	return release, metadata
}

func (syncer *Syncer) saveMetadata(rel *godiscogs.Release) {
	metadataRoot := syncer.saveLocation + "/static-metadata/"
	metadataPath := metadataRoot + strconv.Itoa(int(rel.Id)) + ".metadata"
	if _, err := os.Stat(metadataRoot); os.IsNotExist(err) {
		os.MkdirAll(metadataRoot, 0777)
	}

	metadata := &pb.ReleaseMetadata{}
	if _, err := os.Stat(metadataPath); os.IsNotExist(err) {
		metadata.DateAdded = time.Now().Unix()
		metadata.DateRefreshed = time.Now().Unix()
	} else {
		data, _ := ioutil.ReadFile(metadataPath)
		proto.Unmarshal(data, metadata)
		metadata.DateRefreshed = time.Now().Unix()
	}
	data, _ := proto.Marshal(metadata)
	ioutil.WriteFile(metadataPath, data, 0644)
}

func (syncer *Syncer) deleteRelease(rel *godiscogs.Release, folder int) {
	os.Remove(syncer.saveLocation + "/" + strconv.Itoa(folder) + "/" + strconv.Itoa(int(rel.Id)) + ".release")
}

func (syncer *Syncer) saveRelease(rel *godiscogs.Release, folder int) {
	//Check that the save folder exists
	savePath := syncer.saveLocation + "/" + strconv.Itoa(folder) + "/"
	if _, err := os.Stat(savePath); os.IsNotExist(err) {
		os.MkdirAll(savePath, 0777)
	}

	data, _ := proto.Marshal(rel)
	ioutil.WriteFile(savePath+strconv.Itoa(int(rel.Id))+".release", data, 0644)
	syncer.saveMetadata(rel)
}

type saver interface {
	GetCollection() []godiscogs.Release
	GetFolders() []godiscogs.Folder
	GetRelease(id int) (godiscogs.Release, error)
	MoveToFolder(folderID int, releaseID int, instanceID int, newFolderID int)
	AddToFolder(folderID int, releaseID int)
	SetRating(folderID int, releaseID int, instanceID int, rating int)
	GetWantlist() ([]pbd.Release, error)
}

// SaveCollection writes out the full collection to files.
func (syncer *Syncer) SaveCollection(retr saver) {
	releases := retr.GetCollection()
	for _, release := range releases {
		fullRelease, _ := retr.GetRelease(int(release.Id))
		fullRelease.InstanceId = release.InstanceId
		fullRelease.FolderId = release.FolderId
		fullRelease.Rating = release.Rating
		syncer.saveRelease(&fullRelease, int(release.FolderId))
	}
	folders := retr.GetFolders()
	folderList := pb.FolderList{}
	for i := range folders {
		folder := folders[i]
		folderList.Folders = append(folderList.Folders, &folder)
	}
	syncer.SaveFolders(&folderList)
}

// SyncWantlist syncs the wantlist with the server
func (syncer *Syncer) SyncWantlist() {
	wants, _ := syncer.retr.GetWantlist()

	for _, want := range wants {
		seen := false
		var val *pb.Want
		for _, swant := range syncer.wants.Want {
			if swant.ReleaseId == want.Id {
				seen = true
				val = swant
			}
		}

		if seen {
			val.Wanted = true
		} else {
			syncer.wants.Want = append(syncer.wants.Want, &pb.Want{ReleaseId: want.Id, Valued: false, Wanted: true})
		}
	}

	// Cache the want list releases
	for _, want := range syncer.wants.Want {
		release, _ := syncer.retr.GetRelease(int(want.ReleaseId))
		syncer.saveRelease(&release, -5)
	}

	syncer.saveWantList()
}

func (syncer *Syncer) getFolders() *pb.FolderList {
	data, _ := ioutil.ReadFile(syncer.saveLocation + "/metadata/folders")
	folderData := &pb.FolderList{}
	proto.Unmarshal(data, folderData)
	return folderData
}

// GetSingleRelease gets a single release
func (syncer *Syncer) GetSingleRelease(ctx context.Context, in *pbd.Release) (*pbd.Release, error) {
	if val, ok := syncer.relMap[in.Id]; ok {
		return val, nil
	}
	return nil, errors.New("Unable to find release")
}

// MoveToFolder moves a release to the specified folder
func (syncer *Syncer) MoveToFolder(ctx context.Context, in *pb.ReleaseMove) (*pb.Empty, error) {
	syncer.retr.MoveToFolder(int(in.Release.FolderId), int(in.Release.Id), int(in.Release.InstanceId), int(in.NewFolderId))
	oldFolder := int(in.Release.FolderId)
	fullRelease, _ := syncer.retr.GetRelease(int(in.Release.Id))
	fullRelease.FolderId = int32(in.NewFolderId)
	syncer.relMap[fullRelease.Id] = &fullRelease

	syncer.Log(fmt.Sprintf("Moving %v from %v to %v", in.Release.Id, in.Release.FolderId, in.NewFolderId))
	syncer.saveRelease(&fullRelease, int(in.NewFolderId))
	syncer.deleteRelease(&fullRelease, oldFolder)
	return &pb.Empty{}, nil
}

// AddToFolder adds a release to the specified folder
func (syncer *Syncer) AddToFolder(ctx context.Context, in *pb.ReleaseMove) (*pb.Empty, error) {
	syncer.retr.AddToFolder(int(in.NewFolderId), int(in.Release.Id))
	fullRelease, _ := syncer.retr.GetRelease(int(in.Release.Id))
	fullRelease.FolderId = int32(in.NewFolderId)
	syncer.saveRelease(&fullRelease, int(in.NewFolderId))
	syncer.relMap[in.Release.Id] = &fullRelease
	return &pb.Empty{}, nil
}

// UpdateRating updates the rating of a release
func (syncer *Syncer) UpdateRating(ctx context.Context, in *pbd.Release) (*pb.Empty, error) {
	syncer.retr.SetRating(int(in.FolderId), int(in.Id), int(in.InstanceId), int(in.Rating))
	fullRelease, _ := syncer.GetRelease(int(in.Id), int(in.FolderId))
	fullRelease.Rating = int32(in.Rating)
	syncer.relMap[in.Id] = fullRelease
	syncer.saveRelease(fullRelease, int(fullRelease.FolderId))
	return &pb.Empty{}, nil
}

// UpdateMetadata updates the metadata of a given record
func (syncer *Syncer) UpdateMetadata(ctx context.Context, in *pb.MetadataUpdate) (*pb.ReleaseMetadata, error) {
	release, metadata := syncer.GetRelease(int(in.Release.Id), int(in.Release.FolderId))
	if release == nil {
		return nil, errors.New("Unable to locate release")
	}
	proto.Merge(metadata, in.Update)

	metadataRoot := syncer.saveLocation + "/static-metadata/"
	metadataPath := metadataRoot + strconv.Itoa(int(in.Release.Id)) + ".metadata"
	data, _ := proto.Marshal(metadata)
	ioutil.WriteFile(metadataPath, data, 0644)

	return metadata, nil
}

// GetWantlist gets the wantlist
func (syncer *Syncer) GetWantlist(ctx context.Context, in *pb.Empty) (*pb.Wantlist, error) {
	return &syncer.wants, nil
}

// GetMetadata gets the metadata for a given release
func (syncer *Syncer) GetMetadata(ctx context.Context, in *pbd.Release) (*pb.ReleaseMetadata, error) {
	_, metadata := syncer.GetRelease(int(in.Id), int(in.FolderId))
	return metadata, nil
}

// GetReleasesInFolder serves up the releases in a given folder
func (syncer *Syncer) GetReleasesInFolder(ctx context.Context, in *pb.FolderList) (*pb.ReleaseList, error) {

	releases := pb.ReleaseList{}
	for _, folderSpec := range in.Folders {
		folders := syncer.getFolders()
		for _, folder := range folders.Folders {
			if folder.Name == folderSpec.Name || folder.Id == folderSpec.Id {
				innerReleases := syncer.getReleases(int(folder.Id))
				releases.Releases = append(releases.Releases, innerReleases.Releases...)
			}
		}
	}

	return &releases, nil
}

func (syncer *Syncer) getReleases(folderID int) *pb.ReleaseList {
	releases := pb.ReleaseList{}
	files, _ := ioutil.ReadDir(syncer.saveLocation + "/" + strconv.Itoa(folderID) + "/")
	for _, file := range files {
		if strings.HasSuffix(file.Name(), ".release") {
			data, _ := ioutil.ReadFile(syncer.saveLocation + "/" + strconv.Itoa(folderID) + "/" + file.Name())
			release := &pbd.Release{}
			proto.Unmarshal(data, release)
			releases.Releases = append(releases.Releases, release)
		}
	}
	return &releases
}

func (syncer *Syncer) initWantlist() {
	wldata, _ := ioutil.ReadFile(syncer.saveLocation + "/metadata/wantlist")
	proto.Unmarshal(wldata, &syncer.wants)
}

func (syncer *Syncer) saveWantList() {
	data, _ := proto.Marshal(&syncer.wants)
	savePath := syncer.saveLocation + "/metadata/"
	if _, err := os.Stat(savePath); os.IsNotExist(err) {
		os.MkdirAll(savePath, 0777)
	}
	ioutil.WriteFile(savePath+"wantlist", data, 0644)
}

// SaveFolders saves out the list of folders
func (syncer *Syncer) SaveFolders(list *pb.FolderList) {
	savePath := syncer.saveLocation + "/metadata/"
	if _, err := os.Stat(savePath); os.IsNotExist(err) {
		os.MkdirAll(savePath, 0777)
	}

	data, _ := proto.Marshal(list)
	ioutil.WriteFile(savePath+"folders", data, 0644)
}

// GetCollection serves up the whole of the collection
func (syncer *Syncer) GetCollection(ctx context.Context, in *pb.Empty) (*pb.ReleaseList, error) {
	releases := pb.ReleaseList{}
	bfiles, _ := ioutil.ReadDir(syncer.saveLocation)
	for _, bfile := range bfiles {
		if bfile.IsDir() {
			folderID, _ := strconv.Atoi(bfile.Name())
			for _, release := range syncer.getReleases(folderID).Releases {
				releases.Releases = append(releases.Releases, release)
			}
		}
	}
	return &releases, nil
}
<commit_msg>Added cache of new wants<commit_after>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/brotherlogic/godiscogs"
	"golang.org/x/net/context"

	pbd "github.com/brotherlogic/godiscogs"
)

import "github.com/golang/protobuf/proto"

import "strconv"

import "time"

import pb "github.com/brotherlogic/discogssyncer/server"

// GetRelease Gets the release and metadata for the release
func (syncer *Syncer) GetRelease(id int, folder int) (*pbd.Release, *pb.ReleaseMetadata) {
	releaseData, err := ioutil.ReadFile(syncer.saveLocation + "/" + strconv.Itoa(folder) + "/" + strconv.Itoa(id) + ".release")
	if err != nil {
		return nil, nil
	}

	metadataData, _ := ioutil.ReadFile(syncer.saveLocation + "/static-metadata/" + strconv.Itoa(id) + ".metadata")
	release := &pbd.Release{}
	metadata := &pb.ReleaseMetadata{}

	proto.Unmarshal(releaseData, release)
	proto.Unmarshal(metadataData, metadata)

	return release, metadata
}

func (syncer *Syncer) saveMetadata(rel *godiscogs.Release) {
	metadataRoot := syncer.saveLocation + "/static-metadata/"
	metadataPath := metadataRoot + strconv.Itoa(int(rel.Id)) + ".metadata"
	if _, err := os.Stat(metadataRoot); os.IsNotExist(err) {
		os.MkdirAll(metadataRoot, 0777)
	}

	metadata := &pb.ReleaseMetadata{}
	if _, err := os.Stat(metadataPath); os.IsNotExist(err) {
		metadata.DateAdded = time.Now().Unix()
		metadata.DateRefreshed = time.Now().Unix()
	} else {
		data, _ := ioutil.ReadFile(metadataPath)
		proto.Unmarshal(data, metadata)
		metadata.DateRefreshed = time.Now().Unix()
	}
	data, _ := proto.Marshal(metadata)
	ioutil.WriteFile(metadataPath, data, 0644)
}

func (syncer *Syncer) deleteRelease(rel *godiscogs.Release, folder int) {
	os.Remove(syncer.saveLocation + "/" + strconv.Itoa(folder) + "/" + strconv.Itoa(int(rel.Id)) + ".release")
}

func (syncer *Syncer) saveRelease(rel *godiscogs.Release, folder int) {
	//Check that the save folder exists
	savePath := syncer.saveLocation + "/" + strconv.Itoa(folder) + "/"
	if _, err := os.Stat(savePath); os.IsNotExist(err) {
		os.MkdirAll(savePath, 0777)
	}

	data, _ := proto.Marshal(rel)
	ioutil.WriteFile(savePath+strconv.Itoa(int(rel.Id))+".release", data, 0644)
	syncer.saveMetadata(rel)
}

type saver interface {
	GetCollection() []godiscogs.Release
	GetFolders() []godiscogs.Folder
	GetRelease(id int) (godiscogs.Release, error)
	MoveToFolder(folderID int, releaseID int, instanceID int, newFolderID int)
	AddToFolder(folderID int, releaseID int)
	SetRating(folderID int, releaseID int, instanceID int, rating int)
	GetWantlist() ([]pbd.Release, error)
}

// SaveCollection writes out the full collection to files.
func (syncer *Syncer) SaveCollection(retr saver) {
	releases := retr.GetCollection()
	for _, release := range releases {
		fullRelease, _ := retr.GetRelease(int(release.Id))
		fullRelease.InstanceId = release.InstanceId
		fullRelease.FolderId = release.FolderId
		fullRelease.Rating = release.Rating
		syncer.saveRelease(&fullRelease, int(release.FolderId))
	}
	folders := retr.GetFolders()
	folderList := pb.FolderList{}
	for i := range folders {
		folder := folders[i]
		folderList.Folders = append(folderList.Folders, &folder)
	}
	syncer.SaveFolders(&folderList)
}

// SyncWantlist syncs the wantlist with the server
func (syncer *Syncer) SyncWantlist() {
	wants, _ := syncer.retr.GetWantlist()

	for _, want := range wants {
		seen := false
		var val *pb.Want
		for _, swant := range syncer.wants.Want {
			if swant.ReleaseId == want.Id {
				seen = true
				val = swant
			}
		}

		if seen {
			val.Wanted = true
		} else {
			syncer.wants.Want = append(syncer.wants.Want, &pb.Want{ReleaseId: want.Id, Valued: false, Wanted: true})
		}
	}

	// Cache the want list releases
	for _, want := range syncer.wants.Want {
		release, _ := syncer.retr.GetRelease(int(want.ReleaseId))
		syncer.saveRelease(&release, -5)
	}

	syncer.saveWantList()
}

func (syncer *Syncer) getFolders() *pb.FolderList {
	data, _ := ioutil.ReadFile(syncer.saveLocation + "/metadata/folders")
	folderData := &pb.FolderList{}
	proto.Unmarshal(data, folderData)
	return folderData
}

// GetSingleRelease gets a single release
func (syncer *Syncer) GetSingleRelease(ctx context.Context, in *pbd.Release) (*pbd.Release, error) {
	if val, ok := syncer.relMap[in.Id]; ok {
		return val, nil
	}
	return nil, errors.New("Unable to find release")
}

// MoveToFolder moves a release to the specified folder
func (syncer *Syncer) MoveToFolder(ctx context.Context, in *pb.ReleaseMove) (*pb.Empty, error) {
	syncer.retr.MoveToFolder(int(in.Release.FolderId), int(in.Release.Id), int(in.Release.InstanceId), int(in.NewFolderId))
	oldFolder := int(in.Release.FolderId)
	fullRelease, _ := syncer.retr.GetRelease(int(in.Release.Id))
	fullRelease.FolderId = int32(in.NewFolderId)
	syncer.relMap[fullRelease.Id] = &fullRelease

	syncer.Log(fmt.Sprintf("Moving %v from %v to %v", in.Release.Id, in.Release.FolderId, in.NewFolderId))
	syncer.saveRelease(&fullRelease, int(in.NewFolderId))
	syncer.deleteRelease(&fullRelease, oldFolder)
	return &pb.Empty{}, nil
}

// AddToFolder adds a release to the specified folder
func (syncer *Syncer) AddToFolder(ctx context.Context, in *pb.ReleaseMove) (*pb.Empty, error) {
	syncer.retr.AddToFolder(int(in.NewFolderId), int(in.Release.Id))
	fullRelease, _ := syncer.retr.GetRelease(int(in.Release.Id))
	fullRelease.FolderId = int32(in.NewFolderId)
	syncer.saveRelease(&fullRelease, int(in.NewFolderId))
	syncer.relMap[in.Release.Id] = &fullRelease
	return &pb.Empty{}, nil
}

// UpdateRating updates the rating of a release
func (syncer *Syncer) UpdateRating(ctx context.Context, in *pbd.Release) (*pb.Empty, error) {
	syncer.retr.SetRating(int(in.FolderId), int(in.Id), int(in.InstanceId), int(in.Rating))
	fullRelease, _ := syncer.GetRelease(int(in.Id), int(in.FolderId))
	fullRelease.Rating = int32(in.Rating)
	syncer.relMap[in.Id] = fullRelease
	syncer.saveRelease(fullRelease, int(fullRelease.FolderId))
	return &pb.Empty{}, nil
}

// UpdateMetadata updates the metadata of a given record
func (syncer *Syncer) UpdateMetadata(ctx context.Context, in *pb.MetadataUpdate) (*pb.ReleaseMetadata, error) {
	release, metadata := syncer.GetRelease(int(in.Release.Id), int(in.Release.FolderId))
	if release == nil {
		return nil, errors.New("Unable to locate release")
	}
	proto.Merge(metadata, in.Update)

	metadataRoot := syncer.saveLocation + "/static-metadata/"
	metadataPath := metadataRoot + strconv.Itoa(int(in.Release.Id)) + ".metadata"
	data, _ := proto.Marshal(metadata)
	ioutil.WriteFile(metadataPath, data, 0644)

	return metadata, nil
}

// GetWantlist gets the wantlist
func (syncer *Syncer) GetWantlist(ctx context.Context, in *pb.Empty) (*pb.Wantlist, error) {
	return &syncer.wants, nil
}

// GetMetadata gets the metadata for a given release
func (syncer *Syncer) GetMetadata(ctx context.Context, in *pbd.Release) (*pb.ReleaseMetadata, error) {
	_, metadata := syncer.GetRelease(int(in.Id), int(in.FolderId))
	return metadata, nil
}

// GetReleasesInFolder serves up the releases in a given folder
func (syncer *Syncer) GetReleasesInFolder(ctx context.Context, in *pb.FolderList) (*pb.ReleaseList, error) {

	releases := pb.ReleaseList{}
	for _, folderSpec := range in.Folders {
		folders := syncer.getFolders()
		for _, folder := range folders.Folders {
			if folder.Name == folderSpec.Name || folder.Id == folderSpec.Id {
				innerReleases := syncer.getReleases(int(folder.Id))
				releases.Releases = append(releases.Releases, innerReleases.Releases...)
			}
		}
	}

	return &releases, nil
}

func (syncer *Syncer) getReleases(folderID int) *pb.ReleaseList {
	releases := pb.ReleaseList{}
	files, _ := ioutil.ReadDir(syncer.saveLocation + "/" + strconv.Itoa(folderID) + "/")
	for _, file := range files {
		if strings.HasSuffix(file.Name(), ".release") {
			data, _ := ioutil.ReadFile(syncer.saveLocation + "/" + strconv.Itoa(folderID) + "/" + file.Name())
			release := &pbd.Release{}
			proto.Unmarshal(data, release)
			releases.Releases = append(releases.Releases, release)
		}
	}
	return &releases
}

func (syncer *Syncer) initWantlist() {
	wldata, _ := ioutil.ReadFile(syncer.saveLocation + "/metadata/wantlist")
	proto.Unmarshal(wldata, &syncer.wants)

	for _, want := range syncer.wants.Want {
		rel, _ := syncer.GetRelease(int(want.ReleaseId), -5)
		syncer.relMap[rel.Id] = rel
	}
}

func (syncer *Syncer) saveWantList() {
	data, _ := proto.Marshal(&syncer.wants)
	savePath := syncer.saveLocation + "/metadata/"
	if _, err := os.Stat(savePath); os.IsNotExist(err) {
		os.MkdirAll(savePath, 0777)
	}
	ioutil.WriteFile(savePath+"wantlist", data, 0644)
}

// SaveFolders saves out the list of folders
func (syncer *Syncer) SaveFolders(list *pb.FolderList) {
	savePath := syncer.saveLocation + "/metadata/"
	if _, err := os.Stat(savePath); os.IsNotExist(err) {
		os.MkdirAll(savePath, 0777)
	}

	data, _ := proto.Marshal(list)
	ioutil.WriteFile(savePath+"folders", data, 0644)
}

// GetCollection serves up the whole of the collection
func (syncer *Syncer) GetCollection(ctx context.Context, in *pb.Empty) (*pb.ReleaseList, error) {
	releases := pb.ReleaseList{}
	bfiles, _ := ioutil.ReadDir(syncer.saveLocation)
	for _, bfile := range bfiles {
		if bfile.IsDir() {
			folderID, _ := strconv.Atoi(bfile.Name())
			for _, release := range syncer.getReleases(folderID).Releases {
				releases.Releases = append(releases.Releases, release)
			}
		}
	}
	return &releases, nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"time"
)

type ReadFn func() (reader io.ReadCloser, err error)
type File struct {
	Url      url.URL
	Mtime    time.Time
	FileFunc ReadFn
}

func (f File) ReadAll() (content []byte, err error) {
	reader, err := f.FileFunc()
	if err != nil {
		return
	}
	return ioutil.ReadAll(reader)
}

func Sync(from, to string, lookup LookupFn) (chan File, chan error) {
	files := make(chan File)
	errs := make(chan error)

	fromUri, err := url.Parse(from)
	if err != nil {
		errs <- err
		return nil, errs
	}

	go func() {
		todos := []File{File{Url: *fromUri}}
		for i := 0; i < len(todos); i++ {
			todo := todos[i]

			indexFn, err := lookup(todo)
			errs <- err
			if indexFn == nil {
				errs <- errors.New("Not Supported: " + todo.Url.String())
				continue
			}

			hfiles := make(chan File)
			finish := make(chan bool)
			go func(f chan bool) {
				indexFn(todo, hfiles, errs)
				f <- true
			}(finish)

		LOOP:
			for {
				select {
				case <-finish:
					break LOOP
				case f := <-hfiles:
					if f.FileFunc == nil {
						todos = append(todos, f)
					} else {
						f.Url.Path = filepath.Join(to, f.Url.Path)
						err = Local(f)
						if err != nil {
							errs <- err
						} else {
							files <- f
						}
					}
				}
			}
		}
		close(files)
		close(errs)
	}()
	return files, errs
}

func removeNanoseconds(in time.Time) time.Time {
	return time.Date(
		in.Year(),
		in.Month(),
		in.Day(),
		in.Hour(),
		in.Minute(),
		in.Second(),
		0, in.Location())
}

func Local(file File) (err error) {
	st, err := os.Stat(file.Url.Path)
	if err != nil && !os.IsNotExist(err) {
		return
	}
	if st == nil || file.Mtime.After(st.ModTime()) {
		err = os.MkdirAll(filepath.Dir(file.Url.Path), os.ModeDir|os.ModePerm)
		if err != nil {
			return err
		}
		osfile, err := os.Create(file.Url.Path)
		if err != nil {
			return err
		}
		defer osfile.Close()
		r, err := file.FileFunc()
		if err != nil {
			return err
		}
		_, err = io.Copy(osfile, r)
		if err != nil {
			return err
		}
		os.Chtimes(file.Url.Path, file.Mtime, file.Mtime)

		osfile.Sync()
	}
	return
}
<commit_msg>skip nil error values<commit_after>package main

import (
	"errors"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"time"
)

type ReadFn func() (reader io.ReadCloser, err error)
type File struct {
	Url      url.URL
	Mtime    time.Time
	FileFunc ReadFn
}

func (f File) ReadAll() (content []byte, err error) {
	reader, err := f.FileFunc()
	if err != nil {
		return
	}
	return ioutil.ReadAll(reader)
}

func Sync(from, to string, lookup LookupFn) (chan File, chan error) {
	files := make(chan File)
	errs := make(chan error)

	fromUri, err := url.Parse(from)
	if err != nil {
		errs <- err
		return nil, errs
	}

	go func() {
		todos := []File{File{Url: *fromUri}}
		for i := 0; i < len(todos); i++ {
			todo := todos[i]

			indexFn, err := lookup(todo)
			errs <- err
			if indexFn == nil {
				errs <- errors.New("Not Supported: " + todo.Url.String())
				continue
			}

			hfiles := make(chan File)
			finish := make(chan bool)
			go func(f chan bool) {
				indexFn(todo, hfiles, errs)
				f <- true
			}(finish)

		LOOP:
			for {
				select {
				case <-finish:
					break LOOP
				case f := <-hfiles:
					if f.FileFunc == nil {
						todos = append(todos, f)
					} else {
						f.Url.Path = filepath.Join(to, f.Url.Path)
						err = Local(f)
						if err != nil {
							errs <- err
						} else {
							files <- f
						}
					}
				}
			}
		}
		close(files)
		close(errs)
	}()
	return files, skipNil(errs)
}

func skipNil(in chan error) chan error {
	out := make(chan error)
	go func() {
		for e := range in {
			if e != nil {
				out <- e
			}
		}
		close(out)
	}()
	return out
}

func removeNanoseconds(in time.Time) time.Time {
	return time.Date(
		in.Year(),
		in.Month(),
		in.Day(),
		in.Hour(),
		in.Minute(),
		in.Second(),
		0, in.Location())
}

func Local(file File) (err error) {
	st, err := os.Stat(file.Url.Path)
	if err != nil && !os.IsNotExist(err) {
		return
	}
	if st == nil || file.Mtime.After(st.ModTime()) {
		err = os.MkdirAll(filepath.Dir(file.Url.Path), os.ModeDir|os.ModePerm)
		if err != nil {
			return err
		}
		osfile, err := os.Create(file.Url.Path)
		if err != nil {
			return err
		}
		defer osfile.Close()
		r, err := file.FileFunc()
		if err != nil {
			return err
		}
		_, err = io.Copy(osfile, r)
		if err != nil {
			return err
		}
		os.Chtimes(file.Url.Path, file.Mtime, file.Mtime)

		osfile.Sync()
	}
	return
}
<|endoftext|>


<commit_before>package tail

import (
	"bufio"
	"bytes"
	"context"
	"errors"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"

	fsnotify "github.com/fsnotify/fsnotify"
)

const (
	openRetryInterval = time.Second
	tailOldFileDelay  = 15 * time.Second
	linesCapacity     = 1024
	errorsCapacity    = 16
)

// Line is a line of the target file.
type Line struct {
	Text string
	Time time.Time
}

// Tail tails a file.
type Tail struct {
	Lines  <-chan *Line
	Errors <-chan error

	opt      Options
	lines    chan<- *Line
	errors   chan<- error
	filename string
	wg       sync.WaitGroup
	ctx      context.Context
	cancel   context.CancelFunc
}

// Options is options for Tail
type Options struct {
	// MaxBytesLine is maximum length of lines in bytes.
	// If it is zero, there is no limit.
	MaxBytesLine int64
}

type tail struct {
	parent *Tail

	file    *os.File
	reader  *bufio.Reader
	watcher *fsnotify.Watcher
	buf     bytes.Buffer
	ctx     context.Context
	cancel  context.CancelFunc
}

// NewTailFile starts tailing a file with opt options.
func NewTailFileWithOptions(filename string, opts Options) (*Tail, error) {
	filename, err := filepath.Abs(filename)
	if err != nil {
		return nil, err
	}

	ctx, cancel := context.WithCancel(context.Background())
	lines := make(chan *Line, linesCapacity)
	errs := make(chan error, errorsCapacity)
	parent := &Tail{
		Lines:    lines,
		Errors:   errs,
		lines:    lines,
		errors:   errs,
		filename: filename,
		ctx:      ctx,
		cancel:   cancel,
	}

	parent.wg.Add(1)
	go func() {
		defer parent.wg.Done()
		parent.runFile(os.SEEK_END)
	}()
	go parent.wait()

	return parent, nil
}

// NewTailFile starts tailing a file with the default configuration.
func NewTailFile(filename string) (*Tail, error) {
	return NewTailFileWithOptions(filename, Options{})
}

// NewTailReader starts tailing io.Reader
func NewTailReaderWithOptions(reader io.Reader, opts Options) (*Tail, error) {
	ctx, cancel := context.WithCancel(context.Background())
	lines := make(chan *Line, linesCapacity)
	errs := make(chan error, errorsCapacity)
	parent := &Tail{
		Lines:  lines,
		Errors: errs,
		lines:  lines,
		errors: errs,
		ctx:    ctx,
		cancel: cancel,
	}
	r := ctxReader{
		ctx: ctx,
		r:   reader,
	}
	t := &tail{
		parent: parent,
		reader: bufio.NewReader(r),
		ctx:    ctx,
		cancel: cancel,
	}

	parent.wg.Add(1)
	go func() {
		defer parent.wg.Done()
		t.runReader()
	}()
	go parent.wait()

	return parent, nil
}

// NewTailReader starts tailing io.Reader with the default configuration.
func NewTailReader(reader io.Reader) (*Tail, error) {
	return NewTailReaderWithOptions(reader, Options{})
}

// Close stops tailing the file.
func (t *Tail) Close() error {
	t.cancel()
	t.wg.Wait()
	return nil
}

func (t *Tail) wait() {
	t.wg.Wait()
	close(t.errors)
	close(t.lines)
}

// open opens the target file.
// If it does not exist, wait for creating new file.
func (t *Tail) open(seek int) (*tail, error) {
	const defaultBufSize = 4096
	bufSize := defaultBufSize
	if t.opt.MaxBytesLine != 0 && int64(bufSize) > t.opt.MaxBytesLine {
		bufSize = int(t.opt.MaxBytesLine)
	}

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}
	for {
		file, err := os.Open(t.filename)
		if err == nil {
			// success, seek and watch the file.
			if _, err := file.Seek(0, seek); err != nil {
				file.Close()
				watcher.Close()
				return nil, err
			}
			if err := watcher.Add(t.filename); err != nil {
				file.Close()
				watcher.Close()
				return nil, err
			}
			ctx, cancel := context.WithCancel(t.ctx)
			r := ctxReader{
				ctx: ctx,
				r:   file,
			}

			return &tail{
				parent:  t,
				file:    file,
				reader:  bufio.NewReaderSize(r, bufSize),
				watcher: watcher,
				ctx:     ctx,
				cancel:  cancel,
			}, nil
		}

		// fail. retry...
		seek = io.SeekStart
		timer := time.NewTimer(openRetryInterval)
		select {
		case <-t.ctx.Done():
			timer.Stop()
			return nil, t.ctx.Err()
		case <-timer.C:
		}
	}
}

// runFile tails target files
func (t *Tail) runFile(seek int) {
	child, err := t.open(seek)
	if err != nil {
		if !errors.Is(err, context.Canceled) {
			t.errors <- err
		}
		return
	}

	t.wg.Add(1)
	go func() {
		defer t.wg.Done()
		child.runFile()
	}()
}

// runFile tails a file
func (t *tail) runFile() {
	defer t.watcher.Close()
	defer t.cancel()

	cherr := make(chan error, 1)
	ch := make(chan struct{}, 1)
	defer close(ch)

	t.parent.wg.Add(1)
	go func() {
		defer t.parent.wg.Done()
		for {
			if err := t.restrict(); err != nil {
				select {
				case cherr <- err:
				case <-t.ctx.Done():
				}
				return
			}
			err := t.tail()
			if err == nil {
				continue
			}

			// wait for writing new lines.
			select {
			case cherr <- err:
			case <-t.ctx.Done():
				return
			}
			select {
			case _, ok := <-ch:
				if !ok {
					return
				}
			case <-t.ctx.Done():
				return
			}
		}
	}()

	var renamed bool
	var waiting bool // waiting for writing new lines?
	for {
		select {
		case event := <-t.watcher.Events:
			if event.Op.Has(fsnotify.Remove) {
				// the target file is removed, stop tailing.
				return
			}
			if event.Op.Has(fsnotify.Rename) {
				// log rotation is detected.
				if !renamed {
					// start to watch creating new file.
					t.parent.wg.Add(1)
					go func() {
						defer t.parent.wg.Done()
						t.parent.runFile(io.SeekStart)
					}()

					// wait a little, and stop tailing old file.
					go func() {
						timer := time.NewTimer(tailOldFileDelay)
						defer timer.Stop()
						select {
						case <-timer.C:
							t.cancel()
						case <-t.ctx.Done():
						}
					}()
				}
				name, err := getFileName(t.file)
				if err != nil {
					t.parent.errors <- err
					return
				}
				if err := t.watcher.Add(name); err != nil {
					t.parent.errors <- err
					return
				}
				renamed = true
			}

			// notify new lines are wrote.
			if waiting {
				ch <- struct{}{}
				waiting = false
			}
		case err := <-cherr:
			if errors.Is(err, io.EOF) {
				waiting = true
				continue
			}
			t.parent.errors <- err
			return
		case err := <-t.watcher.Errors:
			t.parent.errors <- err
			return
		case <-t.ctx.Done():
			return
		}
	}
}

// runReader tails io.Reader
func (t *tail) runReader() {
	defer t.cancel()
	err := t.tail()
	if errors.Is(err, io.EOF) || errors.Is(err, io.ErrClosedPipe) {
		return
	}
	if err != nil {
		if !errors.Is(err, context.Canceled) {
			t.parent.errors <- err
		}
		return
	}
}

// restrict detects a file that is truncated
func (t *tail) restrict() error {
	stat, err := t.file.Stat()
	if err != nil {
		return err
	}
	pos, err := t.file.Seek(0, io.SeekCurrent)
	if err != nil {
		return err
	}
	if stat.Size() < pos {
		// file is truncated. seek to head of file.
		_, err := t.file.Seek(0, io.SeekStart)
		if err != nil {
			return err
		}
	}
	return nil
}

// tail reads lines until EOF
func (t *tail) tail() error {
	opts := t.parent.opt
	for {
		line, err := t.reader.ReadSlice('\n')
		t.buf.Write(line)
		if err == bufio.ErrBufferFull {
			// the reader cannot find EOL in its buffer.
			// continue to read a line.
			if opts.MaxBytesLine == 0 || int64(t.buf.Len()) < opts.MaxBytesLine {
				continue
			}
		}
		if err != nil {
			return err
		}
		t.parent.lines <- &Line{t.buf.String(), time.Now()}
		t.buf.Reset()
	}
}
<commit_msg>fix opts is ignored<commit_after>package tail

import (
	"bufio"
	"bytes"
	"context"
	"errors"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"

	fsnotify "github.com/fsnotify/fsnotify"
)

const (
	openRetryInterval = time.Second
	tailOldFileDelay  = 15 * time.Second
	linesCapacity     = 1024
	errorsCapacity    = 16
)

// Line is a line of the target file.
type Line struct {
	Text string
	Time time.Time
}

// Tail tails a file.
type Tail struct {
	Lines  <-chan *Line
	Errors <-chan error

	opts     Options
	lines    chan<- *Line
	errors   chan<- error
	filename string
	wg       sync.WaitGroup
	ctx      context.Context
	cancel   context.CancelFunc
}

// Options is options for Tail
type Options struct {
	// MaxBytesLine is maximum length of lines in bytes.
	// If it is zero, there is no limit.
	MaxBytesLine int64
}

type tail struct {
	parent *Tail

	file    *os.File
	reader  *bufio.Reader
	watcher *fsnotify.Watcher
	buf     bytes.Buffer
	ctx     context.Context
	cancel  context.CancelFunc
}

// NewTailFile starts tailing a file with opt options.
func NewTailFileWithOptions(filename string, opts Options) (*Tail, error) {
	filename, err := filepath.Abs(filename)
	if err != nil {
		return nil, err
	}

	ctx, cancel := context.WithCancel(context.Background())
	lines := make(chan *Line, linesCapacity)
	errs := make(chan error, errorsCapacity)
	parent := &Tail{
		Lines:    lines,
		Errors:   errs,
		opts:     opts,
		lines:    lines,
		errors:   errs,
		filename: filename,
		ctx:      ctx,
		cancel:   cancel,
	}

	parent.wg.Add(1)
	go func() {
		defer parent.wg.Done()
		parent.runFile(os.SEEK_END)
	}()
	go parent.wait()

	return parent, nil
}

// NewTailFile starts tailing a file with the default configuration.
func NewTailFile(filename string) (*Tail, error) {
	return NewTailFileWithOptions(filename, Options{})
}

// NewTailReader starts tailing io.Reader
func NewTailReaderWithOptions(reader io.Reader, opts Options) (*Tail, error) {
	ctx, cancel := context.WithCancel(context.Background())
	lines := make(chan *Line, linesCapacity)
	errs := make(chan error, errorsCapacity)
	parent := &Tail{
		Lines:  lines,
		Errors: errs,
		opts:   opts,
		lines:  lines,
		errors: errs,
		ctx:    ctx,
		cancel: cancel,
	}
	r := ctxReader{
		ctx: ctx,
		r:   reader,
	}
	t := &tail{
		parent: parent,
		reader: bufio.NewReader(r),
		ctx:    ctx,
		cancel: cancel,
	}

	parent.wg.Add(1)
	go func() {
		defer parent.wg.Done()
		t.runReader()
	}()
	go parent.wait()

	return parent, nil
}

// NewTailReader starts tailing io.Reader with the default configuration.
func NewTailReader(reader io.Reader) (*Tail, error) {
	return NewTailReaderWithOptions(reader, Options{})
}

// Close stops tailing the file.
func (t *Tail) Close() error {
	t.cancel()
	t.wg.Wait()
	return nil
}

func (t *Tail) wait() {
	t.wg.Wait()
	close(t.errors)
	close(t.lines)
}

// open opens the target file.
// If it does not exist, wait for creating new file.
func (t *Tail) open(seek int) (*tail, error) {
	const defaultBufSize = 4096
	bufSize := defaultBufSize
	if t.opts.MaxBytesLine != 0 && int64(bufSize) > t.opts.MaxBytesLine {
		bufSize = int(t.opts.MaxBytesLine)
	}

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}
	for {
		file, err := os.Open(t.filename)
		if err == nil {
			// success, seek and watch the file.
			if _, err := file.Seek(0, seek); err != nil {
				file.Close()
				watcher.Close()
				return nil, err
			}
			if err := watcher.Add(t.filename); err != nil {
				file.Close()
				watcher.Close()
				return nil, err
			}
			ctx, cancel := context.WithCancel(t.ctx)
			r := ctxReader{
				ctx: ctx,
				r:   file,
			}

			return &tail{
				parent:  t,
				file:    file,
				reader:  bufio.NewReaderSize(r, bufSize),
				watcher: watcher,
				ctx:     ctx,
				cancel:  cancel,
			}, nil
		}

		// fail. retry...
		seek = io.SeekStart
		timer := time.NewTimer(openRetryInterval)
		select {
		case <-t.ctx.Done():
			timer.Stop()
			return nil, t.ctx.Err()
		case <-timer.C:
		}
	}
}

// runFile tails target files
func (t *Tail) runFile(seek int) {
	child, err := t.open(seek)
	if err != nil {
		if !errors.Is(err, context.Canceled) {
			t.errors <- err
		}
		return
	}

	t.wg.Add(1)
	go func() {
		defer t.wg.Done()
		child.runFile()
	}()
}

// runFile tails a file
func (t *tail) runFile() {
	defer t.watcher.Close()
	defer t.cancel()

	cherr := make(chan error, 1)
	ch := make(chan struct{}, 1)
	defer close(ch)

	t.parent.wg.Add(1)
	go func() {
		defer t.parent.wg.Done()
		for {
			if err := t.restrict(); err != nil {
				select {
				case cherr <- err:
				case <-t.ctx.Done():
				}
				return
			}
			err := t.tail()
			if err == nil {
				continue
			}

			// wait for writing new lines.
			select {
			case cherr <- err:
			case <-t.ctx.Done():
				return
			}
			select {
			case _, ok := <-ch:
				if !ok {
					return
				}
			case <-t.ctx.Done():
				return
			}
		}
	}()

	var renamed bool
	var waiting bool // waiting for writing new lines?
	for {
		select {
		case event := <-t.watcher.Events:
			if event.Op.Has(fsnotify.Remove) {
				// the target file is removed, stop tailing.
				return
			}
			if event.Op.Has(fsnotify.Rename) {
				// log rotation is detected.
				if !renamed {
					// start to watch creating new file.
					t.parent.wg.Add(1)
					go func() {
						defer t.parent.wg.Done()
						t.parent.runFile(io.SeekStart)
					}()

					// wait a little, and stop tailing old file.
					go func() {
						timer := time.NewTimer(tailOldFileDelay)
						defer timer.Stop()
						select {
						case <-timer.C:
							t.cancel()
						case <-t.ctx.Done():
						}
					}()
				}
				name, err := getFileName(t.file)
				if err != nil {
					t.parent.errors <- err
					return
				}
				if err := t.watcher.Add(name); err != nil {
					t.parent.errors <- err
					return
				}
				renamed = true
			}

			// notify new lines are wrote.
			if waiting {
				ch <- struct{}{}
				waiting = false
			}
		case err := <-cherr:
			if errors.Is(err, io.EOF) {
				waiting = true
				continue
			}
			t.parent.errors <- err
			return
		case err := <-t.watcher.Errors:
			t.parent.errors <- err
			return
		case <-t.ctx.Done():
			return
		}
	}
}

// runReader tails io.Reader
func (t *tail) runReader() {
	defer t.cancel()
	err := t.tail()
	if errors.Is(err, io.EOF) || errors.Is(err, io.ErrClosedPipe) {
		return
	}
	if err != nil {
		if !errors.Is(err, context.Canceled) {
			t.parent.errors <- err
		}
		return
	}
}

// restrict detects a file that is truncated
func (t *tail) restrict() error {
	stat, err := t.file.Stat()
	if err != nil {
		return err
	}
	pos, err := t.file.Seek(0, io.SeekCurrent)
	if err != nil {
		return err
	}
	if stat.Size() < pos {
		// file is truncated. seek to head of file.
		_, err := t.file.Seek(0, io.SeekStart)
		if err != nil {
			return err
		}
	}
	return nil
}

// tail reads lines until EOF
func (t *tail) tail() error {
	opts := t.parent.opts
	for {
		line, err := t.reader.ReadSlice('\n')
		t.buf.Write(line)
		if err == bufio.ErrBufferFull {
			// the reader cannot find EOL in its buffer.
			// continue to read a line.
			if opts.MaxBytesLine == 0 || int64(t.buf.Len()) < opts.MaxBytesLine {
				continue
			}
		}
		if err != nil {
			return err
		}
		t.parent.lines <- &Line{t.buf.String(), time.Now()}
		t.buf.Reset()
	}
}
<|endoftext|>


<commit_before>package main

// All actions under task command

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/url"
	"strconv"
)

type TaskList struct {
	client *Client
	format Formatter
}

func (t TaskList) Apply(args []string) {
	switch len(args) {
	case 0:
		t.listAll()
	case 1:
		t.listById(args[0])
	default:
		Check(false, "too many arguments")
	}
}

func (t TaskList) listAll() {
	path := "/v2/tasks"
	request := t.client.GET(path)
	response, e := t.client.Do(request)
	Check(e == nil, "failed to get response", e)
	defer response.Body.Close()
	fmt.Println(t.format.Format(response.Body, t.HumanizeAll))
}

func (t TaskList) HumanizeAll(body io.Reader) string {
	dec := json.NewDecoder(body)
	var tasks Tasks
	e := dec.Decode(&tasks)
	Check(e == nil, "failed to unmarshal response", e)
	var b bytes.Buffer
	for _, task := range tasks.Tasks {
		b.WriteString(task.AppID)
		b.WriteString(" ")
		b.WriteString(task.Host)
		b.WriteString(" ")
		b.WriteString(task.Version)
		b.WriteString(" ")
		b.WriteString(task.ID)
		b.WriteString("\n")
	}
	title := "APPID HOST VERSION TASKID\n"
	text := title + b.String()
	return Columnize(text)
}

func (t TaskList) listById(id string) {
	esc := url.QueryEscape(id)
	path := "/v2/apps/" + esc + "?embed=apps.tasks"
	request := t.client.GET(path)
	response, e := t.client.Do(request)
	Check(e == nil, "failed to get response", e)
	defer response.Body.Close()
	fmt.Println(t.format.Format(response.Body, t.HumanizeById))
}

func (t TaskList) HumanizeById(body io.Reader) string {
	dec := json.NewDecoder(body)
	var appbyid AppById
	e := dec.Decode(&appbyid)
	Check(e == nil, "failed to unmarshal response", e)

	var b bytes.Buffer
	for _, task := range appbyid.App.Tasks {
		b.WriteString(task.ID)
		b.WriteString(" ")
		b.WriteString(task.Host)
		b.WriteString(" ")
		b.WriteString(task.Version)
		// ports?
	}
	title := "ID HOST VERSION\n"
	text := title + b.String()
	return Columnize(text)
}

type TaskKill struct {
	client *Client
	format Formatter
}

func (t TaskKill) Apply(args []string) {
	switch len(args) {
	case 1:
		t.killAll(args[0])
	case 2:
		t.killOnly(args[0], args[1])
	default:
		Check(false, "task kill takes 1 or 2 arguments")
	}
}

func (t TaskKill) killAll(id string) {
	esc := url.QueryEscape(id)
	path := "/v2/apps/" + esc + "/tasks"
	request := t.client.DELETE(path)
	response, e := t.client.Do(request)
	Check(e == nil, "failed to get response", e)
	defer response.Body.Close()

	sc := response.StatusCode
	Check(sc != 404, "unknown id")
	Check(sc == 200, "failed with status code", sc)
	t.format.Format(response.Body, t.Humanize)
}

func (t TaskKill) killOnly(id, taskid string) {
	escID := url.QueryEscape(id)
	escTaskID := url.QueryEscape(taskid)
	path := "/v2/apps/" + escID + "/tasks/" + escTaskID
	request := t.client.DELETE(path)
	response, e := t.client.Do(request)
	Check(e == nil, "failed to get response", e)
	defer response.Body.Close()
	sc := response.StatusCode
	Check(sc != 404, "unknown appid or taskid")
	Check(sc == 200, "failed with status code", sc)
	t.format.Format(response.Body, t.Humanize)
}

func (t TaskKill) Humanize(body io.Reader) string {
	// todo does this actually return a list of killed tasks?
	return "success"
}

type TaskQueue struct {
	client *Client
	format Formatter
}

func (t TaskQueue) Apply(args []string) {
	Check(len(args) == 0, "no arguments")
	request := t.client.GET("/v2/queue")
	response, e := t.client.Do(request)
	Check(e == nil, "failed to get response", e)
	defer response.Body.Close()
	fmt.Println(t.format.Format(response.Body, t.Humanize))
}

func (t TaskQueue) Humanize(body io.Reader) string {
	dec := json.NewDecoder(body)
	var queue Queue
	e := dec.Decode(&queue)
	Check(e == nil, "failed to decode response", e)
	title := "APP VERSION OVERDUE\n"
	var b bytes.Buffer
	for _, queuedTask := range queue.Queue {
		b.WriteString(queuedTask.App.ID)
		b.WriteString(" ")
		b.WriteString(queuedTask.App.Version)
		b.WriteString(" ")
		b.WriteString(strconv.FormatBool(queuedTask.Delay["overdue"]))
		b.WriteString("\n")
	}
	text := title + b.String()
	return Columnize(text)
}
<commit_msg>Fixed formatting for task list<commit_after>package main

// All actions under task command

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/url"
	"strconv"
)

type TaskList struct {
	client *Client
	format Formatter
}

func (t TaskList) Apply(args []string) {
	switch len(args) {
	case 0:
		t.listAll()
	case 1:
		t.listById(args[0])
	default:
		Check(false, "too many arguments")
	}
}

func (t TaskList) listAll() {
	path := "/v2/tasks"
	request := t.client.GET(path)
	response, e := t.client.Do(request)
	Check(e == nil, "failed to get response", e)
	defer response.Body.Close()
	fmt.Println(t.format.Format(response.Body, t.HumanizeAll))
}

func (t TaskList) HumanizeAll(body io.Reader) string {
	dec := json.NewDecoder(body)
	var tasks Tasks
	e := dec.Decode(&tasks)
	Check(e == nil, "failed to unmarshal response", e)
	var b bytes.Buffer
	for _, task := range tasks.Tasks {
		b.WriteString(task.AppID)
		b.WriteString(" ")
		b.WriteString(task.Host)
		b.WriteString(" ")
		b.WriteString(task.Version)
		b.WriteString(" ")
		b.WriteString(task.ID)
		b.WriteString("\n")
	}
	title := "APPID HOST VERSION TASKID\n"
	text := title + b.String()
	return Columnize(text)
}

func (t TaskList) listById(id string) {
	esc := url.QueryEscape(id)
	path := "/v2/apps/" + esc + "?embed=apps.tasks"
	request := t.client.GET(path)
	response, e := t.client.Do(request)
	Check(e == nil, "failed to get response", e)
	defer response.Body.Close()
	fmt.Println(t.format.Format(response.Body, t.HumanizeById))
}

func (t TaskList) HumanizeById(body io.Reader) string {
	dec := json.NewDecoder(body)
	var appbyid AppById
	e := dec.Decode(&appbyid)
	Check(e == nil, "failed to unmarshal response", e)

	var b bytes.Buffer
	for _, task := range appbyid.App.Tasks {
		b.WriteString(task.ID)
		b.WriteString(" ")
		b.WriteString(task.Host)
		b.WriteString(" ")
		b.WriteString(task.Version)
		b.WriteString("\n")
		// ports?
	}
	title := "ID HOST VERSION\n"
	text := title + b.String()
	return Columnize(text)
}

type TaskKill struct {
	client *Client
	format Formatter
}

func (t TaskKill) Apply(args []string) {
	switch len(args) {
	case 1:
		t.killAll(args[0])
	case 2:
		t.killOnly(args[0], args[1])
	default:
		Check(false, "task kill takes 1 or 2 arguments")
	}
}

func (t TaskKill) killAll(id string) {
	esc := url.QueryEscape(id)
	path := "/v2/apps/" + esc + "/tasks"
	request := t.client.DELETE(path)
	response, e := t.client.Do(request)
	Check(e == nil, "failed to get response", e)
	defer response.Body.Close()

	sc := response.StatusCode
	Check(sc != 404, "unknown id")
	Check(sc == 200, "failed with status code", sc)
	t.format.Format(response.Body, t.Humanize)
}

func (t TaskKill) killOnly(id, taskid string) {
	escID := url.QueryEscape(id)
	escTaskID := url.QueryEscape(taskid)
	path := "/v2/apps/" + escID + "/tasks/" + escTaskID
	request := t.client.DELETE(path)
	response, e := t.client.Do(request)
	Check(e == nil, "failed to get response", e)
	defer response.Body.Close()
	sc := response.StatusCode
	Check(sc != 404, "unknown appid or taskid")
	Check(sc == 200, "failed with status code", sc)
	t.format.Format(response.Body, t.Humanize)
}

func (t TaskKill) Humanize(body io.Reader) string {
	// todo does this actually return a list of killed tasks?
	return "success"
}

type TaskQueue struct {
	client *Client
	format Formatter
}

func (t TaskQueue) Apply(args []string) {
	Check(len(args) == 0, "no arguments")
	request := t.client.GET("/v2/queue")
	response, e := t.client.Do(request)
	Check(e == nil, "failed to get response", e)
	defer response.Body.Close()
	fmt.Println(t.format.Format(response.Body, t.Humanize))
}

func (t TaskQueue) Humanize(body io.Reader) string {
	dec := json.NewDecoder(body)
	var queue Queue
	e := dec.Decode(&queue)
	Check(e == nil, "failed to decode response", e)
	title := "APP VERSION OVERDUE\n"
	var b bytes.Buffer
	for _, queuedTask := range queue.Queue {
		b.WriteString(queuedTask.App.ID)
		b.WriteString(" ")
		b.WriteString(queuedTask.App.Version)
		b.WriteString(" ")
		b.WriteString(strconv.FormatBool(queuedTask.Delay["overdue"]))
		b.WriteString("\n")
	}
	text := title + b.String()
	return Columnize(text)
}
<|endoftext|>


<commit_before>package tmsh

import (
	"bufio"
	"bytes"
	"fmt"
	"strings"
)

// BigIP is a struct for session state
type BigIP struct {
	host    string
	user    string
	sshconn SSH
}

// NewKeySession is NewSession plus key handling
func NewKeySession(host, port, user string, key []byte) (*BigIP, error) {
       return NewSession(host, port, user, "", key)
}

// NewSession sets up new SSH session to BIG-IP TMSH
func NewSession(host, port, user, password string) (*BigIP, error) {
	return GenSession(host,post,user,password,[]byte{})
}

// GenSession handles either Password or SSH Key based..
func GenSession(host, port, user, password string, key []byte) (*BigIP, error) {
    sshconn, err := newSSHConnection(host+":"+port, user, password, key)

	if err != nil {
		return nil, err
	}

	ret, _ := sshconn.Recv("# ")
	if !strings.Contains(string(ret), "(tmos)") {
		_, err := sshconn.Send("tmsh")
		if err != nil {
			return nil, err
		}

		ret, err = sshconn.Recv("# ")
		if err != nil {
			return nil, err
		}
	}

	bigip := &BigIP{
		host:    host,
		user:    user,
		sshconn: sshconn,
	}

	// Suppress pager output
	if _, err := bigip.ExecuteCommand("modify cli preference pager disabled display-threshold 0"); err != nil {
		return nil, err
	}

	return bigip, nil
}

// ExecuteCommand is used to execute any TMSH commands
func (bigip *BigIP) ExecuteCommand(cmd string) (string, error) {
	promptSuffix := "# "

	_, err := bigip.sshconn.Send(cmd)
	if err != nil {
		return "", err
	}

	results, err := bigip.sshconn.Recv(promptSuffix)
	if err != nil {
		return "", err
	}

	results = removeCarriageReturn(results)
	reader := bytes.NewReader(results)
	scanner := bufio.NewScanner(reader)

	var lines []string

	for scanner.Scan() {
		text := scanner.Text()
		line := removeSpaceAndBackspace(text)

		if strings.HasPrefix(line, "Last login:") ||
			strings.Contains(line, "(tmos)") ||
			strings.HasPrefix(line, cmd) {
			continue
		}

		lines = append(lines, text)
	}

	return strings.Join(lines, "\n"), nil
}

// Save is used to execute 'save /sys config' command
func (bigip *BigIP) Save() error {
	ret, err := bigip.ExecuteCommand("save /sys config current-partition")
	if err != nil {
		fmt.Errorf(ret)
	}

	if strings.Contains(ret, "Syntax Error: \"current-partition\" unknown property") {
		ret, err = bigip.ExecuteCommand("save /sys config")
		if err != nil {
			fmt.Errorf(ret)
		}
	}

	if strings.Contains(ret, "Error") {
		return fmt.Errorf(ret)
	}

	return nil
}

// Close is used to close SSH session
func (bigip *BigIP) Close() {
	bigip.sshconn.Close()
}
<commit_msg>Switch to GenSession<commit_after>package tmsh

import (
	"bufio"
	"bytes"
	"fmt"
	"strings"
)

// BigIP is a struct for session state
type BigIP struct {
	host    string
	user    string
	sshconn SSH
}

// NewKeySession is NewSession plus key handling
func NewKeySession(host, port, user string, key []byte) (*BigIP, error) {
       return GenSession(host, port, user, "", key)
}

// NewSession sets up new SSH session to BIG-IP TMSH
func NewSession(host, port, user, password string) (*BigIP, error) {
	return GenSession(host,post,user,password,[]byte{})
}

// GenSession handles either Password or SSH Key based..
func GenSession(host, port, user, password string, key []byte) (*BigIP, error) {
    sshconn, err := newSSHConnection(host+":"+port, user, password, key)

	if err != nil {
		return nil, err
	}

	ret, _ := sshconn.Recv("# ")
	if !strings.Contains(string(ret), "(tmos)") {
		_, err := sshconn.Send("tmsh")
		if err != nil {
			return nil, err
		}

		ret, err = sshconn.Recv("# ")
		if err != nil {
			return nil, err
		}
	}

	bigip := &BigIP{
		host:    host,
		user:    user,
		sshconn: sshconn,
	}

	// Suppress pager output
	if _, err := bigip.ExecuteCommand("modify cli preference pager disabled display-threshold 0"); err != nil {
		return nil, err
	}

	return bigip, nil
}

// ExecuteCommand is used to execute any TMSH commands
func (bigip *BigIP) ExecuteCommand(cmd string) (string, error) {
	promptSuffix := "# "

	_, err := bigip.sshconn.Send(cmd)
	if err != nil {
		return "", err
	}

	results, err := bigip.sshconn.Recv(promptSuffix)
	if err != nil {
		return "", err
	}

	results = removeCarriageReturn(results)
	reader := bytes.NewReader(results)
	scanner := bufio.NewScanner(reader)

	var lines []string

	for scanner.Scan() {
		text := scanner.Text()
		line := removeSpaceAndBackspace(text)

		if strings.HasPrefix(line, "Last login:") ||
			strings.Contains(line, "(tmos)") ||
			strings.HasPrefix(line, cmd) {
			continue
		}

		lines = append(lines, text)
	}

	return strings.Join(lines, "\n"), nil
}

// Save is used to execute 'save /sys config' command
func (bigip *BigIP) Save() error {
	ret, err := bigip.ExecuteCommand("save /sys config current-partition")
	if err != nil {
		fmt.Errorf(ret)
	}

	if strings.Contains(ret, "Syntax Error: \"current-partition\" unknown property") {
		ret, err = bigip.ExecuteCommand("save /sys config")
		if err != nil {
			fmt.Errorf(ret)
		}
	}

	if strings.Contains(ret, "Error") {
		return fmt.Errorf(ret)
	}

	return nil
}

// Close is used to close SSH session
func (bigip *BigIP) Close() {
	bigip.sshconn.Close()
}
<|endoftext|>


<commit_before>// Copyright (c) 2014, Mauro Toffanin. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"os"

	"github.com/toffanin/go-todo/commands"
	"github.com/toffanin/go-todo/utils"

	"github.com/codegangsta/cli"
	//"github.com/fatih/color"
)

var (
	appName = "todo"

	// The text template for the Default help topic
	appHelpTemplate = `
NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   [environment variables] {{.Name}} [global options] command [...]

VERSION:
   {{.Version}}

COMMANDS:
   {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
   {{end}}
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}
ENVIRONMENT VARIABLES:
   TODOTXT_AUTO_ARCHIVE=0,1{{"\t"}}is equivalent to global options -a (0) / -A (1)
   TODOTXT_CFG_FILE=CONFIG_FILE{{"\t"}}is equivalent to global option -d CONFIG_FILE
   TODOTXT_FORCE=1{{"\t"}}is equivalent to global option -f
   TODOTXT_PRESERVE_LINE_NUMBERS=0,1{{"\t"}}is equivalent to global options -n (0) / -N (1)
   TODOTXT_PLAIN=0,1{{"\t"}}is equivalent to global options -p (1) / -c (0)
   TODOTXT_DATE_ON_ADD=0,1{{"\t"}}is equivalent to global options -t (1) / -T (0)
   TODOTXT_VERBOSE=1{{ "\t" }}is equivalent to global option -v
   TODOTXT_DISABLE_FILTER=1{{ "\t" }}is equivalent to global option -x
   TODOTXT_DEFAULT_ACTION=""{{ "\t" }}run this when called with no arguments
   TODOTXT_SORT_COMMAND="sort ..."{{ "\t" }}customize list output
   TODOTXT_FINAL_FILTER="sed ..."{{ "\t" }}customize list after color, P@+ hiding
   TODOTXT_SOURCEVAR=\$DONE_FILE{{ "\t" }}use another source for listcon, listproj

`

	// The text template for the command help topic.
	commandHelpTemplate = `
NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   ` + appName + ` {{.Name}} [options] [arguments...]

DESCRIPTION:
   {{.Description}}

OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}

`
)

func main() {

	// Load Todo.txt CLI environment variables
	utils.LoadConfig()

	// Initialize the templates for help sections
	cli.AppHelpTemplate = appHelpTemplate
	cli.CommandHelpTemplate = commandHelpTemplate

	// Initialize the app CLI
	app := cli.NewApp()

	app.Name = appName
	app.Usage = "A simple and extensible utility for managing your todo.txt files"
	app.Version = "1.0.1"
	app.Author = "Mauro Toffanin"
	app.Email = "toffanin.mauro@gmail.com"
	app.EnableBashCompletion = true
	app.Flags = []cli.Flag{
		cli.BoolFlag{"t", "Prefixes the current date to a task automatically when it's added"},
		cli.BoolFlag{"T", "Do not prefix the current date to a task automatically when it's added"},
		cli.BoolFlag{"f", "Forces actions without confirmation or interactive input"},
	}
	app.Commands = []cli.Command{
		commands.GetEnv(),
		commands.GetInit(),
		commands.GetShorthelp(),
		commands.GetAdd(),
		commands.GetAddm(),
		commands.GetList(),
		/*{
			Name:  "status",
			Usage: "Obtain a summary of the todo.txt structure",
			Flags: []cli.Flag{
				cli.StringFlag{"dest, d", "", "specifies a different destination path"},
			},
			Action: func(c *cli.Context) {
				//fmt.Println("status: ", c.Args().First())
			},
		},*/
	}

	if err := app.Run(os.Args); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<commit_msg>Print the name of the app in the command help section to avoid confusion<commit_after>// Copyright (c) 2014, Mauro Toffanin. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"os"

	"github.com/toffanin/go-todo/commands"
	"github.com/toffanin/go-todo/utils"

	"github.com/codegangsta/cli"
	//"github.com/fatih/color"
)

var (
	appName = "todo"

	// The text template for the Default help topic
	appHelpTemplate = `
NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   [environment variables] {{.Name}} [global options] command [...]

VERSION:
   {{.Version}}

COMMANDS:
   {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
   {{end}}
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}
ENVIRONMENT VARIABLES:
   TODOTXT_AUTO_ARCHIVE=0,1{{"\t"}}is equivalent to global options -a (0) / -A (1)
   TODOTXT_CFG_FILE=CONFIG_FILE{{"\t"}}is equivalent to global option -d CONFIG_FILE
   TODOTXT_FORCE=1{{"\t"}}is equivalent to global option -f
   TODOTXT_PRESERVE_LINE_NUMBERS=0,1{{"\t"}}is equivalent to global options -n (0) / -N (1)
   TODOTXT_PLAIN=0,1{{"\t"}}is equivalent to global options -p (1) / -c (0)
   TODOTXT_DATE_ON_ADD=0,1{{"\t"}}is equivalent to global options -t (1) / -T (0)
   TODOTXT_VERBOSE=1{{ "\t" }}is equivalent to global option -v
   TODOTXT_DISABLE_FILTER=1{{ "\t" }}is equivalent to global option -x
   TODOTXT_DEFAULT_ACTION=""{{ "\t" }}run this when called with no arguments
   TODOTXT_SORT_COMMAND="sort ..."{{ "\t" }}customize list output
   TODOTXT_FINAL_FILTER="sed ..."{{ "\t" }}customize list after color, P@+ hiding
   TODOTXT_SOURCEVAR=\$DONE_FILE{{ "\t" }}use another source for listcon, listproj

`

	// The text template for the command help topic.
	commandHelpTemplate = `
NAME:
   ` + appName + `

COMMAND:
   {{.Name}} - {{.Usage}}

USAGE:
   ` + appName + ` {{.Name}} [options] [arguments...]

DESCRIPTION:
   {{.Description}}

OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}

`
)

func main() {

	// Load Todo.txt CLI environment variables
	utils.LoadConfig()

	// Initialize the templates for help sections
	cli.AppHelpTemplate = appHelpTemplate
	cli.CommandHelpTemplate = commandHelpTemplate

	// Initialize the app CLI
	app := cli.NewApp()

	app.Name = appName
	app.Usage = "A simple and extensible utility for managing your todo.txt files"
	app.Version = "1.0.1"
	app.Author = "Mauro Toffanin"
	app.Email = "toffanin.mauro@gmail.com"
	app.EnableBashCompletion = true
	app.Flags = []cli.Flag{
		cli.BoolFlag{"t", "Prefixes the current date to a task automatically when it's added"},
		cli.BoolFlag{"T", "Do not prefix the current date to a task automatically when it's added"},
		cli.BoolFlag{"f", "Forces actions without confirmation or interactive input"},
	}
	app.Commands = []cli.Command{
		commands.GetEnv(),
		commands.GetInit(),
		commands.GetShorthelp(),
		commands.GetAdd(),
		commands.GetAddm(),
		commands.GetList(),
		/*{
			Name:  "status",
			Usage: "Obtain a summary of the todo.txt structure",
			Flags: []cli.Flag{
				cli.StringFlag{"dest, d", "", "specifies a different destination path"},
			},
			Action: func(c *cli.Context) {
				//fmt.Println("status: ", c.Args().First())
			},
		},*/
	}

	if err := app.Run(os.Args); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<|endoftext|>