<commit_before>// Copyright (c) 2016-2017 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package calc

import (
	log "github.com/Sirupsen/logrus"

	"fmt"

	"github.com/projectcalico/felix/multidict"
	"github.com/projectcalico/felix/set"
	"github.com/projectcalico/libcalico-go/lib/backend/model"
	"github.com/projectcalico/libcalico-go/lib/net"
	"github.com/projectcalico/libcalico-go/lib/numorstring"
	"github.com/projectcalico/libcalico-go/lib/selector"
)

// RuleScanner scans the rules sent to it by the ActiveRulesCalculator, looking for tags and
// selectors. It calculates the set of active tags and selectors and emits events when they become
// active/inactive.  To avoid the occupancy overhead of doing two types of indexing, it maps tags
// to selectors of the form "has(tag-name)"; those are broadly equivalent due to the inheritance
// mechanism implemented in the labelindex.InheritIndex.
//
// The RuleScanner also emits events when rules are updated:  since the input rule
// structs contain tags and selectors but downstream, we only care about IP sets, the
// RuleScanner converts rules from model.Rule objects to calc.ParsedRule objects.
// The latter share most fields, but the tags and selector fields are replaced by lists of
// IP sets.
//
// The RuleScanner only calculates which selectors and tags are active/inactive.  It doesn't
// match endpoints against tags/selectors.  (That is done downstream in a labelindex.InheritIndex
// created in NewCalculationGraph.)
type RuleScanner struct {
	// selectorsByUID maps from the selector's hash back to the selector.
	selectorsByUID map[string]selector.Selector
	// activeUidsByResource maps from policy or profile ID to "set" of selector UIDs
	rulesIDToUIDs multidict.IfaceToString
	// activeResourcesByUid maps from selector UID back to the "set" of resources using it.
	uidsToRulesIDs multidict.StringToIface

	OnSelectorActive   func(selector selector.Selector)
	OnSelectorInactive func(selector selector.Selector)

	RulesUpdateCallbacks rulesUpdateCallbacks
}

func NewRuleScanner() *RuleScanner {
	calc := &RuleScanner{
		selectorsByUID: make(map[string]selector.Selector),
		rulesIDToUIDs:  multidict.NewIfaceToString(),
		uidsToRulesIDs: multidict.NewStringToIface(),
	}
	return calc
}

func (rs *RuleScanner) OnProfileActive(key model.ProfileRulesKey, profile *model.ProfileRules) {
	parsedRules := rs.updateRules(key, profile.InboundRules, profile.OutboundRules, false)
	rs.RulesUpdateCallbacks.OnProfileActive(key, parsedRules)
}

func (rs *RuleScanner) OnProfileInactive(key model.ProfileRulesKey) {
	rs.updateRules(key, nil, nil, false)
	rs.RulesUpdateCallbacks.OnProfileInactive(key)
}

func (rs *RuleScanner) OnPolicyActive(key model.PolicyKey, policy *model.Policy) {
	parsedRules := rs.updateRules(key, policy.InboundRules, policy.OutboundRules, policy.DoNotTrack)
	rs.RulesUpdateCallbacks.OnPolicyActive(key, parsedRules)
}

func (rs *RuleScanner) OnPolicyInactive(key model.PolicyKey) {
	rs.updateRules(key, nil, nil, false)
	rs.RulesUpdateCallbacks.OnPolicyInactive(key)
}

func (rs *RuleScanner) updateRules(key interface{}, inbound, outbound []model.Rule, untracked bool) (parsedRules *ParsedRules) {
	log.Debugf("Scanning rules (%v in, %v out) for key %v",
		len(inbound), len(outbound), key)
	// Extract all the new selectors/tags.
	currentUIDToSel := make(map[string]selector.Selector)
	parsedInbound := make([]*ParsedRule, len(inbound))
	for ii, rule := range inbound {
		parsed, allSels, err := ruleToParsedRule(&rule)
		if err != nil {
			log.Fatalf("Bad selector in %v: %v", key, err)
		}
		parsedInbound[ii] = parsed
		for _, sel := range allSels {
			currentUIDToSel[sel.UniqueId()] = sel
		}
	}
	parsedOutbound := make([]*ParsedRule, len(outbound))
	for ii, rule := range outbound {
		parsed, allSels, err := ruleToParsedRule(&rule)
		if err != nil {
			log.Fatalf("Bad selector in %v: %v", key, err)
		}
		parsedOutbound[ii] = parsed
		for _, sel := range allSels {
			currentUIDToSel[sel.UniqueId()] = sel
		}
	}
	parsedRules = &ParsedRules{
		InboundRules:  parsedInbound,
		OutboundRules: parsedOutbound,
		Untracked:     untracked,
	}

	// Figure out which selectors/tags are new.
	addedUids := set.New()
	for uid := range currentUIDToSel {
		log.Debugf("Checking if UID %v is new.", uid)
		if !rs.rulesIDToUIDs.Contains(key, uid) {
			log.Debugf("UID %v is new", uid)
			addedUids.Add(uid)
		}
	}

	// Figure out which selectors/tags are no-longer in use.
	removedUids := set.New()
	rs.rulesIDToUIDs.Iter(key, func(uid string) {
		if _, ok := currentUIDToSel[uid]; !ok {
			log.Debugf("Removed UID: %v", uid)
			removedUids.Add(uid)
		}
	})

	// Add the new into the index, triggering events as we discover
	// newly-active tags/selectors.
	addedUids.Iter(func(item interface{}) error {
		uid := item.(string)
		rs.rulesIDToUIDs.Put(key, uid)
		if !rs.uidsToRulesIDs.ContainsKey(uid) {
			sel := currentUIDToSel[uid]
			rs.selectorsByUID[uid] = sel
			log.Debugf("Selector became active: %v -> %v",
				uid, sel)
			// This selector just became active, trigger event.
			rs.OnSelectorActive(sel)
		}
		rs.uidsToRulesIDs.Put(uid, key)
		return nil
	})

	// And remove the old, triggering events as we clean up unused
	// selectors/tags.
	removedUids.Iter(func(item interface{}) error {
		uid := item.(string)
		rs.rulesIDToUIDs.Discard(key, uid)
		rs.uidsToRulesIDs.Discard(uid, key)
		if !rs.uidsToRulesIDs.ContainsKey(uid) {
			log.Debugf("Selector/tag became inactive: %v", uid)
			sel := rs.selectorsByUID[uid]
			delete(rs.selectorsByUID, uid)

			// This selector just became inactive, trigger event.
			log.Debugf("Selector became inactive: %v -> %v",
				uid, sel)
			rs.OnSelectorInactive(sel)
		}
		return nil
	})
	return
}

type ParsedRules struct {
	InboundRules  []*ParsedRule
	OutboundRules []*ParsedRule

	// Untracked is true if these rules should not be "conntracked".
	Untracked bool
}

// Rule is like a backend.model.Rule, except the tag and selector matches are
// replaced with pre-calculated ipset IDs.
type ParsedRule struct {
	Action string

	IPVersion *int

	Protocol *numorstring.Protocol

	SrcNet      *net.IPNet
	SrcPorts    []numorstring.Port
	DstNet      *net.IPNet
	DstPorts    []numorstring.Port
	ICMPType    *int
	ICMPCode    *int
	SrcIPSetIDs []string
	DstIPSetIDs []string

	NotProtocol    *numorstring.Protocol
	NotSrcNet      *net.IPNet
	NotSrcPorts    []numorstring.Port
	NotDstNet      *net.IPNet
	NotDstPorts    []numorstring.Port
	NotICMPType    *int
	NotICMPCode    *int
	NotSrcIPSetIDs []string
	NotDstIPSetIDs []string
}

func ruleToParsedRule(rule *model.Rule) (parsedRule *ParsedRule, allTagOrSels []selector.Selector, err error) {
	src, dst, notSrc, notDst, err := extractTagsAndSelectors(rule)
	if err != nil {
		return
	}

	parsedRule = &ParsedRule{
		Action: rule.Action,

		IPVersion: rule.IPVersion,

		Protocol: rule.Protocol,

		SrcNet:      rule.SrcNet,
		SrcPorts:    rule.SrcPorts,
		DstNet:      rule.DstNet,
		DstPorts:    rule.DstPorts,
		ICMPType:    rule.ICMPType,
		ICMPCode:    rule.ICMPCode,
		SrcIPSetIDs: selectors(src).ToUIDs(),
		DstIPSetIDs: selectors(dst).ToUIDs(),

		NotProtocol:    rule.NotProtocol,
		NotSrcNet:      rule.NotSrcNet,
		NotSrcPorts:    rule.NotSrcPorts,
		NotDstNet:      rule.NotDstNet,
		NotDstPorts:    rule.NotDstPorts,
		NotICMPType:    rule.NotICMPType,
		NotICMPCode:    rule.NotICMPCode,
		NotSrcIPSetIDs: selectors(notSrc).ToUIDs(),
		NotDstIPSetIDs: selectors(notDst).ToUIDs(),
	}

	allTagOrSels = append(allTagOrSels, src...)
	allTagOrSels = append(allTagOrSels, dst...)
	allTagOrSels = append(allTagOrSels, notSrc...)
	allTagOrSels = append(allTagOrSels, notDst...)

	return
}

func extractTagsAndSelectors(rule *model.Rule) (src, dst, notSrc, notDst []selector.Selector, err error) {
	var sel selector.Selector
	if rule.SrcTag != "" {
		sel, err = selFromTag(rule.SrcTag)
		if err != nil {
			return
		}
		src = append(src, sel)
	}
	if rule.DstTag != "" {
		sel, err = selFromTag(rule.DstTag)
		if err != nil {
			return
		}
		dst = append(dst, sel)
	}
	if rule.NotSrcTag != "" {
		sel, err = selFromTag(rule.NotSrcTag)
		if err != nil {
			return
		}
		notSrc = append(notSrc, sel)
	}
	if rule.NotDstTag != "" {
		sel, err = selFromTag(rule.NotDstTag)
		if err != nil {
			return
		}
		notDst = append(notDst, sel)
	}
	if rule.SrcSelector != "" {
		sel, err = selector.Parse(rule.SrcSelector)
		if err != nil {
			return
		}
		src = append(src, sel)
	}
	if rule.DstSelector != "" {
		sel, err = selector.Parse(rule.DstSelector)
		if err != nil {
			return
		}
		dst = append(dst, sel)
	}
	if rule.NotSrcSelector != "" {
		sel, err = selector.Parse(rule.NotSrcSelector)
		if err != nil {
			return
		}
		notSrc = append(notSrc, sel)
	}
	if rule.NotDstSelector != "" {
		sel, err = selector.Parse(rule.NotDstSelector)
		if err != nil {
			return
		}
		notDst = append(notDst, sel)
	}
	return
}

func selFromTag(tag string) (selector.Selector, error) {
	return selector.Parse(fmt.Sprintf("has(%s)", tag))
}

type selectors []selector.Selector

func (ss selectors) ToUIDs() []string {
	if len(ss) == 0 {
		return nil
	}
	uids := make([]string, len(ss))
	for i, sel := range ss {
		uids[i] = sel.UniqueId()
	}
	return uids
}
<commit_msg>Code review markups.<commit_after>// Copyright (c) 2016-2017 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package calc

import (
	log "github.com/Sirupsen/logrus"

	"fmt"

	"github.com/projectcalico/felix/multidict"
	"github.com/projectcalico/felix/set"
	"github.com/projectcalico/libcalico-go/lib/backend/model"
	"github.com/projectcalico/libcalico-go/lib/net"
	"github.com/projectcalico/libcalico-go/lib/numorstring"
	"github.com/projectcalico/libcalico-go/lib/selector"
)

// RuleScanner scans the rules sent to it by the ActiveRulesCalculator, looking for tags and
// selectors. It calculates the set of active tags and selectors and emits events when they become
// active/inactive.
//
// Previously, Felix tracked tags and selectors separately, with a separate tag and label index.
// However, we found that had a high occupancy cost.  The current code uses a shared index and
// maps tags onto labels, so a tag named tagName, becomes a label tagName="".  The RuleScanner
// maps tags to label selectors of the form "has(tagName)", taking advantage of the mapping.
// Such a selector is almost equivalent to having the tag; the only case where the behaviour would
// differ is if the user was using the same name for a tag and a label and the label and tags
// of the same name were applied to different endpoints.  Since tags are being deprecated, we can
// live with that potential aliasing issue in return for a significant occupancy improvement at
// high scale.
//
// The RuleScanner also emits events when rules are updated:  since the input rule
// structs contain tags and selectors but downstream, we only care about IP sets, the
// RuleScanner converts rules from model.Rule objects to calc.ParsedRule objects.
// The latter share most fields, but the tags and selector fields are replaced by lists of
// IP sets.
//
// The RuleScanner only calculates which selectors and tags are active/inactive.  It doesn't
// match endpoints against tags/selectors.  (That is done downstream in a labelindex.InheritIndex
// created in NewCalculationGraph.)
type RuleScanner struct {
	// selectorsByUID maps from the selector's hash back to the selector.
	selectorsByUID map[string]selector.Selector
	// activeUidsByResource maps from policy or profile ID to "set" of selector UIDs
	rulesIDToUIDs multidict.IfaceToString
	// activeResourcesByUid maps from selector UID back to the "set" of resources using it.
	uidsToRulesIDs multidict.StringToIface

	OnSelectorActive   func(selector selector.Selector)
	OnSelectorInactive func(selector selector.Selector)

	RulesUpdateCallbacks rulesUpdateCallbacks
}

func NewRuleScanner() *RuleScanner {
	calc := &RuleScanner{
		selectorsByUID: make(map[string]selector.Selector),
		rulesIDToUIDs:  multidict.NewIfaceToString(),
		uidsToRulesIDs: multidict.NewStringToIface(),
	}
	return calc
}

func (rs *RuleScanner) OnProfileActive(key model.ProfileRulesKey, profile *model.ProfileRules) {
	parsedRules := rs.updateRules(key, profile.InboundRules, profile.OutboundRules, false)
	rs.RulesUpdateCallbacks.OnProfileActive(key, parsedRules)
}

func (rs *RuleScanner) OnProfileInactive(key model.ProfileRulesKey) {
	rs.updateRules(key, nil, nil, false)
	rs.RulesUpdateCallbacks.OnProfileInactive(key)
}

func (rs *RuleScanner) OnPolicyActive(key model.PolicyKey, policy *model.Policy) {
	parsedRules := rs.updateRules(key, policy.InboundRules, policy.OutboundRules, policy.DoNotTrack)
	rs.RulesUpdateCallbacks.OnPolicyActive(key, parsedRules)
}

func (rs *RuleScanner) OnPolicyInactive(key model.PolicyKey) {
	rs.updateRules(key, nil, nil, false)
	rs.RulesUpdateCallbacks.OnPolicyInactive(key)
}

func (rs *RuleScanner) updateRules(key interface{}, inbound, outbound []model.Rule, untracked bool) (parsedRules *ParsedRules) {
	log.Debugf("Scanning rules (%v in, %v out) for key %v",
		len(inbound), len(outbound), key)
	// Extract all the new selectors/tags.
	currentUIDToSel := make(map[string]selector.Selector)
	parsedInbound := make([]*ParsedRule, len(inbound))
	for ii, rule := range inbound {
		parsed, allSels := ruleToParsedRule(&rule)
		parsedInbound[ii] = parsed
		for _, sel := range allSels {
			currentUIDToSel[sel.UniqueId()] = sel
		}
	}
	parsedOutbound := make([]*ParsedRule, len(outbound))
	for ii, rule := range outbound {
		parsed, allSels := ruleToParsedRule(&rule)
		parsedOutbound[ii] = parsed
		for _, sel := range allSels {
			currentUIDToSel[sel.UniqueId()] = sel
		}
	}
	parsedRules = &ParsedRules{
		InboundRules:  parsedInbound,
		OutboundRules: parsedOutbound,
		Untracked:     untracked,
	}

	// Figure out which selectors/tags are new.
	addedUids := set.New()
	for uid := range currentUIDToSel {
		log.Debugf("Checking if UID %v is new.", uid)
		if !rs.rulesIDToUIDs.Contains(key, uid) {
			log.Debugf("UID %v is new", uid)
			addedUids.Add(uid)
		}
	}

	// Figure out which selectors/tags are no-longer in use.
	removedUids := set.New()
	rs.rulesIDToUIDs.Iter(key, func(uid string) {
		if _, ok := currentUIDToSel[uid]; !ok {
			log.Debugf("Removed UID: %v", uid)
			removedUids.Add(uid)
		}
	})

	// Add the new into the index, triggering events as we discover
	// newly-active tags/selectors.
	addedUids.Iter(func(item interface{}) error {
		uid := item.(string)
		rs.rulesIDToUIDs.Put(key, uid)
		if !rs.uidsToRulesIDs.ContainsKey(uid) {
			sel := currentUIDToSel[uid]
			rs.selectorsByUID[uid] = sel
			log.Debugf("Selector became active: %v -> %v",
				uid, sel)
			// This selector just became active, trigger event.
			rs.OnSelectorActive(sel)
		}
		rs.uidsToRulesIDs.Put(uid, key)
		return nil
	})

	// And remove the old, triggering events as we clean up unused
	// selectors/tags.
	removedUids.Iter(func(item interface{}) error {
		uid := item.(string)
		rs.rulesIDToUIDs.Discard(key, uid)
		rs.uidsToRulesIDs.Discard(uid, key)
		if !rs.uidsToRulesIDs.ContainsKey(uid) {
			log.Debugf("Selector/tag became inactive: %v", uid)
			sel := rs.selectorsByUID[uid]
			delete(rs.selectorsByUID, uid)

			// This selector just became inactive, trigger event.
			log.Debugf("Selector became inactive: %v -> %v",
				uid, sel)
			rs.OnSelectorInactive(sel)
		}
		return nil
	})
	return
}

type ParsedRules struct {
	InboundRules  []*ParsedRule
	OutboundRules []*ParsedRule

	// Untracked is true if these rules should not be "conntracked".
	Untracked bool
}

// Rule is like a backend.model.Rule, except the tag and selector matches are
// replaced with pre-calculated ipset IDs.
type ParsedRule struct {
	Action string

	IPVersion *int

	Protocol *numorstring.Protocol

	SrcNet      *net.IPNet
	SrcPorts    []numorstring.Port
	DstNet      *net.IPNet
	DstPorts    []numorstring.Port
	ICMPType    *int
	ICMPCode    *int
	SrcIPSetIDs []string
	DstIPSetIDs []string

	NotProtocol    *numorstring.Protocol
	NotSrcNet      *net.IPNet
	NotSrcPorts    []numorstring.Port
	NotDstNet      *net.IPNet
	NotDstPorts    []numorstring.Port
	NotICMPType    *int
	NotICMPCode    *int
	NotSrcIPSetIDs []string
	NotDstIPSetIDs []string
}

func ruleToParsedRule(rule *model.Rule) (parsedRule *ParsedRule, allTagOrSels []selector.Selector) {
	src, dst, notSrc, notDst := extractTagsAndSelectors(rule)

	parsedRule = &ParsedRule{
		Action: rule.Action,

		IPVersion: rule.IPVersion,

		Protocol: rule.Protocol,

		SrcNet:      rule.SrcNet,
		SrcPorts:    rule.SrcPorts,
		DstNet:      rule.DstNet,
		DstPorts:    rule.DstPorts,
		ICMPType:    rule.ICMPType,
		ICMPCode:    rule.ICMPCode,
		SrcIPSetIDs: selectors(src).ToUIDs(),
		DstIPSetIDs: selectors(dst).ToUIDs(),

		NotProtocol:    rule.NotProtocol,
		NotSrcNet:      rule.NotSrcNet,
		NotSrcPorts:    rule.NotSrcPorts,
		NotDstNet:      rule.NotDstNet,
		NotDstPorts:    rule.NotDstPorts,
		NotICMPType:    rule.NotICMPType,
		NotICMPCode:    rule.NotICMPCode,
		NotSrcIPSetIDs: selectors(notSrc).ToUIDs(),
		NotDstIPSetIDs: selectors(notDst).ToUIDs(),
	}

	allTagOrSels = append(allTagOrSels, src...)
	allTagOrSels = append(allTagOrSels, dst...)
	allTagOrSels = append(allTagOrSels, notSrc...)
	allTagOrSels = append(allTagOrSels, notDst...)

	return
}

func extractTagsAndSelectors(rule *model.Rule) (src, dst, notSrc, notDst []selector.Selector) {
	appendTagSelector := func(slice []selector.Selector, tagName string) []selector.Selector {
		if tagName == "" {
			return slice
		}
		sel, err := selFromTag(tagName)
		if err != nil {
			// This shouldn't happen because the data should have been validated
			// further back in the pipeline.
			log.WithField("tag", tagName).Panic(
				"Failed to convert tag to selector; but tag should have been " +
					"validated already.")
		}
		return append(slice, sel)
	}

	appendSelector := func(slice []selector.Selector, rawSelector string) []selector.Selector {
		if rawSelector == "" {
			return slice
		}
		sel, err := selector.Parse(rawSelector)
		if err != nil {
			// This shouldn't happen because the data should have been validated
			// further back in the pipeline.
			log.WithField("selector", rawSelector).Panic(
				"Failed to parse selector that should have been validated already.")
		}
		return append(slice, sel)
	}

	src = appendTagSelector(src, rule.SrcTag)
	src = appendSelector(src, rule.SrcSelector)

	dst = appendTagSelector(dst, rule.DstTag)
	dst = appendSelector(dst, rule.DstSelector)

	notSrc = appendTagSelector(notSrc, rule.NotSrcTag)
	notSrc = appendSelector(notSrc, rule.NotSrcSelector)

	notDst = appendTagSelector(notDst, rule.NotDstTag)
	notDst = appendSelector(notDst, rule.NotDstSelector)

	return
}

func selFromTag(tag string) (selector.Selector, error) {
	return selector.Parse(fmt.Sprintf("has(%s)", tag))
}

type selectors []selector.Selector

func (ss selectors) ToUIDs() []string {
	if len(ss) == 0 {
		return nil
	}
	uids := make([]string, len(ss))
	for i, sel := range ss {
		uids[i] = sel.UniqueId()
	}
	return uids
}
<|endoftext|>


<commit_before>package irmaclient

import (
	"crypto/rand"
	"encoding/binary"

	"fmt"
	"math"
	"sync"
	"time"

	"github.com/getsentry/raven-go"
	"github.com/go-errors/errors"
	"github.com/privacybydesign/gabi/revocation"
	irma "github.com/privacybydesign/irmago"
	"github.com/sirupsen/logrus"
)

func (client *Client) initRevocation() {
	// For every credential supporting revocation, compute nonrevocation caches in async jobs
	for id, attrsets := range client.attributes {
		for i, attrs := range attrsets {
			if attrs.CredentialType() == nil || !attrs.CredentialType().RevocationSupported() {
				continue
			}
			id := id // make copy of same name to capture the value for closure below
			i := i   // see https://golang.org/doc/faq#closures_and_goroutines
			client.jobs <- func() {
				if err := client.nonrevPrepareCache(id, i); err != nil {
					client.reportError(err)
				}
			}
		}
	}

	// Of each credential supporting revocation, we periodically update its nonrevocation witness
	// by fetching updates from the issuer's server, such that:
	// - The time interval between two updates is random so that the server cannot recognize us
	//   using the update interval,
	// - Updating happens regularly even if the app is rarely used.
	// We do this by every 10 seconds updating the credential with a low probability, which
	// increases over time since the last update.
	client.Configuration.Scheduler.Every(irma.RevocationParameters.ClientUpdateInterval).Seconds().Do(func() {
		for id, attrsets := range client.attributes {
			for i, attrs := range attrsets {
				if attrs.CredentialType() == nil || !attrs.CredentialType().RevocationSupported() {
					continue
				}
				cred, err := client.credential(id, i)
				if err != nil {
					client.reportError(err)
					continue
				}
				if cred.NonRevocationWitness == nil {
					continue
				}
				r, err := randomfloat()
				if err != nil {
					client.reportError(err)
					break
				}
				speed := attrs.CredentialType().RevocationUpdateSpeed * 60 * 60
				p := probability(cred.NonRevocationWitness.Updated, speed)
				irma.Logger.Tracef("random update: updated, speed, r, p = %s, %d, %.6f, %.6f",
					cred.NonRevocationWitness.Updated.String(), speed, r, p)
				if r < p {
					irma.Logger.Debugf("scheduling nonrevocation witness remote update for %s-%s", id, attrs.Hash())
					client.jobs <- func() {
						if err = client.nonrevUpdateFromServer(id); err != nil {
							client.reportError(err)
							return
						}
					}
				}
			}
		}
	})
}

// NonrevPrepare updates the revocation state for each credential in the request
// requiring a nonrevocation proof, using the updates included in the request, or the remote
// revocation server if those do not suffice.
func (client *Client) NonrevPrepare(request irma.SessionRequest) error {
	base := request.Base()
	var err error
	var wg sync.WaitGroup
	for id := range request.Disclosure().Identifiers().CredentialTypes {
		credtype := client.Configuration.CredentialTypes[id]
		if !credtype.RevocationSupported() {
			continue
		}
		if !base.RequestsRevocation(id) {
			continue
		}
		irma.Logger.WithField("credtype", id).Debug("updating witnesses")
		wg.Add(1)
		go func() {
			if e := client.nonrevUpdate(id, base.Revocation[id].Updates); e != nil {
				err = e // overwrites err from previously finished call, if any
			}
			wg.Done()
		}()
	}
	wg.Wait()
	return err
}

// nonrevUpdate updates all contained instances of the specified type, using the specified
// updates if present and if they suffice, and contacting the issuer's server to download updates
// otherwise.
func (client *Client) nonrevUpdate(id irma.CredentialTypeIdentifier, updates map[uint]*revocation.Update) error {
	lowest := map[uint]uint64{}
	attrs := client.attrs(id)

	// Per credential and issuer key counter we may posess multiple credential instances.
	// Of the nonrevocation witnesses of these, take the lowest index.
	for i := 0; i < len(attrs); i++ {
		cred, err := client.credential(id, i)
		if err != nil {
			return err
		}
		if cred.NonRevocationWitness == nil {
			continue
		}
		pkid := cred.Pk.Counter
		_, present := lowest[pkid]
		if !present || cred.NonRevocationWitness.SignedAccumulator.Accumulator.Index < lowest[pkid] {
			lowest[pkid] = cred.NonRevocationWitness.SignedAccumulator.Accumulator.Index
		}
	}

	// For each key counter, get an update message starting at the lowest index computed above,
	// that can update all of our credential instance of the given type and key counter,
	// using the specified update messags if they suffice, or the issuer's server otherwise.
	u := map[uint]*revocation.Update{}
	for counter, l := range lowest {
		update := updates[counter]
		if updates != nil && (update == nil || len(update.Events) == 0) {
			return errors.Errorf("missing revocation update for %s-%d", id, counter)
		}
		if update != nil && update.Events[0].Index <= l+1 {
			u[counter] = update
		} else {
			var err error
			u[counter], err = irma.RevocationClient{Conf: client.Configuration}.
				FetchUpdateFrom(id, counter, l+1)
			if err != nil {
				return err
			}
		}
	}

	// Apply the update messages to all instances of the given type and key counter
	for counter, update := range u {
		if err := client.nonrevApplyUpdates(id, counter, update); err != nil {
			return err
		}
	}
	return nil
}

func (client *Client) nonrevApplyUpdates(id irma.CredentialTypeIdentifier, counter uint, update *revocation.Update) error {
	client.credMutex.Lock()
	defer client.credMutex.Unlock()

	attrs := client.attrs(id)
	var save bool
	for i := 0; i < len(attrs); i++ {
		cred, err := client.credential(id, i)
		if err != nil {
			return err
		}
		if cred.NonRevocationWitness == nil || cred.Pk.Counter != counter {
			continue
		}
		updated, err := cred.nonrevApplyUpdates(update, irma.RevocationKeys{Conf: client.Configuration})
		if updated {
			save = true
		}
		if err == revocation.ErrorRevoked {
			id := cred.CredentialType().Identifier()
			hash := cred.attrs.Hash()
			irma.Logger.Warnf("credential %s %s revoked", id, hash)
			attrs[i].Revoked = true
			cred.attrs.Revoked = true
			save = true
			client.handler.Revoked(&irma.CredentialIdentifier{Type: id, Hash: hash})
			// Even if this credential is revoked during a session, we may have
			// other instances that can satisfy the request. So don't return an
			// error which would halt the session.
			continue
		}
		if err != nil {
			return err
		}
		// Asynchroniously update nonrevocation proof cache from updated witness
		irma.Logger.WithField("credtype", id).Debug("scheduling nonrevocation cache update")
		go func(cred *credential) {
			if err := cred.NonrevPrepareCache(); err != nil {
				raven.CaptureError(err, nil)
			}
		}(cred)
	}
	if save {
		if err := client.storage.StoreAttributes(id, client.attributes[id]); err != nil {
			client.reportError(err)
			return err
		}
	}
	return nil
}

func (client *Client) nonrevUpdateFromServer(id irma.CredentialTypeIdentifier) error {
	return client.nonrevUpdate(id, nil)
}

func (client *Client) nonrevPrepareCache(id irma.CredentialTypeIdentifier, index int) error {
	logger := irma.Logger.WithFields(logrus.Fields{"credtype": id, "index": index})
	logger.Debug("preparing cache")
	defer logger.Debug("Preparing cache done")
	cred, err := client.credential(id, index)
	if err != nil {
		return err
	}
	return cred.NonrevPrepareCache()
}

// nonrevRepopulateCaches repopulates the consumed nonrevocation caches of the credentials involved
// in the request, in background jobs, after the request has finished.
func (client *Client) nonrevRepopulateCaches(request irma.SessionRequest) {
	for id := range request.Disclosure().Identifiers().CredentialTypes {
		credtype := client.Configuration.CredentialTypes[id]
		if !credtype.RevocationSupported() {
			continue
		}
		for i := range client.attrs(id) {
			id := id
			i := i
			client.jobs <- func() {
				if err := client.nonrevPrepareCache(id, i); err != nil {
					client.reportError(err)
				}
			}
		}
	}
}

// nonrevApplyUpdates updates the credential's nonrevocation witness using the specified messages,
// if they all verify and if their indices are ahead and adjacent to that of our witness.
func (cred *credential) nonrevApplyUpdates(update *revocation.Update, keys irma.RevocationKeys) (bool, error) {
	oldindex := cred.NonRevocationWitness.SignedAccumulator.Accumulator.Index

	pk, err := keys.PublicKey(cred.CredentialType().IssuerIdentifier(), update.SignedAccumulator.PKCounter)
	if err != nil {
		return false, err
	}
	logger := irma.Logger.WithFields(logrus.Fields{"credtype": cred.CredentialType().Identifier(), "hash": cred.attrs.Hash()})
	logger.Debugf("updating witness")
	defer logger.Debug("updating witness done")
	if err = cred.NonRevocationWitness.Update(pk, update); err != nil {
		return false, err
	}

	return cred.NonRevocationWitness.SignedAccumulator.Accumulator.Index != oldindex, nil
}

// probability returns a float between 0 and asymptote, representing a probability
// that asymptotically increases to the asymptote, reaching
// a reference probability at a reference index.
func probability(lastUpdate time.Time, refindex uint64) float64 {
	const (
		asymptote      = 1.0 / 3          // max probability
		refprobability = 0.75 * asymptote // probability after one week
	)
	f := math.Tan(math.Pi * refprobability / (2 * asymptote))
	i := time.Now().Sub(lastUpdate).Seconds()
	return 2 * asymptote / math.Pi * math.Atan(i/float64(refindex)*f)
}

// randomfloat between 0 and 1
func randomfloat() (float64, error) {
	b := make([]byte, 4)
	_, err := rand.Read(b)
	if err != nil {
		fmt.Println("error:", err)
		return 0, err
	}
	c := float64(binary.BigEndian.Uint32(b)) / float64(^uint32(0)) // random int / max int
	return c, nil
}
<commit_msg>fix: nonrevocation witness age bookkeeping<commit_after>package irmaclient

import (
	"crypto/rand"
	"encoding/binary"

	"fmt"
	"math"
	"sync"
	"time"

	"github.com/getsentry/raven-go"
	"github.com/go-errors/errors"
	"github.com/privacybydesign/gabi/revocation"
	irma "github.com/privacybydesign/irmago"
	"github.com/sirupsen/logrus"
)

func (client *Client) initRevocation() {
	// For every credential supporting revocation, compute nonrevocation caches in async jobs
	for id, attrsets := range client.attributes {
		for i, attrs := range attrsets {
			if attrs.CredentialType() == nil || !attrs.CredentialType().RevocationSupported() {
				continue
			}
			id := id // make copy of same name to capture the value for closure below
			i := i   // see https://golang.org/doc/faq#closures_and_goroutines
			client.jobs <- func() {
				if err := client.nonrevPrepareCache(id, i); err != nil {
					client.reportError(err)
				}
			}
		}
	}

	// Of each credential supporting revocation, we periodically update its nonrevocation witness
	// by fetching updates from the issuer's server, such that:
	// - The time interval between two updates is random so that the server cannot recognize us
	//   using the update interval,
	// - Updating happens regularly even if the app is rarely used.
	// We do this by every 10 seconds updating the credential with a low probability, which
	// increases over time since the last update.
	client.Configuration.Scheduler.Every(irma.RevocationParameters.ClientUpdateInterval).Seconds().Do(func() {
		for id, attrsets := range client.attributes {
			for i, attrs := range attrsets {
				if attrs.CredentialType() == nil || !attrs.CredentialType().RevocationSupported() {
					continue
				}
				cred, err := client.credential(id, i)
				if err != nil {
					client.reportError(err)
					continue
				}
				if cred.NonRevocationWitness == nil {
					continue
				}
				r, err := randomfloat()
				if err != nil {
					client.reportError(err)
					break
				}
				speed := attrs.CredentialType().RevocationUpdateSpeed * 60 * 60
				p := probability(cred.NonRevocationWitness.Updated, speed)
				irma.Logger.Tracef("random update: updated, speed, r, p = %s, %d, %.6f, %.6f",
					cred.NonRevocationWitness.Updated.String(), speed, r, p)
				if r < p {
					irma.Logger.Debugf("scheduling nonrevocation witness remote update for %s-%s", id, attrs.Hash())
					client.jobs <- func() {
						if err = client.nonrevUpdateFromServer(id); err != nil {
							client.reportError(err)
							return
						}
					}
				}
			}
		}
	})
}

// NonrevPrepare updates the revocation state for each credential in the request
// requiring a nonrevocation proof, using the updates included in the request, or the remote
// revocation server if those do not suffice.
func (client *Client) NonrevPrepare(request irma.SessionRequest) error {
	base := request.Base()
	var err error
	var wg sync.WaitGroup
	for id := range request.Disclosure().Identifiers().CredentialTypes {
		credtype := client.Configuration.CredentialTypes[id]
		if !credtype.RevocationSupported() {
			continue
		}
		if !base.RequestsRevocation(id) {
			continue
		}
		irma.Logger.WithField("credtype", id).Debug("updating witnesses")
		wg.Add(1)
		go func() {
			if e := client.nonrevUpdate(id, base.Revocation[id].Updates); e != nil {
				err = e // overwrites err from previously finished call, if any
			}
			wg.Done()
		}()
	}
	wg.Wait()
	return err
}

// nonrevUpdate updates all contained instances of the specified type, using the specified
// updates if present and if they suffice, and contacting the issuer's server to download updates
// otherwise.
func (client *Client) nonrevUpdate(id irma.CredentialTypeIdentifier, updates map[uint]*revocation.Update) error {
	lowest := map[uint]uint64{}
	attrs := client.attrs(id)

	// Per credential and issuer key counter we may posess multiple credential instances.
	// Of the nonrevocation witnesses of these, take the lowest index.
	for i := 0; i < len(attrs); i++ {
		cred, err := client.credential(id, i)
		if err != nil {
			return err
		}
		if cred.NonRevocationWitness == nil {
			continue
		}
		pkid := cred.Pk.Counter
		_, present := lowest[pkid]
		if !present || cred.NonRevocationWitness.SignedAccumulator.Accumulator.Index < lowest[pkid] {
			lowest[pkid] = cred.NonRevocationWitness.SignedAccumulator.Accumulator.Index
		}
	}

	// For each key counter, get an update message starting at the lowest index computed above,
	// that can update all of our credential instance of the given type and key counter,
	// using the specified update messags if they suffice, or the issuer's server otherwise.
	u := map[uint]*revocation.Update{}
	for counter, l := range lowest {
		update := updates[counter]
		if updates != nil && (update == nil || len(update.Events) == 0) {
			return errors.Errorf("missing revocation update for %s-%d", id, counter)
		}
		if update != nil && update.Events[0].Index <= l+1 {
			u[counter] = update
		} else {
			var err error
			u[counter], err = irma.RevocationClient{Conf: client.Configuration}.
				FetchUpdateFrom(id, counter, l+1)
			if err != nil {
				return err
			}
		}
	}

	// Apply the update messages to all instances of the given type and key counter
	for counter, update := range u {
		if err := client.nonrevApplyUpdates(id, counter, update); err != nil {
			return err
		}
	}
	return nil
}

func (client *Client) nonrevApplyUpdates(id irma.CredentialTypeIdentifier, counter uint, update *revocation.Update) error {
	client.credMutex.Lock()
	defer client.credMutex.Unlock()

	attrs := client.attrs(id)
	var save bool
	for i := 0; i < len(attrs); i++ {
		cred, err := client.credential(id, i)
		if err != nil {
			return err
		}
		if cred.NonRevocationWitness == nil || cred.Pk.Counter != counter {
			continue
		}
		updated, err := cred.nonrevApplyUpdates(update, irma.RevocationKeys{Conf: client.Configuration})
		if updated {
			save = true
		}
		if err == revocation.ErrorRevoked {
			id := cred.CredentialType().Identifier()
			hash := cred.attrs.Hash()
			irma.Logger.Warnf("credential %s %s revoked", id, hash)
			attrs[i].Revoked = true
			cred.attrs.Revoked = true
			save = true
			client.handler.Revoked(&irma.CredentialIdentifier{Type: id, Hash: hash})
			// Even if this credential is revoked during a session, we may have
			// other instances that can satisfy the request. So don't return an
			// error which would halt the session.
			continue
		}
		if err != nil {
			return err
		}
		// Asynchroniously update nonrevocation proof cache from updated witness
		irma.Logger.WithField("credtype", id).Debug("scheduling nonrevocation cache update")
		go func(cred *credential) {
			if err := cred.NonrevPrepareCache(); err != nil {
				raven.CaptureError(err, nil)
			}
		}(cred)
	}
	if save {
		if err := client.storage.StoreAttributes(id, client.attributes[id]); err != nil {
			client.reportError(err)
			return err
		}
	}
	return nil
}

func (client *Client) nonrevUpdateFromServer(id irma.CredentialTypeIdentifier) error {
	return client.nonrevUpdate(id, nil)
}

func (client *Client) nonrevPrepareCache(id irma.CredentialTypeIdentifier, index int) error {
	logger := irma.Logger.WithFields(logrus.Fields{"credtype": id, "index": index})
	logger.Debug("preparing cache")
	defer logger.Debug("Preparing cache done")
	cred, err := client.credential(id, index)
	if err != nil {
		return err
	}
	return cred.NonrevPrepareCache()
}

// nonrevRepopulateCaches repopulates the consumed nonrevocation caches of the credentials involved
// in the request, in background jobs, after the request has finished.
func (client *Client) nonrevRepopulateCaches(request irma.SessionRequest) {
	for id := range request.Disclosure().Identifiers().CredentialTypes {
		credtype := client.Configuration.CredentialTypes[id]
		if !credtype.RevocationSupported() {
			continue
		}
		for i := range client.attrs(id) {
			id := id
			i := i
			client.jobs <- func() {
				if err := client.nonrevPrepareCache(id, i); err != nil {
					client.reportError(err)
				}
			}
		}
	}
}

// nonrevApplyUpdates updates the credential's nonrevocation witness using the specified messages,
// if they all verify and if their indices are ahead and adjacent to that of our witness.
func (cred *credential) nonrevApplyUpdates(update *revocation.Update, keys irma.RevocationKeys) (bool, error) {
	t := cred.NonRevocationWitness.SignedAccumulator.Accumulator.Time

	pk, err := keys.PublicKey(cred.CredentialType().IssuerIdentifier(), update.SignedAccumulator.PKCounter)
	if err != nil {
		return false, err
	}
	logger := irma.Logger.WithFields(logrus.Fields{"credtype": cred.CredentialType().Identifier(), "hash": cred.attrs.Hash()})
	logger.Debugf("updating witness")
	defer logger.Debug("updating witness done")
	if err = cred.NonRevocationWitness.Update(pk, update); err != nil {
		return false, err
	}

	return cred.NonRevocationWitness.SignedAccumulator.Accumulator.Time != t, nil
}

// probability returns a float between 0 and asymptote, representing a probability
// that asymptotically increases to the asymptote, reaching
// a reference probability at a reference index.
func probability(lastUpdate time.Time, refindex uint64) float64 {
	const (
		asymptote      = 1.0 / 3          // max probability
		refprobability = 0.75 * asymptote // probability after one week
	)
	f := math.Tan(math.Pi * refprobability / (2 * asymptote))
	i := time.Now().Sub(lastUpdate).Seconds()
	return 2 * asymptote / math.Pi * math.Atan(i/float64(refindex)*f)
}

// randomfloat between 0 and 1
func randomfloat() (float64, error) {
	b := make([]byte, 4)
	_, err := rand.Read(b)
	if err != nil {
		fmt.Println("error:", err)
		return 0, err
	}
	c := float64(binary.BigEndian.Uint32(b)) / float64(^uint32(0)) // random int / max int
	return c, nil
}
<|endoftext|>


<commit_before>package sum

func SubArraySum(nums []int, k int) int {
	// O(n^2)
	n := len(nums)
	if n < 1 {
		return 0
	}
	var ret int
	for i := range nums {
		cur := nums[i]
		if cur == k {
			ret++
		}

		for j := i + 1; j < n; j++ {
			cur += nums[j]
			if cur == k {
				ret++
			}
		}
	}
	return ret
}

func SubArraySum2(nums []int, k int) int {
	var count, sum int
	// record[k]=v means how many sub-arrays sums equals to k
	// preSum for nums [-3,1,2,-3,5] is
	// preSum = [0,-3,-2,0,-3,2]
	// so to get SUM[1,3] which is [1,2,-3] we can use preSum to caculcate which is
	// preSum[4] - preSum[1] = -3 + 3 = 0
	// so if a subarray i to j sums equal to k , which means
	// preSum[j+1] - preSum[i] = k
	record := make(map[int]int)
	record[0] = 1
	for _, n := range nums {
		sum += n
		if v, ok := record[sum-k]; ok {
			count += v
		}
		record[sum]++
	}
	return count
}
<commit_msg>simplify<commit_after>package sum

func SubArraySum(nums []int, k int) int {
	// O(n^2)
	n := len(nums)
	if n < 1 {
		return 0
	}
	var ret int
	for i := range nums {
		// no need to set to nums[i]
		// cur := nums[i]
		// if cur == k {
		// 	ret++
		// }
		cur := 0
		for j := i; j < n; j++ {
			cur += nums[j]
			if cur == k {
				ret++
			}
		}
	}
	return ret
}

func SubArraySum2(nums []int, k int) int {
	var count, sum int
	// record[k]=v means how many sub-arrays sums equals to k
	// preSum for nums [-3,1,2,-3,5] is
	// preSum = [0,-3,-2,0,-3,2]
	// so to get SUM[1,3] which is [1,2,-3] we can use preSum to caculcate which is
	// preSum[4] - preSum[1] = -3 + 3 = 0
	// so if a subarray i to j sums equal to k , which means
	// preSum[j+1] - preSum[i] = k
	record := make(map[int]int)
	record[0] = 1
	for _, n := range nums {
		sum += n
		if v, ok := record[sum-k]; ok {
			count += v
		}
		record[sum]++
	}
	return count
}
<|endoftext|>


<commit_before>package logger

import (
	"fmt"
	"os"
	"runtime"
	"strconv"
	"sync"
	"time"
)

const Version = "0.90.103"

//log level, from low to high, more high means more serious
const (
	TraceLevel = iota
	DebugLevel
	InfoLevel
	WarnLevel
	ErrorLevel
	FatalLevel
)

const (
	Ltime  = 1 << iota //time format "2006/01/02 15:04:05"
	Lfile              //file.go:123
	Llevel             //[Trace|Debug|Info...]
)

var LevelName [6]string = [6]string{"TRACE", "DEBUG", "INFO ", "WARN ", "ERROR", "FATAL"}

const TimeFormat = "15:04:05.000"

const maxBufPoolSize = 16

type Logger struct {
	sync.Mutex

	level int
	flag  int

	handler Handler

	quit chan struct{}
	msg  chan []byte

	bufs [][]byte
}

//new a logger with specified handler and flag
func New(handler Handler, flag int) *Logger {
	var l = new(Logger)

	l.level = InfoLevel
	l.handler = handler

	l.flag = flag

	l.quit = make(chan struct{})

	l.msg = make(chan []byte, 1024)

	l.bufs = make([][]byte, 0, 16)

	go l.run()

	return l
}

//new a default logger with specified handler and flag: Ltime|Lfile|Llevel
func NewLogger(handler Handler) *Logger {
	return New(handler, Ltime|Lfile|Llevel)
}

func newStdHandler() *StreamHandler {
	h, _ := NewStreamHandler(os.Stdout)
	return h
}

var std = NewLogger(newStdHandler())

func (l *Logger) run() {
	for {
		select {
		case msg := <-l.msg:
			l.handler.Write(msg)
			l.putBuf(msg)
		case <-l.quit:
			l.handler.Close()
		}
	}
}

func (l *Logger) popBuf() []byte {
	l.Lock()
	var buf []byte
	if len(l.bufs) == 0 {
		buf = make([]byte, 0, 1024)
	} else {
		buf = l.bufs[len(l.bufs)-1]
		l.bufs = l.bufs[0 : len(l.bufs)-1]
	}
	l.Unlock()

	return buf
}

func (l *Logger) putBuf(buf []byte) {
	l.Lock()
	if len(l.bufs) < maxBufPoolSize {
		buf = buf[0:0]
		l.bufs = append(l.bufs, buf)
	}
	l.Unlock()
}

func (l *Logger) Close() {
	if l.quit == nil {
		return
	}

	close(l.quit)
	l.quit = nil
}

//set log level, any log level less than it will not log
func (l *Logger) SetLevel(level int) {
	l.level = level
}

//a low interface, maybe you can use it for your special log format
//but it may be not exported later......
func (l *Logger) Output(callDepth int, level int, format string, v ...interface{}) {
	if l.level > level {
		return
	}

	buf := l.popBuf()

	if l.flag&Ltime > 0 {
		now := time.Now().Format(TimeFormat)
		// buf = append(buf, '[')
		buf = append(buf, now...)
		buf = append(buf, " "...)
	}

	if l.flag&Llevel > 0 {
		// buf = append(buf, '[')
		buf = append(buf, LevelName[level]...)
		buf = append(buf, " "...)
	}

	if l.flag&Lfile > 0 {
		_, file, line, ok := runtime.Caller(callDepth)
		if !ok {
			file = "???"
			line = 0
		} else {
			for i := len(file) - 1; i > 0; i-- {
				if file[i] == '/' {
					file = file[i+1:]
					break
				}
			}
		}

		buf = append(buf, file...)
		buf = append(buf, ':')

		buf = strconv.AppendInt(buf, int64(line), 10)
		buf = append(buf, ' ')
	}

	s := fmt.Sprintf(format, v...)

	buf = append(buf, s...)

	if s[len(s)-1] != '\n' {
		buf = append(buf, '\n')
	}

	l.msg <- buf
}

//log with Trace level
func (l *Logger) Trace(format string, v ...interface{}) {
	l.Output(2, TraceLevel, format, v...)
}

//log with Debug level
func (l *Logger) Debug(format string, v ...interface{}) {
	l.Output(2, DebugLevel, format, v...)
}

//log with info level
func (l *Logger) Info(format string, v ...interface{}) {
	l.Output(2, InfoLevel, format, v...)
}

//log with warn level
func (l *Logger) Warn(format string, v ...interface{}) {
	l.Output(2, WarnLevel, format, v...)
}

//log with error level
func (l *Logger) Error(format string, v ...interface{}) {
	l.Output(2, ErrorLevel, format, v...)
}

//log with fatal level
func (l *Logger) Fatal(format string, v ...interface{}) {
	l.Output(2, FatalLevel, format, v...)
}

func SetLevel(level int) {
	std.SetLevel(level)
}

func Trace(format string, v ...interface{}) {
	std.Output(2, TraceLevel, format, v...)
}

func Debug(format string, v ...interface{}) {
	std.Output(2, DebugLevel, format, v...)
}

func Info(format string, v ...interface{}) {
	std.Output(2, InfoLevel, format, v...)
}

func Warn(format string, v ...interface{}) {
	std.Output(2, WarnLevel, format, v...)
}

func Error(format string, v ...interface{}) {
	std.Output(2, ErrorLevel, format, v...)
}

func Fatal(format string, v ...interface{}) {
	std.Output(2, FatalLevel, format, v...)
}
<commit_msg>version update<commit_after>package logger

import (
	"fmt"
	"os"
	"runtime"
	"strconv"
	"sync"
	"time"
)

const Version = "0.90.104"

//log level, from low to high, more high means more serious
const (
	TraceLevel = iota
	DebugLevel
	InfoLevel
	WarnLevel
	ErrorLevel
	FatalLevel
)

const (
	Ltime  = 1 << iota //time format "2006/01/02 15:04:05"
	Lfile              //file.go:123
	Llevel             //[Trace|Debug|Info...]
)

var LevelName [6]string = [6]string{"TRACE", "DEBUG", "INFO ", "WARN ", "ERROR", "FATAL"}

const TimeFormat = "15:04:05.000"

const maxBufPoolSize = 16

type Logger struct {
	sync.Mutex

	level int
	flag  int

	handler Handler

	quit chan struct{}
	msg  chan []byte

	bufs [][]byte
}

//new a logger with specified handler and flag
func New(handler Handler, flag int) *Logger {
	var l = new(Logger)

	l.level = InfoLevel
	l.handler = handler

	l.flag = flag

	l.quit = make(chan struct{})

	l.msg = make(chan []byte, 1024)

	l.bufs = make([][]byte, 0, 16)

	go l.run()

	return l
}

//new a default logger with specified handler and flag: Ltime|Lfile|Llevel
func NewLogger(handler Handler) *Logger {
	return New(handler, Ltime|Lfile|Llevel)
}

func newStdHandler() *StreamHandler {
	h, _ := NewStreamHandler(os.Stdout)
	return h
}

var std = NewLogger(newStdHandler())

func (l *Logger) run() {
	for {
		select {
		case msg := <-l.msg:
			l.handler.Write(msg)
			l.putBuf(msg)
		case <-l.quit:
			l.handler.Close()
		}
	}
}

func (l *Logger) popBuf() []byte {
	l.Lock()
	var buf []byte
	if len(l.bufs) == 0 {
		buf = make([]byte, 0, 1024)
	} else {
		buf = l.bufs[len(l.bufs)-1]
		l.bufs = l.bufs[0 : len(l.bufs)-1]
	}
	l.Unlock()

	return buf
}

func (l *Logger) putBuf(buf []byte) {
	l.Lock()
	if len(l.bufs) < maxBufPoolSize {
		buf = buf[0:0]
		l.bufs = append(l.bufs, buf)
	}
	l.Unlock()
}

func (l *Logger) Close() {
	if l.quit == nil {
		return
	}

	close(l.quit)
	l.quit = nil
}

//set log level, any log level less than it will not log
func (l *Logger) SetLevel(level int) {
	l.level = level
}

//a low interface, maybe you can use it for your special log format
//but it may be not exported later......
func (l *Logger) Output(callDepth int, level int, format string, v ...interface{}) {
	if l.level > level {
		return
	}

	buf := l.popBuf()

	if l.flag&Ltime > 0 {
		now := time.Now().Format(TimeFormat)
		// buf = append(buf, '[')
		buf = append(buf, now...)
		buf = append(buf, " "...)
	}

	if l.flag&Llevel > 0 {
		// buf = append(buf, '[')
		buf = append(buf, LevelName[level]...)
		buf = append(buf, " "...)
	}

	if l.flag&Lfile > 0 {
		_, file, line, ok := runtime.Caller(callDepth)
		if !ok {
			file = "???"
			line = 0
		} else {
			for i := len(file) - 1; i > 0; i-- {
				if file[i] == '/' {
					file = file[i+1:]
					break
				}
			}
		}

		buf = append(buf, file...)
		buf = append(buf, ':')

		buf = strconv.AppendInt(buf, int64(line), 10)
		buf = append(buf, ' ')
	}

	s := fmt.Sprintf(format, v...)

	buf = append(buf, s...)

	if s[len(s)-1] != '\n' {
		buf = append(buf, '\n')
	}

	l.msg <- buf
}

//log with Trace level
func (l *Logger) Trace(format string, v ...interface{}) {
	l.Output(2, TraceLevel, format, v...)
}

//log with Debug level
func (l *Logger) Debug(format string, v ...interface{}) {
	l.Output(2, DebugLevel, format, v...)
}

//log with info level
func (l *Logger) Info(format string, v ...interface{}) {
	l.Output(2, InfoLevel, format, v...)
}

//log with warn level
func (l *Logger) Warn(format string, v ...interface{}) {
	l.Output(2, WarnLevel, format, v...)
}

//log with error level
func (l *Logger) Error(format string, v ...interface{}) {
	l.Output(2, ErrorLevel, format, v...)
}

//log with fatal level
func (l *Logger) Fatal(format string, v ...interface{}) {
	l.Output(2, FatalLevel, format, v...)
}

func SetLevel(level int) {
	std.SetLevel(level)
}

func Trace(format string, v ...interface{}) {
	std.Output(2, TraceLevel, format, v...)
}

func Debug(format string, v ...interface{}) {
	std.Output(2, DebugLevel, format, v...)
}

func Info(format string, v ...interface{}) {
	std.Output(2, InfoLevel, format, v...)
}

func Warn(format string, v ...interface{}) {
	std.Output(2, WarnLevel, format, v...)
}

func Error(format string, v ...interface{}) {
	std.Output(2, ErrorLevel, format, v...)
}

func Fatal(format string, v ...interface{}) {
	std.Output(2, FatalLevel, format, v...)
}
<|endoftext|>


<commit_before>// Copyright (c) 2016-2017 Eric Barkie. All rights reserved.
// Use of this source code is governed by the MIT license
// that can be found in the LICENSE file.

package main

import "sync"

// loopBuffer is a circular buffer that holds up to loopsMax of
// historical loops.  It's designed to be efficient at adding
// new items since it happens very frequently.
type loopBuffer struct {
	buf [loopsMax]Loop
	cur int
	len int
	sync.RWMutex
}

// Add adds a wrapped loop packet to the loop buffer.  Once the
// buffer is full each new packet overwrites the oldest.
func (lb *loopBuffer) Add(l Loop) {
	lb.Lock()
	defer lb.Unlock()

	lb.cur = (lb.cur + 1) % loopsMax
	lb.buf[lb.cur] = l
	if lb.len < loopsMax {
		lb.len++
	}
}

// Last returns the number of items in the loop buffer and the
// last one added.
func (lb *loopBuffer) Last() (n int, l Loop) {
	// This is a bit ugly to read but it's about 30% faster
	// without defer and multi-assignment.
	lb.RLock()
	n = lb.len
	l = lb.buf[lb.cur]
	lb.RUnlock()
	return
}

// Loops returns the loop buffer as a slice.
func (lb *loopBuffer) Loops() []Loop {
	lb.RLock()
	defer lb.RUnlock()

	ls := make([]Loop, lb.len)
	j := lb.cur
	for i := 0; i < lb.len; i++ {
		ls[i] = lb.buf[j]
		j = (j - 1 + loopsMax) % loopsMax
	}

	return ls
}
<commit_msg>Comments<commit_after>// Copyright (c) 2016-2017 Eric Barkie. All rights reserved.
// Use of this source code is governed by the MIT license
// that can be found in the LICENSE file.

package main

import "sync"

// loopBuffer is a circular buffer that holds up to loopsMax of
// historical loops.  It's designed to be efficient at adding
// new items since it happens very frequently.
type loopBuffer struct {
	buf [loopsMax]Loop
	cur int
	len int
	sync.RWMutex
}

// Add adds a Loop packet to the loop buffer.  Once the
// buffer is full each new packet overwrites the oldest.
func (lb *loopBuffer) Add(l Loop) {
	lb.Lock()
	defer lb.Unlock()

	lb.cur = (lb.cur + 1) % loopsMax
	lb.buf[lb.cur] = l
	if lb.len < loopsMax {
		lb.len++
	}
}

// Last returns the number of items in the loop buffer and the
// last one added.
func (lb *loopBuffer) Last() (n int, l Loop) {
	// This is a bit ugly to read but it's about 30% faster
	// without defer and multi-assignment.
	lb.RLock()
	n = lb.len
	l = lb.buf[lb.cur]
	lb.RUnlock()
	return
}

// Loops returns the loop buffer as a slice.
func (lb *loopBuffer) Loops() []Loop {
	lb.RLock()
	defer lb.RUnlock()

	ls := make([]Loop, lb.len)
	j := lb.cur
	for i := 0; i < lb.len; i++ {
		ls[i] = lb.buf[j]
		j = (j - 1 + loopsMax) % loopsMax
	}

	return ls
}
<|endoftext|>


<commit_before>package response

import (
	"time"
)

type BaseResponse struct {
	Command   string
	Timestamp int64
}

func (r *BaseResponse) MakeATimestamp() {
	r.Timestamp = time.Now().UnixNano() / 1e6
}
<commit_msg>Add response.Send() method<commit_after>package response

import (
	"encoding/json"
	"time"
)

type BaseResponse struct {
	Command   string
	Timestamp int64
}

func (r *BaseResponse) MakeATimestamp() {
	r.Timestamp = time.Now().UnixNano() / 1e6
}

func (r *BaseResponse) Send(sender func([]byte) error {
	r.MakeATimestamp()
	serialized, err := json.Marshal(r)
	if err == nil {
		sender(serialized)
	}
	return err
}
<|endoftext|>


<commit_before>package cable

import (
	"bufio"
	"errors"
	"log"
	"net"
	"regexp"
	"strconv"
	"strings"
)

// CRLF constant represents the End-of-Line sequence.
const CRLF = "\r\n"

// A FTP contain options to handle the FTP connection.
type FTP struct {
	debug    bool
	addr     string
	conn     net.Conn
	isLogged bool
	passPort int

	reader *bufio.Reader
	writer *bufio.Writer
}

type message struct {
	code    int
	message string
}

// New returns a FTP struct initializated to default values.
func New() *FTP {
	return &FTP{}
}

// Debug set the level of verbosity.
func (ftp *FTP) Debug(debug bool) {
	ftp.debug = debug
}

// Connect perform the dial to the ftp server.
func (ftp *FTP) Connect(addr string) (err error) {
	if ftp.conn, err = net.Dial("tcp", addr); err != nil {
		return err
	}

	ftp.addr = addr
	ftp.writer = bufio.NewWriter(ftp.conn)
	ftp.reader = bufio.NewReader(ftp.conn)

	_, err = ftp.receive()

	if err != nil {
		return err
	}

	return nil
}

// Close closes the current connection, if exists any.
func (ftp *FTP) Close() {
	// We should ensure the connection was established before try to close it.
	if ftp.conn != nil {
		ftp.conn.Close()
		ftp.conn = nil
	}
}

// Login command to FTP service, this methos is sequence of command to perform
// the login operation.
func (ftp *FTP) Login(user string, password string) (err error) {
	if err = ftp.user(user); err != nil {
		return err
	}

	if err = ftp.pass(password); err != nil {
		return err
	}

	ftp.isLogged = true

	return nil
}

// Quit terminates the connection to the FTP server.
func (ftp *FTP) Quit() (err error) {
	defer ftp.Close()

	if err = ftp.send("QUIT"); err != nil {
		return err
	}

	_, err = ftp.receive()

	if err != nil {
		return err
	}

	return nil
}

// User command send the user's name to the ftp server.
// This command belons to the Access Control Command set of commands.
func (ftp *FTP) user(user string) (err error) {
	if len(user) == 0 {
		user = "anonymous"
	}

	if err := ftp.send("USER " + user); err != nil {
		return err
	}

	_, err = ftp.receive()

	if err != nil {
		return err
	}

	return nil
}

// Password command send the user's password to the ftp server.
// This command belons to the Access Control Command set of commands.
func (ftp *FTP) pass(password string) (err error) {
	if len(password) == 0 {
		password = ""
	}

	if err = ftp.send("PASS " + password); err != nil {
		return err
	}

	_, err = ftp.receive()

	if err != nil {
		return err
	}

	return nil
}

// Pasv command requests the server-DTP to "listen" on data port ant wait for a
// connection rather than iniciate one upon receipt of a transfer command.
func (ftp *FTP) Pasv() (err error) {
	if err = ftp.send("PASV"); err != nil {
		return err
	}

	// The response to this command includes the host and port address this
	// server is listening on.
	var msg *message
	msg, err = ftp.receive()

	if err != nil {
		return err
	}

	re, err := regexp.Compile(`\((.*)\)`)
	res := re.FindAllStringSubmatch(msg.message, -1)

	s := strings.Split(res[0][1], ",")

	l1, _ := strconv.Atoi(s[len(s)-2])
	l2, _ := strconv.Atoi(s[len(s)-1])

	ftp.passPort = l1<<8 + l2

	return nil
}

// Pwd comand causes the name of the current working directory to be returned
// in the replay.
func (ftp *FTP) Pwd() (err error) {
	if err = ftp.send("PWD"); err != nil {
		return err
	}

	_, err = ftp.receive()

	if err != nil {
		return err
	}

	return nil
}

// Cwd command allows the user to work with a different directory or dataset
// for file storage or retrieval without altering his login or accounting
// information.
func (ftp *FTP) Cwd(pathname string) (err error) {
	if err = ftp.send("CWD " + pathname + CRLF); err != nil {
		return err
	}

	_, err = ftp.receive()

	if err != nil {
		return err
	}

	return nil
}

func (ftp *FTP) send(cmd string) (err error) {
	// We need to ensure that we have a connection open before send the command.
	if ftp.writer == nil {
		return errors.New("No Connection open.")
	}

	if _, err = ftp.writer.WriteString(cmd + CRLF); err != nil {
		return err
	}

	ftp.writer.Flush()

	return nil
}

func (ftp *FTP) receive() (*message, error) {
	var reply string
	var err error

	if reply, err = ftp.reader.ReadString('\n'); err != nil {
		return &message{}, err
	}

	code, err := strconv.Atoi(reply[:3])
	msg := reply[4:]

	if ftp.debug {
		log.Printf("Code: %d Message: %s", code, msg)
	}

	return &message{code: code, message: msg}, nil
}
<commit_msg>:100: alpha release 0.0.1<commit_after>package cable

import (
	"bufio"
	"errors"
	"log"
	"net"
	"regexp"
	"strconv"
	"strings"
)

// VERSION constant represents the current Cable version in
// "major.minor.release" notation.
const VERSION = "0.0.1"

// CRLF constant represents the End-of-Line sequence.
const CRLF = "\r\n"

// A FTP contain options to handle the FTP connection.
type FTP struct {
	debug    bool
	addr     string
	conn     net.Conn
	isLogged bool
	passPort int

	reader *bufio.Reader
	writer *bufio.Writer
}

type message struct {
	code    int
	message string
}

// New returns a FTP struct initializated to default values.
func New() *FTP {
	return &FTP{}
}

// Debug set the level of verbosity.
func (ftp *FTP) Debug(debug bool) {
	ftp.debug = debug
}

// Connect perform the dial to the ftp server.
func (ftp *FTP) Connect(addr string) (err error) {
	if ftp.conn, err = net.Dial("tcp", addr); err != nil {
		return err
	}

	ftp.addr = addr
	ftp.writer = bufio.NewWriter(ftp.conn)
	ftp.reader = bufio.NewReader(ftp.conn)

	_, err = ftp.receive()

	if err != nil {
		return err
	}

	return nil
}

// Close closes the current connection, if exists any.
func (ftp *FTP) Close() {
	// We should ensure the connection was established before try to close it.
	if ftp.conn != nil {
		ftp.conn.Close()
		ftp.conn = nil
	}
}

// Login command to FTP service, this methos is sequence of command to perform
// the login operation.
func (ftp *FTP) Login(user string, password string) (err error) {
	if err = ftp.user(user); err != nil {
		return err
	}

	if err = ftp.pass(password); err != nil {
		return err
	}

	ftp.isLogged = true

	return nil
}

// Quit terminates the connection to the FTP server.
func (ftp *FTP) Quit() (err error) {
	defer ftp.Close()

	if err = ftp.send("QUIT"); err != nil {
		return err
	}

	_, err = ftp.receive()

	if err != nil {
		return err
	}

	return nil
}

// User command send the user's name to the ftp server.
// This command belons to the Access Control Command set of commands.
func (ftp *FTP) user(user string) (err error) {
	if len(user) == 0 {
		user = "anonymous"
	}

	if err := ftp.send("USER " + user); err != nil {
		return err
	}

	_, err = ftp.receive()

	if err != nil {
		return err
	}

	return nil
}

// Password command send the user's password to the ftp server.
// This command belons to the Access Control Command set of commands.
func (ftp *FTP) pass(password string) (err error) {
	if len(password) == 0 {
		password = ""
	}

	if err = ftp.send("PASS " + password); err != nil {
		return err
	}

	_, err = ftp.receive()

	if err != nil {
		return err
	}

	return nil
}

// Pasv command requests the server-DTP to "listen" on data port ant wait for a
// connection rather than iniciate one upon receipt of a transfer command.
func (ftp *FTP) Pasv() (err error) {
	if err = ftp.send("PASV"); err != nil {
		return err
	}

	// The response to this command includes the host and port address this
	// server is listening on.
	var msg *message
	msg, err = ftp.receive()

	if err != nil {
		return err
	}

	re, err := regexp.Compile(`\((.*)\)`)
	res := re.FindAllStringSubmatch(msg.message, -1)

	s := strings.Split(res[0][1], ",")

	l1, _ := strconv.Atoi(s[len(s)-2])
	l2, _ := strconv.Atoi(s[len(s)-1])

	ftp.passPort = l1<<8 + l2

	return nil
}

// Pwd comand causes the name of the current working directory to be returned
// in the replay.
func (ftp *FTP) Pwd() (err error) {
	if err = ftp.send("PWD"); err != nil {
		return err
	}

	_, err = ftp.receive()

	if err != nil {
		return err
	}

	return nil
}

// Cwd command allows the user to work with a different directory or dataset
// for file storage or retrieval without altering his login or accounting
// information.
func (ftp *FTP) Cwd(pathname string) (err error) {
	if err = ftp.send("CWD " + pathname + CRLF); err != nil {
		return err
	}

	_, err = ftp.receive()

	if err != nil {
		return err
	}

	return nil
}

func (ftp *FTP) send(cmd string) (err error) {
	// We need to ensure that we have a connection open before send the command.
	if ftp.writer == nil {
		return errors.New("No Connection open.")
	}

	if _, err = ftp.writer.WriteString(cmd + CRLF); err != nil {
		return err
	}

	ftp.writer.Flush()

	return nil
}

func (ftp *FTP) receive() (*message, error) {
	var reply string
	var err error

	if reply, err = ftp.reader.ReadString('\n'); err != nil {
		return &message{}, err
	}

	code, err := strconv.Atoi(reply[:3])
	msg := reply[4:]

	if ftp.debug {
		log.Printf("Code: %d Message: %s", code, msg)
	}

	return &message{code: code, message: msg}, nil
}
<|endoftext|>


<commit_before>package cache

import (
	"log"
	"sync"
	"time"
)

const (
	kDefaultCleanInterval = 30 * time.Second
	kMaxCleanupDuration   = 50 // in milliseconds
)

type Cache struct {
	sync.RWMutex
	items map[string][]byte
	exps  map[string]int64 // in milliseconds
}

func (c *Cache) Set(key string, value []byte, expirations ...time.Duration) {
	expiration := expireSeconds(expirations)
	if expiration == 0 || value == nil {
		c.Del(key)
		return
	}

	c.Lock()
	c.items[key] = value
	if expiration > 0 {
		c.exps[key] = expiration
	}
	c.Unlock()
}

func (c *Cache) Get(key string) ([]byte, bool) {
	now := time.Now().UnixNano() / 1e6
	var (
		mayExpire bool   = false
		result    []byte = nil
		getit     bool   = false
	)

	c.RLock()
	if exp, ok := c.exps[key]; ok && exp <= now {
		mayExpire = true
	} else {
		result, getit = c.items[key]
	}
	c.RUnlock()

	if mayExpire {
		c.Lock()
		if exp, ok := c.exps[key]; ok && exp <= now {
			delete(c.items, key)
			delete(c.exps, key)
			if IsDebug {
				log.Printf("Lazy clean up for key=%q", key)
			}
		}
		c.Unlock()
	}

	return result, getit
}

func (c *Cache) Del(key string) {
	c.Lock()
	defer c.Unlock()
	delete(c.items, key)
	delete(c.exps, key)
}

func (c *Cache) Renew(key string, expirations ...time.Duration) {
	expiration := expireSeconds(expirations)
	if expiration == 0 {
		c.Del(key)
		return
	}

	c.Lock()
	defer c.Unlock()
	if _, ok := c.items[key]; ok {
		if expiration > 0 {
			c.exps[key] = expiration
		} else {
			delete(c.exps, key)
		}
	}
}

func expireSeconds(expirations []time.Duration) int64 {
	// all in milliseconds
	var expiration int64 = -1
	if len(expirations) > 0 {
		expiration = int64(expirations[0]) / 1e6
	}
	if expiration == 0 {
		return 0
	} else if expiration > 0 {
		expiration = expiration + time.Now().UnixNano()/1e6
	} else {
		expiration = -1
	}
	return expiration
}

// This will be running inside a goroutine
func (c *Cache) initCleanup(interval time.Duration) {
	cTick := time.Tick(interval)
	for tick := range cTick {
		now := tick.UnixNano() / 1e6
		expireKeys, counter := 0, 0
		c.Lock()
		if len(c.exps) > 0 {
			// since the go map is unordered, the key would be randomized which is great
			for key, expire := range c.exps {
				if expire <= now {
					delete(c.items, key)
					delete(c.exps, key)
					expireKeys++
				}
				counter++
				if counter >= 10000 {
					lasting := time.Now().UnixNano()/1e6 - now
					if lasting > kMaxCleanupDuration {
						break
					}
					counter = 0
				}
			}
		}
		if IsDebug {
			log.Printf("Cache cleanup, cleaned=%d, size=%d/%d, duration=%s",
				expireKeys, len(c.items), len(c.exps), time.Since(tick))
		}
		c.Unlock()
	}
}

func New(cleanInterval ...time.Duration) *Cache {
	ci := kDefaultCleanInterval
	if len(cleanInterval) > 0 && cleanInterval[0] >= 10*time.Second {
		ci = cleanInterval[0]
	}

	cache := &Cache{
		items: make(map[string][]byte),
		exps:  make(map[string]int64),
	}
	go func() {
		cache.initCleanup(ci)
	}()
	return cache
}

var IsDebug bool = false
<commit_msg>Fix the debug condition<commit_after>package cache

import (
	"log"
	"sync"
	"time"
)

const (
	kDefaultCleanInterval = 30 * time.Second
	kMaxCleanupDuration   = 50 // in milliseconds
)

type Cache struct {
	sync.RWMutex
	items map[string][]byte
	exps  map[string]int64 // in milliseconds
}

func (c *Cache) Set(key string, value []byte, expirations ...time.Duration) {
	expiration := expireSeconds(expirations)
	if expiration == 0 || value == nil {
		c.Del(key)
		return
	}

	c.Lock()
	c.items[key] = value
	if expiration > 0 {
		c.exps[key] = expiration
	}
	c.Unlock()
}

func (c *Cache) Get(key string) ([]byte, bool) {
	now := time.Now().UnixNano() / 1e6
	var (
		mayExpire bool   = false
		result    []byte = nil
		getit     bool   = false
	)

	c.RLock()
	if exp, ok := c.exps[key]; ok && exp <= now {
		mayExpire = true
	} else {
		result, getit = c.items[key]
	}
	c.RUnlock()

	if mayExpire {
		c.Lock()
		if exp, ok := c.exps[key]; ok && exp <= now {
			delete(c.items, key)
			delete(c.exps, key)
			if IsDebug {
				log.Printf("Lazy clean up for key=%q", key)
			}
		}
		c.Unlock()
	}

	return result, getit
}

func (c *Cache) Del(key string) {
	c.Lock()
	defer c.Unlock()
	delete(c.items, key)
	delete(c.exps, key)
}

func (c *Cache) Renew(key string, expirations ...time.Duration) {
	expiration := expireSeconds(expirations)
	if expiration == 0 {
		c.Del(key)
		return
	}

	c.Lock()
	defer c.Unlock()
	if _, ok := c.items[key]; ok {
		if expiration > 0 {
			c.exps[key] = expiration
		} else {
			delete(c.exps, key)
		}
	}
}

func expireSeconds(expirations []time.Duration) int64 {
	// all in milliseconds
	var expiration int64 = -1
	if len(expirations) > 0 {
		expiration = int64(expirations[0]) / 1e6
	}
	if expiration == 0 {
		return 0
	} else if expiration > 0 {
		expiration = expiration + time.Now().UnixNano()/1e6
	} else {
		expiration = -1
	}
	return expiration
}

// This will be running inside a goroutine
func (c *Cache) initCleanup(interval time.Duration) {
	cTick := time.Tick(interval)
	for tick := range cTick {
		now := tick.UnixNano() / 1e6
		expireKeys, counter := 0, 0
		c.Lock()
		if len(c.exps) > 0 {
			// since the go map is unordered, the key would be randomized which is great
			for key, expire := range c.exps {
				if expire <= now {
					delete(c.items, key)
					delete(c.exps, key)
					expireKeys++
				}
				counter++
				if counter >= 10000 {
					lasting := time.Now().UnixNano()/1e6 - now
					if lasting > kMaxCleanupDuration {
						break
					}
					counter = 0
				}
			}
			if IsDebug && expireKeys > 0 {
				log.Printf("Cache cleanup, cleaned=%d, size=%d/%d, duration=%s",
					expireKeys, len(c.items), len(c.exps), time.Since(tick))
			}
		}
		c.Unlock()
	}
}

func New(cleanInterval ...time.Duration) *Cache {
	ci := kDefaultCleanInterval
	if len(cleanInterval) > 0 && cleanInterval[0] >= 10*time.Second {
		ci = cleanInterval[0]
	}

	cache := &Cache{
		items: make(map[string][]byte),
		exps:  make(map[string]int64),
	}
	go func() {
		cache.initCleanup(ci)
	}()
	return cache
}

var IsDebug bool = false
<|endoftext|>


<commit_before>package cache

import (
	"container/list"
	"context"
	"errors"
	"sync"
	"sync/atomic"
	"time"
)

const DefaultMaxItems = 1000
const DefaultExpireInterval = time.Second

type Node struct {
	Key      interface{}
	Data     interface{}
	Exp      time.Time
	Requests int64
	Element  *list.Element
}

type Upstream interface {
	Fetch(interface{}) (interface{}, error)
}
type UpstreamFunc func(interface{}) (interface{}, error)

func (f UpstreamFunc) Fetch(x interface{}) (interface{}, error) {
	return f(x)
}

type Cache struct {
	Namespace string
	Upstream  Upstream
	MaxAge    time.Duration
	MaxItems  int

	nodes map[interface{}]*Node
	lst   *list.List
	pool  *sync.Pool

	requests  chan interface{}
	responses chan *Node
	add       chan *Node

	stats struct {
		hit      uint64
		miss     uint64
		maxage   uint64
		maxitems uint64
	}
}

func (c *Cache) evict(n *Node) {
	delete(c.nodes, n.Key)
	c.lst.Remove(n.Element)
	c.pool.Put(n)
}

func (c *Cache) Stats() (uint64, uint64, uint64, uint64) {
	return atomic.LoadUint64(&c.stats.hit), atomic.LoadUint64(&c.stats.miss),
		atomic.LoadUint64(&c.stats.maxage), atomic.LoadUint64(&c.stats.maxitems)
}

func (c *Cache) RunInContext(ctx context.Context) (context.Context, error) {
	if nil != c.requests {
		return ctx, errors.New("Cache already running")
	}
	c.add = make(chan *Node, 2*(c.MaxItems+1))
	c.requests = make(chan interface{}, 2*(c.MaxItems+1))
	c.responses = make(chan *Node, 2*(c.MaxItems+1))
	c.nodes = make(map[interface{}]*Node)
	c.lst = list.New()

	c.pool = &sync.Pool{
		New: func() interface{} {
			return &Node{}
		},
	}

	evict := make(chan *Node, 2*(c.MaxItems+1))

	go func() {
		defer close(evict)
		defer close(c.add)
		defer close(c.requests)
		defer close(c.responses)

		for {
			select {
			case <-ctx.Done():
				return
			case n := <-evict:
				c.evict(n)
			case n := <-c.add:
				if c.MaxItems > 0 && len(c.nodes) > c.MaxItems {
					if el := c.lst.Back(); el != nil {
						if e, ok := el.Value.(*Node); ok {
							c.stats.maxitems++
							evict <- e
						}
					}
				}
				c.nodes[n.Key] = n
				n.Element = c.lst.PushFront(n)
			case x := <-c.requests:
				if n := c.nodes[x]; n != nil {
					if c.MaxAge > 0 && n.Exp.Before(time.Now()) {
						c.responses <- nil
						c.stats.maxage++
						evict <- n
					} else {
						if el := n.Element; el != nil {
							c.lst.MoveToFront(el)
						}
						n.Requests++
						c.stats.hit++
						c.responses <- n
					}
				} else {
					c.stats.miss++
					c.responses <- nil

				}
			}
		}
	}()
	return ctx, nil
}

func (c *Cache) Get(x interface{}) (interface{}, bool, error) {
	c.requests <- x
	n := <-c.responses
	if n == nil {
		y, err := c.fetch(x)
		return y, false, err
	} else {
		return n.Data, true, nil
	}
}

func (c *Cache) Fetch(x interface{}) (interface{}, error) {
	y, _, err := c.Get(x)
	return y, err
}

var zerotime = time.Time{}

func (c *Cache) node(x interface{}, y interface{}) *Node {
	n, _ := c.pool.Get().(*Node)
	if c.MaxAge > 0 {
		n.Exp = time.Now().Add(c.MaxAge)
	} else {
		n.Exp = zerotime
	}
	n.Key = x
	n.Data = y
	n.Requests = 0
	return n
}

var (
	NoUpstreamError = errors.New("No Upstream assigned to cache")
)

func (c *Cache) fetch(x interface{}) (interface{}, error) {
	if nil == c.Upstream {
		return nil, NoUpstreamError
	}
	rc := make(chan interface{}, 1)
	ec := make(chan error, 1)
	go func() {
		y, err := c.Upstream.Fetch(x)
		if err == nil {
			c.add <- c.node(x, y)
		}
		rc <- y
		ec <- err
	}()
	return <-rc, <-ec
}
<commit_msg>Change Run signature<commit_after>package cache

import (
	"container/list"
	"context"
	"errors"
	"sync"
	"sync/atomic"
	"time"
)

const DefaultMaxItems = 1000
const DefaultExpireInterval = time.Second

type Node struct {
	Key      interface{}
	Data     interface{}
	Exp      time.Time
	Requests int64
	Element  *list.Element
}

type Upstream interface {
	Fetch(interface{}) (interface{}, error)
}
type UpstreamFunc func(interface{}) (interface{}, error)

func (f UpstreamFunc) Fetch(x interface{}) (interface{}, error) {
	return f(x)
}

type Cache struct {
	Namespace string
	Upstream  Upstream
	MaxAge    time.Duration
	MaxItems  int

	nodes map[interface{}]*Node
	lst   *list.List
	pool  *sync.Pool

	requests  chan interface{}
	responses chan *Node
	add       chan *Node

	stats struct {
		hit      uint64
		miss     uint64
		maxage   uint64
		maxitems uint64
	}
}

func (c *Cache) evict(n *Node) {
	delete(c.nodes, n.Key)
	c.lst.Remove(n.Element)
	c.pool.Put(n)
}

func (c *Cache) Stats() (uint64, uint64, uint64, uint64) {
	return atomic.LoadUint64(&c.stats.hit), atomic.LoadUint64(&c.stats.miss),
		atomic.LoadUint64(&c.stats.maxage), atomic.LoadUint64(&c.stats.maxitems)
}

func (c *Cache) Run(ctx context.Context) error {
	if nil != c.requests {
		return errors.New("Cache already running")
	}
	if nil == ctx {
		ctx = context.Background()
	}
	c.add = make(chan *Node, 2*(c.MaxItems+1))
	c.requests = make(chan interface{}, 2*(c.MaxItems+1))
	c.responses = make(chan *Node, 2*(c.MaxItems+1))
	c.nodes = make(map[interface{}]*Node)
	c.lst = list.New()

	c.pool = &sync.Pool{
		New: func() interface{} {
			return &Node{}
		},
	}

	evict := make(chan *Node, 2*(c.MaxItems+1))

	go func() {
		defer close(evict)
		defer close(c.add)
		defer close(c.requests)
		defer close(c.responses)

		for {
			select {
			case <-ctx.Done():
				return
			case n := <-evict:
				c.evict(n)
			case n := <-c.add:
				if c.MaxItems > 0 && len(c.nodes) > c.MaxItems {
					if el := c.lst.Back(); el != nil {
						if e, ok := el.Value.(*Node); ok {
							c.stats.maxitems++
							evict <- e
						}
					}
				}
				c.nodes[n.Key] = n
				n.Element = c.lst.PushFront(n)
			case x := <-c.requests:
				if n := c.nodes[x]; n != nil {
					if c.MaxAge > 0 && n.Exp.Before(time.Now()) {
						c.responses <- nil
						c.stats.maxage++
						evict <- n
					} else {
						if el := n.Element; el != nil {
							c.lst.MoveToFront(el)
						}
						n.Requests++
						c.stats.hit++
						c.responses <- n
					}
				} else {
					c.stats.miss++
					c.responses <- nil

				}
			}
		}
	}()
	return nil
}

func (c *Cache) Get(x interface{}) (interface{}, bool, error) {
	c.requests <- x
	n := <-c.responses
	if n == nil {
		y, err := c.fetch(x)
		return y, false, err
	} else {
		return n.Data, true, nil
	}
}

func (c *Cache) Fetch(x interface{}) (interface{}, error) {
	y, _, err := c.Get(x)
	return y, err
}

var zerotime = time.Time{}

func (c *Cache) node(x interface{}, y interface{}) *Node {
	n, _ := c.pool.Get().(*Node)
	if c.MaxAge > 0 {
		n.Exp = time.Now().Add(c.MaxAge)
	} else {
		n.Exp = zerotime
	}
	n.Key = x
	n.Data = y
	n.Requests = 0
	return n
}

var (
	NoUpstreamError = errors.New("No Upstream assigned to cache")
)

func (c *Cache) fetch(x interface{}) (interface{}, error) {
	if nil == c.Upstream {
		return nil, NoUpstreamError
	}
	rc := make(chan interface{}, 1)
	ec := make(chan error, 1)
	go func() {
		y, err := c.Upstream.Fetch(x)
		if err == nil {
			c.add <- c.node(x, y)
		}
		rc <- y
		ec <- err
	}()
	return <-rc, <-ec
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"log"

	"github.com/juntaki/transparent"
)

func (w *Wikidata) initializeMarkdownCache() error {
	getfunc := func(key interface{}) (interface{}, error) {
		titleHash, _ := key.(string)
		log.Println("Get from source", titleHash)
		page, err := w.loadMarkdown(titleHash, "")
		if err != nil {
			return nil, err
		}
		return page, nil
	}

	addfunc := func(key interface{}, value interface{}) (err error) {
		pageData, _ := value.(*pageData)
		log.Println("Set source", key)
		err = w.saveMarkdown(pageData)
		return err
	}

	removefunc := func(key interface{}) (err error) {
		titleHash, _ := key.(string)
		err = w.deleteMarkdown(titleHash)
		return err
	}
	storage, err := transparent.NewCustomStorage(getfunc, addfunc, removefunc)
	if err != nil {
		return err
	}
	source, err := transparent.NewSource(storage)
	if err != nil {
		return err
	}
	w.pageCache = transparent.NewLRUCache(10, 10)
	transparent.Stack(w.pageCache, source)
	return nil
}

func (w *Wikidata) loadMarkdownAsync(titleHash string) (*pageData, error) {
	value, err := w.pageCache.Get(titleHash)
	if err != nil {
		return nil, err
	}
	pagedata, ok := value.(*pageData)
	if !ok {
		return nil, errors.New("Failed to get page data")
	}
	return pagedata, nil
}

func (w *Wikidata) saveMarkdownAsync(titleHash string, pageData *pageData) error {
	return w.pageCache.Set(titleHash, pageData)
}

func (w *Wikidata) deleteMarkdownAsync(titleHash string) error {
	return w.pageCache.Remove(titleHash)
}

// for UserData
func (w *Wikidata) initializeUserCache() error {
	getfunc := func(key interface{}) (interface{}, error) {
		name, _ := key.(string)
		log.Println("Get from source", name)
		user, err := w.loadUser(name)
		if err != nil {
			return nil, err
		}
		return user, nil
	}

	addfunc := func(key interface{}, value interface{}) (err error) {
		userData, _ := value.(*userData)
		log.Println("Set source", key)
		err = w.saveUser(userData)
		return err
	}

	removefunc := func(key interface{}) (err error) {
		name, _ := key.(string)
		err = w.deleteUser(name)
		return err
	}
	storage, err := transparent.NewCustomStorage(getfunc, addfunc, removefunc)
	if err != nil {
		return err
	}
	source, err := transparent.NewSource(storage)
	if err != nil {
		return err
	}
	w.userCache = transparent.NewLRUCache(10, 10)
	transparent.Stack(w.userCache, source)
	return nil
}

func (w *Wikidata) loadUserAsync(name string) (*userData, error) {
	value, err := w.userCache.Get(name)
	if err != nil {
		return nil, err
	}
	userdata, ok := value.(*userData)
	if !ok {
		return nil, errors.New("Failed to get user data")
	}
	return userdata, nil
}

func (w *Wikidata) saveUserAsync(name string, userData *userData) error {
	return w.userCache.Set(name, userData)
}

func (w *Wikidata) deleteUserAsync(name string) error {
	return w.userCache.Remove(name)
}

// for File
func (w *Wikidata) initializeFileCache() error {
	getfunc := func(key interface{}) (interface{}, error) {
		fileDataKey, _ := key.(fileDataKey)
		log.Println("Get from source", fileDataKey)
		file, err := w.loadFile(&fileDataKey)
		if err != nil {
			return nil, err
		}
		return file, nil
	}

	addfunc := func(key interface{}, value interface{}) (err error) {
		fileData, _ := value.(*fileData)
		log.Println("Set source", key)
		err = w.saveFile(fileData)
		return err
	}

	removefunc := func(key interface{}) (err error) {
		fileDataKey, _ := key.(fileDataKey)
		err = w.deleteFile(fileDataKey)
		return err
	}
	storage, err := transparent.NewCustomStorage(getfunc, addfunc, removefunc)
	if err != nil {
		return err
	}
	source, err := transparent.NewSource(storage)
	if err != nil {
		return err
	}
	w.fileCache = transparent.NewLRUCache(10, 10)
	transparent.Stack(w.fileCache, source)
	return nil
}

func (w *Wikidata) loadFileAsync(key fileDataKey) (*fileData, error) {
	value, err := w.fileCache.Get(key)
	if err != nil {
		return nil, err
	}
	filedata, ok := value.(*fileData)
	if !ok {
		return nil, errors.New("Failed to get file data")
	}
	return filedata, nil
}

func (w *Wikidata) saveFileAsync(key fileDataKey, fileData *fileData) error {
	return w.fileCache.Set(key, fileData)
}

func (w *Wikidata) deleteFileAsync(key fileDataKey) error {
	return w.fileCache.Remove(key)
}
<commit_msg>Fix File upload<commit_after>package main

import (
	"errors"
	"log"

	"github.com/juntaki/transparent"
)

func (w *Wikidata) initializeMarkdownCache() error {
	getfunc := func(key interface{}) (interface{}, error) {
		titleHash, _ := key.(string)
		log.Println("Get from source", titleHash)
		page, err := w.loadMarkdown(titleHash, "")
		if err != nil {
			return nil, err
		}
		return page, nil
	}

	addfunc := func(key interface{}, value interface{}) (err error) {
		pageData, _ := value.(*pageData)
		log.Println("Set source", key)
		err = w.saveMarkdown(pageData)
		return err
	}

	removefunc := func(key interface{}) (err error) {
		titleHash, _ := key.(string)
		err = w.deleteMarkdown(titleHash)
		return err
	}
	storage, err := transparent.NewCustomStorage(getfunc, addfunc, removefunc)
	if err != nil {
		return err
	}
	source, err := transparent.NewSource(storage)
	if err != nil {
		return err
	}
	w.pageCache, _ = transparent.NewLRUCache(10, 10)
	transparent.Stack(w.pageCache, source)
	return nil
}

func (w *Wikidata) loadMarkdownAsync(titleHash string) (*pageData, error) {
	value, err := w.pageCache.Get(titleHash)
	if err != nil {
		return nil, err
	}
	pagedata, ok := value.(*pageData)
	if !ok {
		return nil, errors.New("Failed to get page data")
	}
	return pagedata, nil
}

func (w *Wikidata) saveMarkdownAsync(titleHash string, pageData *pageData) error {
	return w.pageCache.Set(titleHash, pageData)
}

func (w *Wikidata) deleteMarkdownAsync(titleHash string) error {
	return w.pageCache.Remove(titleHash)
}

// for UserData
func (w *Wikidata) initializeUserCache() error {
	getfunc := func(key interface{}) (interface{}, error) {
		name, _ := key.(string)
		log.Println("Get from source", name)
		user, err := w.loadUser(name)
		if err != nil {
			return nil, err
		}
		return user, nil
	}

	addfunc := func(key interface{}, value interface{}) (err error) {
		userData, _ := value.(*userData)
		log.Println("Set source", key)
		err = w.saveUser(userData)
		return err
	}

	removefunc := func(key interface{}) (err error) {
		name, _ := key.(string)
		err = w.deleteUser(name)
		return err
	}
	storage, err := transparent.NewCustomStorage(getfunc, addfunc, removefunc)
	if err != nil {
		return err
	}
	source, err := transparent.NewSource(storage)
	if err != nil {
		return err
	}
	w.userCache, _ = transparent.NewLRUCache(10, 10)
	transparent.Stack(w.userCache, source)
	return nil
}

func (w *Wikidata) loadUserAsync(name string) (*userData, error) {
	value, err := w.userCache.Get(name)
	if err != nil {
		return nil, err
	}
	userdata, ok := value.(*userData)
	if !ok {
		return nil, errors.New("Failed to get user data")
	}
	return userdata, nil
}

func (w *Wikidata) saveUserAsync(name string, userData *userData) error {
	return w.userCache.Set(name, userData)
}

func (w *Wikidata) deleteUserAsync(name string) error {
	return w.userCache.Remove(name)
}

// for File
func (w *Wikidata) initializeFileCache() error {
	getfunc := func(key interface{}) (interface{}, error) {
		fileDataKey, _ := key.(fileDataKey)
		log.Println("Get from source", fileDataKey)
		file, err := w.loadFile(&fileDataKey)
		if err != nil {
			return nil, err
		}
		return file, nil
	}

	addfunc := func(key interface{}, value interface{}) (err error) {
		fileData, _ := value.(*fileData)
		log.Println("Set source", key)
		err = w.saveFile(fileData)
		return err
	}

	removefunc := func(key interface{}) (err error) {
		fileDataKey, _ := key.(fileDataKey)
		err = w.deleteFile(fileDataKey)
		return err
	}
	storage, err := transparent.NewCustomStorage(getfunc, addfunc, removefunc)
	if err != nil {
		return err
	}
	source, err := transparent.NewSource(storage)
	if err != nil {
		return err
	}
	w.fileCache, _ = transparent.NewLRUCache(10, 10)
	transparent.Stack(w.fileCache, source)
	return nil
}

func (w *Wikidata) loadFileAsync(key fileDataKey) (*fileData, error) {
	value, err := w.fileCache.Get(key)
	if err != nil {
		return nil, err
	}
	filedata, ok := value.(*fileData)
	if !ok {
		return nil, errors.New("Failed to get file data")
	}
	return filedata, nil
}

func (w *Wikidata) saveFileAsync(key fileDataKey, fileData *fileData) error {
	return w.fileCache.Set(key, fileData)
}

func (w *Wikidata) deleteFileAsync(key fileDataKey) error {
	return w.fileCache.Remove(key)
}
<|endoftext|>


<commit_before>package cfdns

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"time"
)

const (
	defaultAPIBase = "https://api.cloudflare.com/client/v4"
)

// A DNSRecord contains information about a single DNS Record.
type DNSRecord struct {
	Name      string      `json:"name"`
	Type      string      `json:"type"`
	Content   string      `json:"content"`
	ID        string      `json:"id,omitempty"`
	TTL       int         `json:"ttl,omitempty"`
	Proxiable bool        `json:"proxiable"`
	Proxied   bool        `json:"proxied"`
	Locked    bool        `json:"locked"`
	ZoneID    string      `json:"zone_id,omitempty"`
	ZoneName  string      `json:"zone_name,omitempty"`
	Created   time.Time   `json:"created_on,omitempty"`
	Modified  time.Time   `json:"modified_on,omitempty"`
	Data      interface{} `json:"data,omitempty"`
}

type cfDNSResult struct {
	Result []DNSRecord
}

type Zone struct {
	ID                  string    `json:"id,omitempty"`
	Name                string    `json:"name"`
	DevelopmentMode     int       `json:"development_mode,omitempty"`
	OriginalRegistrar   string    `json:"original_registrar,omitempty"`
	OriginalDNSHost     string    `json:"original_dnshost,omitempty"`
	OriginalNameServers []string  `json:"original_name_servers,omitempty"`
	Created             time.Time `json:"created_on,omitempty"`
	Modified            time.Time `json:"modified_on,omitempty"`
	NameServers         []string  `json:"name_servers,omitempty"`
	Status              string    `json:"status,omitempty"`
	Paused              bool      `json:"paused,omitempty"`
	Type                string    `json:"type,omitempty"`
	Checked             time.Time `json:"checked_on,omitempty"`
}

type cfZoneResult struct {
	Result []Zone
}

// A Client is used to communicate with the Cloudflare service.
type Client struct {
	apiBase   string
	authEmail string
	authKey   string
}

// NewClient returns a new, initialized Client. The email and API key are used to authenticate against Cloudflare.
func NewClient(authEmail, authKey string) *Client {
	return &Client{
		apiBase:   defaultAPIBase,
		authEmail: authEmail,
		authKey:   authKey,
	}
}

// ListZones returns a list of all zones accessable by the client, or an error.
func (c *Client) ListZones() ([]Zone, error) {
	resp, err := c.doRequest("GET", "/zones?per_page=1000", nil)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var res cfZoneResult
	err = json.NewDecoder(resp.Body).Decode(&res)
	if err != nil {
		return nil, err
	}

	return res.Result, nil
}

// ListDNSRecords returns a list of all DNS records under the given zone, or an error. The zone ID can be found in the results of ListZones.
func (c *Client) ListDNSRecords(zoneID string) ([]DNSRecord, error) {
	resp, err := c.doRequest("GET", fmt.Sprintf("/zones/%s/dns_records?per_page=1000", zoneID), nil)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var res cfDNSResult
	err = json.NewDecoder(resp.Body).Decode(&res)
	if err != nil {
		return nil, err
	}

	return res.Result, nil
}

// CreateDNSRecord creates a new DNS record in the given zone, with the specified name, type and content.
func (c *Client) CreateDNSRecord(zoneID, name, rectype, content string) error {
	body, _ := jsonReader(DNSRecord{
		Name:    name,
		Type:    rectype,
		Content: content,
	})

	resp, err := c.doRequest("POST", fmt.Sprintf("/zones/%s/dns_records", zoneID), body)
	if err != nil {
		return err
	}
	resp.Body.Close()

	if resp.StatusCode != 200 {
		return errors.New(resp.Status)
	}
	return nil
}

// UpdateDNSRecord updates an existing record and returns nil, or an error. The record should be as recieved from ListDNSRecords; specifically, the fields ID and ZoneID must be filled out.
func (c *Client) UpdateDNSRecord(rec DNSRecord) error {
	body, err := jsonReader(rec)
	if err != nil {
		return err
	}

	resp, err := c.doRequest("PUT", fmt.Sprintf("/zones/%s/dns_records/%s", rec.ZoneID, rec.ID), body)
	if err != nil {
		return err
	}
	resp.Body.Close()

	if resp.StatusCode != 200 {
		return errors.New(resp.Status)
	}
	return nil
}

// UpdateDNSRecord deletes an existing record and returns nil, or an error. The record should be as recieved from ListDNSRecords; specifically, the fields ID and ZoneID must be filled out.
func (c *Client) DeleteDNSRecord(rec DNSRecord) error {
	resp, err := c.doRequest("DELETE", fmt.Sprintf("/zones/%s/dns_records/%s", rec.ZoneID, rec.ID), nil)
	if err != nil {
		return err
	}
	resp.Body.Close()

	if resp.StatusCode != 200 {
		return errors.New(resp.Status)
	}
	return nil
}

func (c *Client) doRequest(method, url string, body io.Reader) (*http.Response, error) {
	req, err := http.NewRequest(method, fmt.Sprintf("%s%s", c.apiBase, url), body)
	if err != nil {
		return nil, err
	}
	req.Header.Set("X-Auth-Email", c.authEmail)
	req.Header.Set("X-Auth-Key", c.authKey)

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	return resp, nil
}

func jsonReader(obj interface{}) (io.Reader, error) {
	bs, err := json.Marshal(obj)
	if err != nil {
		return nil, err
	}
	return bytes.NewReader(bs), nil
}
<commit_msg>Add DNSRecord.String()<commit_after>package cfdns

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"time"
)

const (
	defaultAPIBase = "https://api.cloudflare.com/client/v4"
)

// A DNSRecord contains information about a single DNS Record.
type DNSRecord struct {
	Name      string      `json:"name"`
	Type      string      `json:"type"`
	Content   string      `json:"content"`
	ID        string      `json:"id,omitempty"`
	TTL       int         `json:"ttl,omitempty"`
	Proxiable bool        `json:"proxiable"`
	Proxied   bool        `json:"proxied"`
	Locked    bool        `json:"locked"`
	ZoneID    string      `json:"zone_id,omitempty"`
	ZoneName  string      `json:"zone_name,omitempty"`
	Created   time.Time   `json:"created_on,omitempty"`
	Modified  time.Time   `json:"modified_on,omitempty"`
	Data      interface{} `json:"data,omitempty"`
}

func (r DNSRecord) String() string {
	return r.Name + " " + r.Type + " " + r.Content
}

type cfDNSResult struct {
	Result []DNSRecord
}

type Zone struct {
	ID                  string    `json:"id,omitempty"`
	Name                string    `json:"name"`
	DevelopmentMode     int       `json:"development_mode,omitempty"`
	OriginalRegistrar   string    `json:"original_registrar,omitempty"`
	OriginalDNSHost     string    `json:"original_dnshost,omitempty"`
	OriginalNameServers []string  `json:"original_name_servers,omitempty"`
	Created             time.Time `json:"created_on,omitempty"`
	Modified            time.Time `json:"modified_on,omitempty"`
	NameServers         []string  `json:"name_servers,omitempty"`
	Status              string    `json:"status,omitempty"`
	Paused              bool      `json:"paused,omitempty"`
	Type                string    `json:"type,omitempty"`
	Checked             time.Time `json:"checked_on,omitempty"`
}

type cfZoneResult struct {
	Result []Zone
}

// A Client is used to communicate with the Cloudflare service.
type Client struct {
	apiBase   string
	authEmail string
	authKey   string
}

// NewClient returns a new, initialized Client. The email and API key are used to authenticate against Cloudflare.
func NewClient(authEmail, authKey string) *Client {
	return &Client{
		apiBase:   defaultAPIBase,
		authEmail: authEmail,
		authKey:   authKey,
	}
}

// ListZones returns a list of all zones accessable by the client, or an error.
func (c *Client) ListZones() ([]Zone, error) {
	resp, err := c.doRequest("GET", "/zones?per_page=1000", nil)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var res cfZoneResult
	err = json.NewDecoder(resp.Body).Decode(&res)
	if err != nil {
		return nil, err
	}

	return res.Result, nil
}

// ListDNSRecords returns a list of all DNS records under the given zone, or an error. The zone ID can be found in the results of ListZones.
func (c *Client) ListDNSRecords(zoneID string) ([]DNSRecord, error) {
	resp, err := c.doRequest("GET", fmt.Sprintf("/zones/%s/dns_records?per_page=1000", zoneID), nil)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var res cfDNSResult
	err = json.NewDecoder(resp.Body).Decode(&res)
	if err != nil {
		return nil, err
	}

	return res.Result, nil
}

// CreateDNSRecord creates a new DNS record in the given zone, with the specified name, type and content.
func (c *Client) CreateDNSRecord(zoneID, name, rectype, content string) error {
	body, _ := jsonReader(DNSRecord{
		Name:    name,
		Type:    rectype,
		Content: content,
	})

	resp, err := c.doRequest("POST", fmt.Sprintf("/zones/%s/dns_records", zoneID), body)
	if err != nil {
		return err
	}
	resp.Body.Close()

	if resp.StatusCode != 200 {
		return errors.New(resp.Status)
	}
	return nil
}

// UpdateDNSRecord updates an existing record and returns nil, or an error. The record should be as recieved from ListDNSRecords; specifically, the fields ID and ZoneID must be filled out.
func (c *Client) UpdateDNSRecord(rec DNSRecord) error {
	body, err := jsonReader(rec)
	if err != nil {
		return err
	}

	resp, err := c.doRequest("PUT", fmt.Sprintf("/zones/%s/dns_records/%s", rec.ZoneID, rec.ID), body)
	if err != nil {
		return err
	}
	resp.Body.Close()

	if resp.StatusCode != 200 {
		return errors.New(resp.Status)
	}
	return nil
}

// UpdateDNSRecord deletes an existing record and returns nil, or an error. The record should be as recieved from ListDNSRecords; specifically, the fields ID and ZoneID must be filled out.
func (c *Client) DeleteDNSRecord(rec DNSRecord) error {
	resp, err := c.doRequest("DELETE", fmt.Sprintf("/zones/%s/dns_records/%s", rec.ZoneID, rec.ID), nil)
	if err != nil {
		return err
	}
	resp.Body.Close()

	if resp.StatusCode != 200 {
		return errors.New(resp.Status)
	}
	return nil
}

func (c *Client) doRequest(method, url string, body io.Reader) (*http.Response, error) {
	req, err := http.NewRequest(method, fmt.Sprintf("%s%s", c.apiBase, url), body)
	if err != nil {
		return nil, err
	}
	req.Header.Set("X-Auth-Email", c.authEmail)
	req.Header.Set("X-Auth-Key", c.authKey)

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	return resp, nil
}

func jsonReader(obj interface{}) (io.Reader, error) {
	bs, err := json.Marshal(obj)
	if err != nil {
		return nil, err
	}
	return bytes.NewReader(bs), nil
}
<|endoftext|>


<commit_before>package cron

import (
	"fmt"
	"runtime"
	"sync"
	"time"
)

// JobWrapper decorates the given Job with some behavior.
type JobWrapper func(Job) Job

// Chain is a sequence of JobWrappers that decorates submitted jobs with
// cross-cutting behaviors like logging or synchronization.
type Chain struct {
	wrappers []JobWrapper
}

// NewChain returns a Chain consisting of the given JobWrappers.
func NewChain(c ...JobWrapper) Chain {
	return Chain{c}
}

// Then decorates the given job with all JobWrappers in the chain.
//
// This:
//     NewChain(m1, m2, m3).Then(job)
// is equivalent to:
//     m1(m2(m3(job)))
func (c Chain) Then(j Job) Job {
	for i := range c.wrappers {
		j = c.wrappers[len(c.wrappers)-i-1](j)
	}
	return j
}

// Recover panics in wrapped jobs and log them with the provided logger.
func Recover(logger Logger) JobWrapper {
	return func(j Job) Job {
		return FuncJob(func() {
			defer func() {
				if r := recover(); r != nil {
					const size = 64 << 10
					buf := make([]byte, size)
					buf = buf[:runtime.Stack(buf, false)]
					err, ok := r.(error)
					if !ok {
						err = fmt.Errorf("%v", r)
					}
					logger.Error(err, "panic", "stack", "...\n"+string(buf))
				}
			}()
			j.Run()
		})
	}
}

// DelayIfStillRunning serializes jobs, delaying subsequent runs until the
// previous one is complete. Jobs running after a delay of more than a minute
// have the delay logged at Info.
func DelayIfStillRunning(logger Logger) JobWrapper {
	return func(j Job) Job {
		var mu sync.Mutex
		return FuncJob(func() {
			start := time.Now()
			mu.Lock()
			defer mu.Unlock()
			if dur := time.Since(start); dur > time.Minute {
				logger.Info("delay", "duration", dur)
			}
			j.Run()
		})
	}
}

// SkipIfStillRunning skips an invocation of the Job if a previous invocation is
// still running. It logs skips to the given logger at Info level.
func SkipIfStillRunning(logger Logger) JobWrapper {
	return func(j Job) Job {
		var ch = make(chan struct{}, 1)
		ch <- struct{}{}
		return FuncJob(func() {
			select {
			case v := <-ch:
				j.Run()
				ch <- v
			default:
				logger.Info("skip")
			}
		})
	}
}
<commit_msg>fix: gracefully quit jobWrapper SkipIfStillRunning<commit_after>package cron

import (
	"fmt"
	"runtime"
	"sync"
	"time"
)

// JobWrapper decorates the given Job with some behavior.
type JobWrapper func(Job) Job

// Chain is a sequence of JobWrappers that decorates submitted jobs with
// cross-cutting behaviors like logging or synchronization.
type Chain struct {
	wrappers []JobWrapper
}

// NewChain returns a Chain consisting of the given JobWrappers.
func NewChain(c ...JobWrapper) Chain {
	return Chain{c}
}

// Then decorates the given job with all JobWrappers in the chain.
//
// This:
//     NewChain(m1, m2, m3).Then(job)
// is equivalent to:
//     m1(m2(m3(job)))
func (c Chain) Then(j Job) Job {
	for i := range c.wrappers {
		j = c.wrappers[len(c.wrappers)-i-1](j)
	}
	return j
}

// Recover panics in wrapped jobs and log them with the provided logger.
func Recover(logger Logger) JobWrapper {
	return func(j Job) Job {
		return FuncJob(func() {
			defer func() {
				if r := recover(); r != nil {
					const size = 64 << 10
					buf := make([]byte, size)
					buf = buf[:runtime.Stack(buf, false)]
					err, ok := r.(error)
					if !ok {
						err = fmt.Errorf("%v", r)
					}
					logger.Error(err, "panic", "stack", "...\n"+string(buf))
				}
			}()
			j.Run()
		})
	}
}

// DelayIfStillRunning serializes jobs, delaying subsequent runs until the
// previous one is complete. Jobs running after a delay of more than a minute
// have the delay logged at Info.
func DelayIfStillRunning(logger Logger) JobWrapper {
	return func(j Job) Job {
		var mu sync.Mutex
		return FuncJob(func() {
			start := time.Now()
			mu.Lock()
			defer mu.Unlock()
			if dur := time.Since(start); dur > time.Minute {
				logger.Info("delay", "duration", dur)
			}
			j.Run()
		})
	}
}

// SkipIfStillRunning skips an invocation of the Job if a previous invocation is
// still running. It logs skips to the given logger at Info level.
func SkipIfStillRunning(logger Logger) JobWrapper {
	return func(j Job) Job {
		var ch = make(chan struct{}, 1)
		ch <- struct{}{}
		return FuncJob(func() {
			select {
			case v := <-ch:
				defer func() { ch <- v }()
				j.Run()
			default:
				logger.Info("skip")
			}
		})
	}
}
<|endoftext|>