<commit_before>// Copyright 2016 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package facade

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
	"strings"

	"github.com/control-center/serviced/dao"
	"github.com/control-center/serviced/datastore"
	"github.com/control-center/serviced/domain/servicedefinition"
	"github.com/control-center/serviced/domain/servicetemplate"
	"github.com/control-center/serviced/domain/service"
	"github.com/control-center/serviced/utils"
	log "github.com/Sirupsen/logrus"
)

var (
	logstashConfigLock = &sync.Mutex{}
	ErrLogstashUnchanged = errors.New("logstash config unchanged")
)

//
// ReloadLogstashConfig will create a new logstash configuration based on the union of information from all
// templates and all currently deployed services.  A union of values is used because scenarios like a service migration
// can expand the scope of auditable logs or change log filters without touching the currently loaded templates.
//
// If the new configuration is different from the one currently used by logstash,
// then it will rewrite the logstash.conf file, trusting that logstash will recognize the file change
// and reload the new filter set.
//
// This method should be called anytime the available service templates are modified or deployed services are upgraded.
//
// This method depends on the elasticsearch container being up and running.
//
// Note that the strategy of using a union of fields from the templates and deployed services means that if a template
// says a certain log file should be auditable, but the deployed service does not, then the field will still be
// auditable.  Refactoring to update logstash soley on the basis of deployed services might resolve that problem, but
// it still leaves the constraint that in cases where separate tenant applications have conflicting filters/auditable
// types for the same file, the last one wins.
//
func (f *Facade) ReloadLogstashConfig(ctx datastore.Context) error {
	// serialize updates so that we don't have different threads overwriting the same file.
	logstashConfigLock.Lock()
	defer logstashConfigLock.Unlock()

	templates, err := f.GetServiceTemplates(ctx)
	if err != nil {
		plog.WithError(err).Error("Could not retrieve service templates")
		return err
	}

	tenantIDs, err := f.GetTenantIDs(ctx)
	if err != nil {
		plog.WithError(err).Error("Could not retrieve tenant IDs")
		return err
	}

	services := []service.Service{}
	for _, tenantID := range tenantIDs {
		svcs, err := f.GetServices(ctx, dao.ServiceRequest{TenantID: tenantID})
		if err != nil {
			plog.WithError(err).
				WithField("tenantid", tenantID).Error("Could not retrieve services for tenant")
			return err
		}

		services = append(services, svcs...)
	}

	err = writeLogstashConfiguration(templates, services)
	if err == ErrLogstashUnchanged {
		return nil
	} else if err != nil {
		plog.WithError(err).Error("Could not write logstash configuration: %s", err)
		return err
	}
	return nil
}

type reloadLogstashContainer func(ctx datastore.Context, f FacadeInterface) error

var LogstashContainerReloader reloadLogstashContainer = reloadLogstashContainerImpl

func reloadLogstashContainerImpl(ctx datastore.Context, f FacadeInterface) error {
	return f.ReloadLogstashConfig(ctx)
}

// writeLogstashConfiguration takes a map of ServiceTemplates and writes them to the
// appropriate place in the logstash.conf.
// This is required before logstash startup
//
// This method returns nil of logstash configuration was replaced,
// ErrLogstashUnchanged if the configuration is unchanged, or other errors if there was an I/O problem
func writeLogstashConfiguration(templates map[string]servicetemplate.ServiceTemplate, services []service.Service) error {
	// the definitions are a map of filter name to content
	// they are found by recursively going through all the service definitions
	filterDefs := make(map[string]string)
	for _, template := range templates {
		subFilterDefs := getFilterDefinitions(template.Services)
		for name, value := range subFilterDefs {
			filterDefs[name] = value
		}
	}

	// filters will be a syntactically correct logstash filters section
	filters := ""

	typeFilter := []string{}
	auditableTypes := []string{}
	auditLogSection := ""
	plog.Debugf("Checking %d templates", len(templates))
	for _, template := range templates {
		filters += getFiltersFromTemplates(template.Services, filterDefs, &typeFilter)
		auditLogSection = getAuditLogSectionFromTemplates(template.Services, &auditableTypes)
	}
	plog.Debugf("after templates, auditLogSection=%s", auditLogSection)

	plog.Debugf("Checking %d services", len(services))
	for _, svc := range services {
		filters += getFilters(svc.LogConfigs, filterDefs, &typeFilter)
		auditLogSection += getAuditLogSection(svc.LogConfigs,  &auditableTypes)
	}
	plog.Debugf("after services, auditLogSection=%s", auditLogSection)

	logstashDir := getLogstashConfigDirectory()
	newConfigFile := filepath.Join(logstashDir, "logstash.conf.new")
	originalFile :=filepath.Join(logstashDir, "logstash.conf")
	logger := plog.WithFields(log.Fields{
		"newconfigfile": newConfigFile,
		"currentconfigfile": originalFile,
	})

	err := writeLogStashConfigFile(filters, auditLogSection, newConfigFile)
	if err != nil {
		logger.WithError(err).Error("Unable to create new logstash config file")
		return err
	}

	originalContents, err := ioutil.ReadFile(originalFile)
	if err != nil {
		logger.WithError(err).Error("Unable to read current logstash config file")
		return err
	}

	newContents, err := ioutil.ReadFile(newConfigFile)
	if err != nil {
		logger.WithError(err).Error("Unable to read new logstash config file")
		return err
	}

	// Now compare the new config to the current config, and
	// only replace the current config if they are different
	if bytes.Equal(originalContents, newContents) {
		return ErrLogstashUnchanged
	} else if err := os.Rename(newConfigFile, originalFile); err != nil {
		logger.WithError(err).Error("Unable to replace current logstash config file")
		return err
	}
	plog.WithField("currentconfigfile", originalFile).Info("Updated logstash configuration")
	return nil
}

func getFilterDefinitions(services []servicedefinition.ServiceDefinition) map[string]string {
	filterDefs := make(map[string]string)
	for _, service := range services {
		for name, value := range service.LogFilters {
			filterDefs[name] = value
		}

		if len(service.Services) > 0 {
			subFilterDefs := getFilterDefinitions(service.Services)
			for name, value := range subFilterDefs {
				filterDefs[name] = value
			}
		}
	}
	return filterDefs
}

func getFiltersFromTemplates(services []servicedefinition.ServiceDefinition, filterDefs map[string]string, typeFilter *[]string) string {
	filters := ""
	for _, svc := range services {
		filters += getFilters(svc.LogConfigs, filterDefs, typeFilter)
		if len(svc.Services) > 0 {
			subFilts := getFiltersFromTemplates(svc.Services, filterDefs, typeFilter)
			filters += subFilts
		}
	}
	return filters
}

func getFilters(configs []servicedefinition.LogConfig, filterDefs map[string]string, typeFilter *[]string) string {
	filters := ""
	for _, config := range configs {
		for _, filtName := range config.Filters {
			//do not write duplicate types, logstash doesn't handle this
			if !utils.StringInSlice(config.Type, *typeFilter) {
				filters += fmt.Sprintf("\n  if [type] == \"%s\" {\n    %s\n  }\n",
					config.Type, indent(filterDefs[filtName], "    "))
				*typeFilter = append(*typeFilter, config.Type)
			}
		}
	}
	return filters
}

func getAuditLogSectionFromTemplates(services []servicedefinition.ServiceDefinition, auditTypes *[]string) string {
	auditSection := ""
	for _, svc := range services {
		auditSection += getAuditLogSection(svc.LogConfigs, auditTypes)
		if len(svc.Services) > 0 {
			subServiceOutput := getAuditLogSectionFromTemplates(svc.Services, auditTypes)
			auditSection += subServiceOutput
		}
	}
	return auditSection
}

func getAuditLogSection(configs []servicedefinition.LogConfig, auditTypes *[]string) string {
	auditSection := ""
	fileSection := `file {
  path => "%s"
  codec => line { format => "%%{message}"}
}`
	auditLogFile := filepath.Join(utils.LOGSTASH_LOCAL_SERVICED_LOG_DIR, "application-audit.log")
	fileSection = fmt.Sprintf(fileSection, auditLogFile)
	for _, config := range configs {
		if config.IsAudit {
			if !utils.StringInSlice(config.Type, *auditTypes){
				plog.Infof("found type %q enabled for audit, file=%s", config.Type, config.Path)
				auditSection += fmt.Sprintf("\n        if [fields][type] == \"%s\" {\n%s        }",
					config.Type, indent(fileSection, "            "))
				*auditTypes = append(*auditTypes, config.Type)
			}
		}
	}
	return auditSection
}

// This method writes out the config file for logstash. It uses
// the logstash.conf.template and does a variable replacement.
func writeLogStashConfigFile(filters string, auditLogSection string, outputPath string) error {
	// read the log configuration template
	templatePath := filepath.Join(getLogstashConfigDirectory(), "logstash.conf.template")

	contents, err := ioutil.ReadFile(templatePath)
	if err != nil {
		return err
	}

	// the newlines in the string below are deliberate so that the filter
	// syntax is correct
	filterSection := `
filter {
  mutate {
    rename => {
      "source" => "file"
    }
  }
# NOTE the filters are generated from the service definitions
` + string(filters) + `
}
`
	newContents := strings.Replace(string(contents), "${FILTER_SECTION}", filterSection, 1)
	if len(auditLogSection) > 0 {
		newContents = strings.Replace(string(newContents),"${AUDITLOG_SECTION}", auditLogSection, 1)
	}
	newBytes := []byte(newContents)
	// generate the filters section
	// write the log file
	return ioutil.WriteFile(outputPath, newBytes, 0644)
}

func indent(src, tab string) string {
	result := ""
	lines := strings.Split(src, "\n")
	for _, line := range lines {
		result += tab + line + "\n"
	}
	return result
}

func getLogstashConfigDirectory() string {
	return filepath.Join(utils.ResourcesDir(), "logstash")
}
<commit_msg>CC-3174: Multiline Log Parsing Does not Work<commit_after>// Copyright 2016 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package facade

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
	"strings"

	"github.com/control-center/serviced/dao"
	"github.com/control-center/serviced/datastore"
	"github.com/control-center/serviced/domain/servicedefinition"
	"github.com/control-center/serviced/domain/servicetemplate"
	"github.com/control-center/serviced/domain/service"
	"github.com/control-center/serviced/utils"
	log "github.com/Sirupsen/logrus"
)

var (
	logstashConfigLock = &sync.Mutex{}
	ErrLogstashUnchanged = errors.New("logstash config unchanged")
)

//
// ReloadLogstashConfig will create a new logstash configuration based on the union of information from all
// templates and all currently deployed services.  A union of values is used because scenarios like a service migration
// can expand the scope of auditable logs or change log filters without touching the currently loaded templates.
//
// If the new configuration is different from the one currently used by logstash,
// then it will rewrite the logstash.conf file, trusting that logstash will recognize the file change
// and reload the new filter set.
//
// This method should be called anytime the available service templates are modified or deployed services are upgraded.
//
// This method depends on the elasticsearch container being up and running.
//
// Note that the strategy of using a union of fields from the templates and deployed services means that if a template
// says a certain log file should be auditable, but the deployed service does not, then the field will still be
// auditable.  Refactoring to update logstash soley on the basis of deployed services might resolve that problem, but
// it still leaves the constraint that in cases where separate tenant applications have conflicting filters/auditable
// types for the same file, the last one wins.
//
func (f *Facade) ReloadLogstashConfig(ctx datastore.Context) error {
	// serialize updates so that we don't have different threads overwriting the same file.
	logstashConfigLock.Lock()
	defer logstashConfigLock.Unlock()

	templates, err := f.GetServiceTemplates(ctx)
	if err != nil {
		plog.WithError(err).Error("Could not retrieve service templates")
		return err
	}

	tenantIDs, err := f.GetTenantIDs(ctx)
	if err != nil {
		plog.WithError(err).Error("Could not retrieve tenant IDs")
		return err
	}

	services := []service.Service{}
	for _, tenantID := range tenantIDs {
		svcs, err := f.GetServices(ctx, dao.ServiceRequest{TenantID: tenantID})
		if err != nil {
			plog.WithError(err).
				WithField("tenantid", tenantID).Error("Could not retrieve services for tenant")
			return err
		}

		services = append(services, svcs...)
	}

	err = writeLogstashConfiguration(templates, services)
	if err == ErrLogstashUnchanged {
		return nil
	} else if err != nil {
		plog.WithError(err).Error("Could not write logstash configuration: %s", err)
		return err
	}
	return nil
}

type reloadLogstashContainer func(ctx datastore.Context, f FacadeInterface) error

var LogstashContainerReloader reloadLogstashContainer = reloadLogstashContainerImpl

func reloadLogstashContainerImpl(ctx datastore.Context, f FacadeInterface) error {
	return f.ReloadLogstashConfig(ctx)
}

// writeLogstashConfiguration takes a map of ServiceTemplates and writes them to the
// appropriate place in the logstash.conf.
// This is required before logstash startup
//
// This method returns nil of logstash configuration was replaced,
// ErrLogstashUnchanged if the configuration is unchanged, or other errors if there was an I/O problem
func writeLogstashConfiguration(templates map[string]servicetemplate.ServiceTemplate, services []service.Service) error {
	// the definitions are a map of filter name to content
	// they are found by recursively going through all the service definitions
	filterDefs := make(map[string]string)
	for _, template := range templates {
		subFilterDefs := getFilterDefinitions(template.Services)
		for name, value := range subFilterDefs {
			filterDefs[name] = value
		}
	}

	// filters will be a syntactically correct logstash filters section
	filters := ""

	typeFilter := []string{}
	auditableTypes := []string{}
	auditLogSection := ""
	plog.Debugf("Checking %d templates", len(templates))
	for _, template := range templates {
		filters += getFiltersFromTemplates(template.Services, filterDefs, &typeFilter)
		auditLogSection = getAuditLogSectionFromTemplates(template.Services, &auditableTypes)
	}
	plog.Debugf("after templates, auditLogSection=%s", auditLogSection)

	plog.Debugf("Checking %d services", len(services))
	for _, svc := range services {
		filters += getFilters(svc.LogConfigs, filterDefs, &typeFilter)
		auditLogSection += getAuditLogSection(svc.LogConfigs,  &auditableTypes)
	}
	plog.Debugf("after services, auditLogSection=%s", auditLogSection)

	logstashDir := getLogstashConfigDirectory()
	newConfigFile := filepath.Join(logstashDir, "logstash.conf.new")
	originalFile :=filepath.Join(logstashDir, "logstash.conf")
	logger := plog.WithFields(log.Fields{
		"newconfigfile": newConfigFile,
		"currentconfigfile": originalFile,
	})

	err := writeLogStashConfigFile(filters, auditLogSection, newConfigFile)
	if err != nil {
		logger.WithError(err).Error("Unable to create new logstash config file")
		return err
	}

	originalContents, err := ioutil.ReadFile(originalFile)
	if err != nil {
		logger.WithError(err).Error("Unable to read current logstash config file")
		return err
	}

	newContents, err := ioutil.ReadFile(newConfigFile)
	if err != nil {
		logger.WithError(err).Error("Unable to read new logstash config file")
		return err
	}

	// Now compare the new config to the current config, and
	// only replace the current config if they are different
	if bytes.Equal(originalContents, newContents) {
		return ErrLogstashUnchanged
	} else if err := os.Rename(newConfigFile, originalFile); err != nil {
		logger.WithError(err).Error("Unable to replace current logstash config file")
		return err
	}
	plog.WithField("currentconfigfile", originalFile).Info("Updated logstash configuration")
	return nil
}

func getFilterDefinitions(services []servicedefinition.ServiceDefinition) map[string]string {
	filterDefs := make(map[string]string)
	for _, service := range services {
		for name, value := range service.LogFilters {
			filterDefs[name] = value
		}

		if len(service.Services) > 0 {
			subFilterDefs := getFilterDefinitions(service.Services)
			for name, value := range subFilterDefs {
				filterDefs[name] = value
			}
		}
	}
	return filterDefs
}

func getFiltersFromTemplates(services []servicedefinition.ServiceDefinition, filterDefs map[string]string, typeFilter *[]string) string {
	filters := ""
	for _, svc := range services {
		filters += getFilters(svc.LogConfigs, filterDefs, typeFilter)
		if len(svc.Services) > 0 {
			subFilts := getFiltersFromTemplates(svc.Services, filterDefs, typeFilter)
			filters += subFilts
		}
	}
	return filters
}

func getFilters(configs []servicedefinition.LogConfig, filterDefs map[string]string, typeFilter *[]string) string {
	filters := ""
	for _, config := range configs {
		for _, filtName := range config.Filters {
			//do not write duplicate types, logstash doesn't handle this
			if !utils.StringInSlice(config.Type, *typeFilter) {
				filters += fmt.Sprintf("\n  if \"%s\" in [file] {\n    %s\n  }\n",
					config.Type, indent(filterDefs[filtName], "    "))
				*typeFilter = append(*typeFilter, config.Type)
			}
		}
	}
	return filters
}

func getAuditLogSectionFromTemplates(services []servicedefinition.ServiceDefinition, auditTypes *[]string) string {
	auditSection := ""
	for _, svc := range services {
		auditSection += getAuditLogSection(svc.LogConfigs, auditTypes)
		if len(svc.Services) > 0 {
			subServiceOutput := getAuditLogSectionFromTemplates(svc.Services, auditTypes)
			auditSection += subServiceOutput
		}
	}
	return auditSection
}

func getAuditLogSection(configs []servicedefinition.LogConfig, auditTypes *[]string) string {
	auditSection := ""
	fileSection := `file {
  path => "%s"
  codec => line { format => "%%{message}"}
}`
	auditLogFile := filepath.Join(utils.LOGSTASH_LOCAL_SERVICED_LOG_DIR, "application-audit.log")
	fileSection = fmt.Sprintf(fileSection, auditLogFile)
	for _, config := range configs {
		if config.IsAudit {
			if !utils.StringInSlice(config.Type, *auditTypes){
				plog.Infof("found type %q enabled for audit, file=%s", config.Type, config.Path)
				auditSection += fmt.Sprintf("\n        if [fields][type] == \"%s\" {\n%s        }",
					config.Type, indent(fileSection, "            "))
				*auditTypes = append(*auditTypes, config.Type)
			}
		}
	}
	return auditSection
}

// This method writes out the config file for logstash. It uses
// the logstash.conf.template and does a variable replacement.
func writeLogStashConfigFile(filters string, auditLogSection string, outputPath string) error {
	// read the log configuration template
	templatePath := filepath.Join(getLogstashConfigDirectory(), "logstash.conf.template")

	contents, err := ioutil.ReadFile(templatePath)
	if err != nil {
		return err
	}

	// the newlines in the string below are deliberate so that the filter
	// syntax is correct
	filterSection := `
filter {
  mutate {
    rename => {
      "source" => "file"
    }
  }
# NOTE the filters are generated from the service definitions
` + string(filters) + `
}
`
	newContents := strings.Replace(string(contents), "${FILTER_SECTION}", filterSection, 1)
	if len(auditLogSection) > 0 {
		newContents = strings.Replace(string(newContents),"${AUDITLOG_SECTION}", auditLogSection, 1)
	}
	newBytes := []byte(newContents)
	// generate the filters section
	// write the log file
	return ioutil.WriteFile(outputPath, newBytes, 0644)
}

func indent(src, tab string) string {
	result := ""
	lines := strings.Split(src, "\n")
	for _, line := range lines {
		result += tab + line + "\n"
	}
	return result
}

func getLogstashConfigDirectory() string {
	return filepath.Join(utils.ResourcesDir(), "logstash")
}
<|endoftext|>


<commit_before>package protocol

import (
	"context"
	"time"

	"github.com/bytom/errors"
	"github.com/bytom/log"
	"github.com/bytom/protocol/bc/legacy"
	"github.com/bytom/protocol/state"
	"github.com/bytom/protocol/validation"
)

// maxBlockTxs limits the number of transactions
// included in each block.
const maxBlockTxs = 10000

// saveSnapshotFrequency stores how often to save a state
// snapshot to the Store.
const saveSnapshotFrequency = time.Hour

var (
	// ErrBadBlock is returned when a block is invalid.
	ErrBadBlock = errors.New("invalid block")

	// ErrStaleState is returned when the Chain does not have a current
	// blockchain state.
	ErrStaleState = errors.New("stale blockchain state")

	// ErrBadStateRoot is returned when the computed assets merkle root
	// disagrees with the one declared in a block header.
	ErrBadStateRoot = errors.New("invalid state merkle root")
)

// GetBlock returns the block at the given height, if there is one,
// otherwise it returns an error.
func (c *Chain) GetBlock(height uint64) (*legacy.Block, error) {
	return c.store.GetBlock(height)
}

// ValidateBlock validates an incoming block in advance of applying it
// to a snapshot (with ApplyValidBlock) and committing it to the
// blockchain (with CommitAppliedBlock).
func (c *Chain) ValidateBlock(block, prev *legacy.Block) error {
	blockEnts := legacy.MapBlock(block)
	prevEnts := legacy.MapBlock(prev)
	err := validation.ValidateBlock(blockEnts, prevEnts)
	if err != nil {
		return errors.Sub(ErrBadBlock, err)
	}
	return errors.Sub(ErrBadBlock, err)
}

// ApplyValidBlock creates an updated snapshot without validating the
// block.
func (c *Chain) ApplyValidBlock(block *legacy.Block) (*state.Snapshot, error) {
	newSnapshot := state.Copy(c.state.snapshot)
	err := newSnapshot.ApplyBlock(legacy.MapBlock(block))
	if err != nil {
		return nil, err
	}
	//fmt.Printf("want %v, ger %v \n", block.BlockHeader.AssetsMerkleRoot, newSnapshot.Tree.RootHash())
	if block.AssetsMerkleRoot != newSnapshot.Tree.RootHash() {
		return nil, ErrBadStateRoot
	}
	return newSnapshot, nil
}

// CommitBlock commits a block to the blockchain. The block
// must already have been applied with ApplyValidBlock or
// ApplyNewBlock, which will have produced the new snapshot that's
// required here.
//
// This function saves the block to the store and sometimes (not more
// often than saveSnapshotFrequency) saves the state tree to the
// store. New-block callbacks (via asynchronous block-processor pins)
// are triggered.
//
// TODO(bobg): rename to CommitAppliedBlock for clarity (deferred from https://github.com/chain/chain/pull/788)
func (c *Chain) CommitAppliedBlock(ctx context.Context, block *legacy.Block, snapshot *state.Snapshot) error {
	// SaveBlock is the linearization point. Once the block is committed
	// to persistent storage, the block has been applied and everything
	// else can be derived from that block.
	err := c.store.SaveBlock(block)
	if err != nil {
		return errors.Wrap(err, "storing block")
	}
	if block.Time().After(c.lastQueuedSnapshot.Add(saveSnapshotFrequency)) {
		c.queueSnapshot(ctx, block.Height, block.Time(), snapshot)
	}

	err = c.store.FinalizeBlock(ctx, block.Height)
	if err != nil {
		return errors.Wrap(err, "finalizing block")
	}

	// c.setState will update the local blockchain state and height.
	// When c.store is a txdb.Store, and c has been initialized with a
	// channel from txdb.ListenBlocks, then the above call to
	// c.store.FinalizeBlock will have done a postgresql NOTIFY and
	// that will wake up the goroutine in NewChain, which also calls
	// setHeight.  But duplicate calls with the same blockheight are
	// harmless; and the following call is required in the cases where
	// it's not redundant.
	c.setState(block, snapshot)
	return nil
}

func (c *Chain) AddBlock(ctx context.Context, block *legacy.Block) error {
	currentBlock, _ := c.State()
	if err := c.ValidateBlock(block, currentBlock); err != nil {
		return err
	}

	newSnap, err := c.ApplyValidBlock(block)
	if err != nil {
		return err
	}

	return c.CommitAppliedBlock(ctx, block, newSnap)
}

func (c *Chain) queueSnapshot(ctx context.Context, height uint64, timestamp time.Time, s *state.Snapshot) {
	// Non-blockingly queue the snapshot for storage.
	ps := pendingSnapshot{height: height, snapshot: s}
	select {
	case c.pendingSnapshots <- ps:
		c.lastQueuedSnapshot = timestamp
	default:
		// Skip it; saving snapshots is taking longer than the snapshotting period.
		log.Printf(ctx, "snapshot storage is taking too long; last queued at %s",
			c.lastQueuedSnapshot)
	}
}

func (c *Chain) setHeight(h uint64) {
	// We call setHeight from two places independently:
	// CommitBlock and the Postgres LISTEN goroutine.
	// This means we can get here twice for each block,
	// and any of them might be arbitrarily delayed,
	// which means h might be from the past.
	// Detect and discard these duplicate calls.

	c.state.cond.L.Lock()
	defer c.state.cond.L.Unlock()

	if h <= c.state.height {
		return
	}
	c.state.height = h
	c.state.cond.Broadcast()
}
<commit_msg>remove tx from pool after success commit block<commit_after>package protocol

import (
	"context"
	"time"

	"github.com/bytom/errors"
	"github.com/bytom/log"
	"github.com/bytom/protocol/bc/legacy"
	"github.com/bytom/protocol/state"
	"github.com/bytom/protocol/validation"
)

// maxBlockTxs limits the number of transactions
// included in each block.
const maxBlockTxs = 10000

// saveSnapshotFrequency stores how often to save a state
// snapshot to the Store.
const saveSnapshotFrequency = time.Hour

var (
	// ErrBadBlock is returned when a block is invalid.
	ErrBadBlock = errors.New("invalid block")

	// ErrStaleState is returned when the Chain does not have a current
	// blockchain state.
	ErrStaleState = errors.New("stale blockchain state")

	// ErrBadStateRoot is returned when the computed assets merkle root
	// disagrees with the one declared in a block header.
	ErrBadStateRoot = errors.New("invalid state merkle root")
)

// GetBlock returns the block at the given height, if there is one,
// otherwise it returns an error.
func (c *Chain) GetBlock(height uint64) (*legacy.Block, error) {
	return c.store.GetBlock(height)
}

// ValidateBlock validates an incoming block in advance of applying it
// to a snapshot (with ApplyValidBlock) and committing it to the
// blockchain (with CommitAppliedBlock).
func (c *Chain) ValidateBlock(block, prev *legacy.Block) error {
	blockEnts := legacy.MapBlock(block)
	prevEnts := legacy.MapBlock(prev)
	err := validation.ValidateBlock(blockEnts, prevEnts)
	if err != nil {
		return errors.Sub(ErrBadBlock, err)
	}
	return errors.Sub(ErrBadBlock, err)
}

// ApplyValidBlock creates an updated snapshot without validating the
// block.
func (c *Chain) ApplyValidBlock(block *legacy.Block) (*state.Snapshot, error) {
	newSnapshot := state.Copy(c.state.snapshot)
	err := newSnapshot.ApplyBlock(legacy.MapBlock(block))
	if err != nil {
		return nil, err
	}
	//fmt.Printf("want %v, ger %v \n", block.BlockHeader.AssetsMerkleRoot, newSnapshot.Tree.RootHash())
	if block.AssetsMerkleRoot != newSnapshot.Tree.RootHash() {
		return nil, ErrBadStateRoot
	}
	return newSnapshot, nil
}

// CommitBlock commits a block to the blockchain. The block
// must already have been applied with ApplyValidBlock or
// ApplyNewBlock, which will have produced the new snapshot that's
// required here.
//
// This function saves the block to the store and sometimes (not more
// often than saveSnapshotFrequency) saves the state tree to the
// store. New-block callbacks (via asynchronous block-processor pins)
// are triggered.
//
// TODO(bobg): rename to CommitAppliedBlock for clarity (deferred from https://github.com/chain/chain/pull/788)
func (c *Chain) CommitAppliedBlock(ctx context.Context, block *legacy.Block, snapshot *state.Snapshot) error {
	// SaveBlock is the linearization point. Once the block is committed
	// to persistent storage, the block has been applied and everything
	// else can be derived from that block.
	err := c.store.SaveBlock(block)
	if err != nil {
		return errors.Wrap(err, "storing block")
	}
	if block.Time().After(c.lastQueuedSnapshot.Add(saveSnapshotFrequency)) {
		c.queueSnapshot(ctx, block.Height, block.Time(), snapshot)
	}

	err = c.store.FinalizeBlock(ctx, block.Height)
	if err != nil {
		return errors.Wrap(err, "finalizing block")
	}

	// c.setState will update the local blockchain state and height.
	// When c.store is a txdb.Store, and c has been initialized with a
	// channel from txdb.ListenBlocks, then the above call to
	// c.store.FinalizeBlock will have done a postgresql NOTIFY and
	// that will wake up the goroutine in NewChain, which also calls
	// setHeight.  But duplicate calls with the same blockheight are
	// harmless; and the following call is required in the cases where
	// it's not redundant.
	c.setState(block, snapshot)
	return nil
}

func (c *Chain) AddBlock(ctx context.Context, block *legacy.Block) error {
	currentBlock, _ := c.State()
	if err := c.ValidateBlock(block, currentBlock); err != nil {
		return err
	}

	newSnap, err := c.ApplyValidBlock(block)
	if err != nil {
		return err
	}

	if err := c.CommitAppliedBlock(ctx, block, newSnap); err != nil {
		return err
	}

	for _, tx := range block.Transactions {
		c.txPool.RemoveTransaction(&tx.Tx.ID)
	}
	return nil
}

func (c *Chain) queueSnapshot(ctx context.Context, height uint64, timestamp time.Time, s *state.Snapshot) {
	// Non-blockingly queue the snapshot for storage.
	ps := pendingSnapshot{height: height, snapshot: s}
	select {
	case c.pendingSnapshots <- ps:
		c.lastQueuedSnapshot = timestamp
	default:
		// Skip it; saving snapshots is taking longer than the snapshotting period.
		log.Printf(ctx, "snapshot storage is taking too long; last queued at %s",
			c.lastQueuedSnapshot)
	}
}

func (c *Chain) setHeight(h uint64) {
	// We call setHeight from two places independently:
	// CommitBlock and the Postgres LISTEN goroutine.
	// This means we can get here twice for each block,
	// and any of them might be arbitrarily delayed,
	// which means h might be from the past.
	// Detect and discard these duplicate calls.

	c.state.cond.L.Lock()
	defer c.state.cond.L.Unlock()

	if h <= c.state.height {
		return
	}
	c.state.height = h
	c.state.cond.Broadcast()
}
<|endoftext|>


<commit_before><commit_msg>add generic range filter<commit_after><|endoftext|>


<commit_before><commit_msg>Implement signal handling for shutting down the program.<commit_after><|endoftext|>


<commit_before>package panos

import (
	"encoding/xml"
	"errors"
	"fmt"
	"strings"

	"github.com/scottdware/go-rested"
)

// AddressObjects contains a slice of all address objects.
type AddressObjects struct {
	XMLName   xml.Name  `xml:"response"`
	Status    string    `xml:"status,attr"`
	Code      string    `xml:"code,attr"`
	Addresses []Address `xml:"result>address>entry"`
}

// Address contains information about each individual address object.
type Address struct {
	Name        string `xml:"name,attr"`
	IPAddress   string `xml:"ip-netmask,omitempty"`
	IPRange     string `xml:"ip-range,omitempty"`
	FQDN        string `xml:"fqdn,omitempty"`
	Description string `xml:"description,omitempty"`
}

// AddressGroups contains a slice of all address groups.
type AddressGroups struct {
	Groups []AddressGroup
}

// AddressGroup contains information about each individual address group.
type AddressGroup struct {
	Name          string
	Type          string
	Members       []string
	DynamicFilter string
	Description   string
}

// xmlAddressGroups is used for parsing of all address groups.
type xmlAddressGroups struct {
	XMLName xml.Name          `xml:"response"`
	Status  string            `xml:"status,attr"`
	Code    string            `xml:"code,attr"`
	Groups  []xmlAddressGroup `xml:"result>address-group>entry"`
}

// xmlAddressGroup is used for parsing each individual address group.
type xmlAddressGroup struct {
	Name          string   `xml:"name,attr"`
	Members       []string `xml:"static>member,omitempty"`
	DynamicFilter string   `xml:"dynamic>filter,omitempty"`
	Description   string   `xml:"description,omitempty"`
}

// Addresses returns information about all of the address objects. When run against a Panorama device,
// addresses from all device-groups are returned.
func (p *PaloAlto) Addresses() (*AddressObjects, error) {
	var addrs AddressObjects
	xpath := "/config/devices/entry//address"
	// xpath := "/config/devices/entry/vsys/entry/address"
	r := rested.NewRequest()

	if p.DeviceType == "panos" && p.Panorama == true {
		xpath = "/config/panorama//address"
	}

	if p.DeviceType == "panorama" {
		// xpath = "/config/devices/entry/device-group/entry/address"
		xpath = "/config/devices/entry//address"
	}

	query := map[string]string{
		"type":   "config",
		"action": "get",
		"xpath":  xpath,
		"key":    p.Key,
	}
	addrData := r.Send("get", p.URI, nil, headers, query)

	if err := xml.Unmarshal(addrData.Body, &addrs); err != nil {
		return nil, err
	}

	if addrs.Status != "success" {
		return nil, fmt.Errorf("error code %s: %s", addrs.Code, errorCodes[addrs.Code])
	}

	return &addrs, nil
}

// AddressGroups returns information about all of the address groups. When run against a Panorama device,
// address groups from all device-groups are returned.
func (p *PaloAlto) AddressGroups() (*AddressGroups, error) {
	var parsedGroups xmlAddressGroups
	var groups AddressGroups
	xpath := "/config/devices/entry//address-group"
	// xpath := "/config/devices/entry/vsys/entry/address-group"
	r := rested.NewRequest()

	if p.DeviceType == "panos" && p.Panorama == true {
		xpath = "/config/panorama//address-group"
	}

	if p.DeviceType == "panorama" {
		// xpath = "/config/devices/entry/device-group/entry/address-group"
		xpath = "/config/devices/entry//address-group"
	}

	query := map[string]string{
		"type":   "config",
		"action": "get",
		"xpath":  xpath,
		"key":    p.Key,
	}
	groupData := r.Send("get", p.URI, nil, headers, query)

	if err := xml.Unmarshal(groupData.Body, &parsedGroups); err != nil {
		return nil, err
	}

	if parsedGroups.Status != "success" {
		return nil, fmt.Errorf("error code %s: %s", parsedGroups.Code, errorCodes[parsedGroups.Code])
	}

	for _, g := range parsedGroups.Groups {
		gname := g.Name
		gtype := "Static"
		gmembers := g.Members
		gfilter := strings.TrimSpace(g.DynamicFilter)
		gdesc := g.Description

		if g.DynamicFilter != "" {
			gtype = "Dynamic"
		}

		groups.Groups = append(groups.Groups, AddressGroup{Name: gname, Type: gtype, Members: gmembers, DynamicFilter: gfilter, Description: gdesc})
	}

	return &groups, nil
}

// CreateAddress will add a new address object to the device. addrtype should be one of: ip, range, or fqdn. If creating
// an address object on a Panorama device, then specify the given device-group name as the last parameter.
func (p *PaloAlto) CreateAddress(name, addrtype, address, description string, devicegroup ...string) error {
	var xmlBody string
	var xpath string
	var reqError requestError
	r := rested.NewRequest()

	switch addrtype {
	case "ip":
		xmlBody = fmt.Sprintf("<ip-netmask>%s</ip-netmask>", address)
	case "range":
		xmlBody = fmt.Sprintf("<ip-range>%s</ip-range>", address)
	case "fqdn":
		xmlBody = fmt.Sprintf("<fqdn>%s</fqdn>", address)
	}

	if description != "" {
		xmlBody += fmt.Sprintf("<description>%s</description>", description)
	}

	if p.DeviceType == "panos" {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys1']/address/entry[@name='%s']", name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) > 0 {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/device-group/entry[@name='%s']/address/entry[@name='%s']", devicegroup[0], name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) <= 0 {
		return errors.New("you must specify a device-group when connected to a Panorama device")
	}

	query := map[string]string{
		"type":    "config",
		"action":  "set",
		"xpath":   xpath,
		"element": xmlBody,
		"key":     p.Key,
	}

	resp := r.Send("post", p.URI, nil, nil, query)
	if resp.Error != nil {
		return resp.Error
	}

	if err := xml.Unmarshal(resp.Body, &reqError); err != nil {
		return err
	}

	if reqError.Status != "success" {
		return fmt.Errorf("error code %s: %s", reqError.Code, errorCodes[reqError.Code])
	}

	return nil
}

// CreateStaticGroup will create a new static address group on the device. You can specify multiple members
// by separating them with a comma, i.e. "web-server1, web-server2". If creating an address group on
// a Panorama device, then specify the given device-group name as the last parameter.
func (p *PaloAlto) CreateStaticGroup(name, members, description string, devicegroup ...string) error {
	var xmlBody string
	var xpath string
	var reqError requestError
	r := rested.NewRequest()
	m := strings.Split(members, ",")

	if members == "" {
		return errors.New("you cannot create a static address group without any members")
	}

	xmlBody = "<static>"
	for _, member := range m {
		xmlBody += fmt.Sprintf("<member>%s</member>", strings.TrimSpace(member))
	}
	xmlBody += "</static>"

	if description != "" {
		xmlBody += fmt.Sprintf("<description>%s</description>", description)
	}

	if p.DeviceType == "panos" {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys1']/address-group/entry[@name='%s']", name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) > 0 {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/device-group/entry[@name='%s']/address-group/entry[@name='%s']", devicegroup[0], name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) <= 0 {
		return errors.New("you must specify a device-group when connected to a Panorama device")
	}

	query := map[string]string{
		"type":    "config",
		"action":  "set",
		"xpath":   xpath,
		"element": xmlBody,
		"key":     p.Key,
	}

	resp := r.Send("post", p.URI, nil, nil, query)
	if resp.Error != nil {
		return resp.Error
	}

	if err := xml.Unmarshal(resp.Body, &reqError); err != nil {
		return err
	}

	if reqError.Status != "success" {
		return fmt.Errorf("error code %s: %s", reqError.Code, errorCodes[reqError.Code])
	}

	return nil
}

// CreateDynamicGroup will create a new dynamic address group on the device. The filter must be written like so:
// 'vm-servers' and 'some tag' or 'pcs' - using the tags as the match criteria. If creating an address group on a
// Panorama device, then specify the given device-group name as the last parameter.
func (p *PaloAlto) CreateDynamicGroup(name, criteria, description string, devicegroup ...string) error {
	xmlBody := fmt.Sprintf("<dynamic><filter>%s</filter></dynamic>", criteria)
	var xpath string
	var reqError requestError
	r := rested.NewRequest()

	if criteria == "" {
		return errors.New("you cannot create a dynamic address group without any filter")
	}

	if description != "" {
		xmlBody += fmt.Sprintf("<description>%s</description>", description)
	}

	if p.DeviceType == "panos" {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys1']/address-group/entry[@name='%s']", name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) > 0 {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/device-group/entry[@name='%s']/address-group/entry[@name='%s']", devicegroup[0], name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) <= 0 {
		return errors.New("you must specify a device-group when connected to a Panorama device")
	}

	query := map[string]string{
		"type":    "config",
		"action":  "set",
		"xpath":   xpath,
		"element": xmlBody,
		"key":     p.Key,
	}

	resp := r.Send("post", p.URI, nil, nil, query)
	if resp.Error != nil {
		return resp.Error
	}

	if err := xml.Unmarshal(resp.Body, &reqError); err != nil {
		return err
	}

	if reqError.Status != "success" {
		return fmt.Errorf("error code %s: %s", reqError.Code, errorCodes[reqError.Code])
	}

	return nil
}

// DeleteAddress will remove an address object from the device. If deleting an address object on a
// Panorama device, then specify the given device-group name as the last parameter.
func (p *PaloAlto) DeleteAddress(name string, devicegroup ...string) error {
	var xpath string
	var reqError requestError
	r := rested.NewRequest()

	if p.DeviceType == "panos" {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys1']/address/entry[@name='%s']", name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) > 0 {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/device-group/entry[@name='%s']/address/entry[@name='%s']", devicegroup[0], name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) <= 0 {
		return errors.New("you must specify a device-group when connected to a Panorama device")
	}

	query := map[string]string{
		"type":   "config",
		"action": "delete",
		"xpath":  xpath,
		"key":    p.Key,
	}

	resp := r.Send("get", p.URI, nil, nil, query)
	if resp.Error != nil {
		return resp.Error
	}

	if err := xml.Unmarshal(resp.Body, &reqError); err != nil {
		return err
	}

	if reqError.Status != "success" {
		return fmt.Errorf("error code %s: %s", reqError.Code, errorCodes[reqError.Code])
	}

	return nil
}

// DeleteAddressGroup will remove an address group from the device. If deleting an address group on a
// Panorama device, then specify the given device-group name as the last parameter.
func (p *PaloAlto) DeleteAddressGroup(name string, devicegroup ...string) error {
	var xpath string
	var reqError requestError
	r := rested.NewRequest()

	if p.DeviceType == "panos" {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys1']/address-group/entry[@name='%s']", name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) > 0 {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/device-group/entry[@name='%s']/address-group/entry[@name='%s']", devicegroup[0], name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) <= 0 {
		return errors.New("you must specify a device-group when connected to a Panorama device")
	}

	query := map[string]string{
		"type":   "config",
		"action": "delete",
		"xpath":  xpath,
		"key":    p.Key,
	}

	resp := r.Send("get", p.URI, nil, nil, query)
	if resp.Error != nil {
		return resp.Error
	}

	if err := xml.Unmarshal(resp.Body, &reqError); err != nil {
		return err
	}

	if reqError.Status != "success" {
		return fmt.Errorf("error code %s: %s", reqError.Code, errorCodes[reqError.Code])
	}

	return nil
}
<commit_msg>Added device-group support<commit_after>package panos

import (
	"encoding/xml"
	"errors"
	"fmt"
	"strings"

	"github.com/scottdware/go-rested"
)

// AddressObjects contains a slice of all address objects.
type AddressObjects struct {
	XMLName   xml.Name  `xml:"response"`
	Status    string    `xml:"status,attr"`
	Code      string    `xml:"code,attr"`
	Addresses []Address `xml:"result>address>entry"`
}

// Address contains information about each individual address object.
type Address struct {
	Name        string `xml:"name,attr"`
	IPAddress   string `xml:"ip-netmask,omitempty"`
	IPRange     string `xml:"ip-range,omitempty"`
	FQDN        string `xml:"fqdn,omitempty"`
	Description string `xml:"description,omitempty"`
}

// AddressGroups contains a slice of all address groups.
type AddressGroups struct {
	Groups []AddressGroup
}

// AddressGroup contains information about each individual address group.
type AddressGroup struct {
	Name          string
	Type          string
	Members       []string
	DynamicFilter string
	Description   string
}

// xmlAddressGroups is used for parsing of all address groups.
type xmlAddressGroups struct {
	XMLName xml.Name          `xml:"response"`
	Status  string            `xml:"status,attr"`
	Code    string            `xml:"code,attr"`
	Groups  []xmlAddressGroup `xml:"result>address-group>entry"`
}

// xmlAddressGroup is used for parsing each individual address group.
type xmlAddressGroup struct {
	Name          string   `xml:"name,attr"`
	Members       []string `xml:"static>member,omitempty"`
	DynamicFilter string   `xml:"dynamic>filter,omitempty"`
	Description   string   `xml:"description,omitempty"`
}

// Addresses returns information about all of the address objects. You can (optionally) specify a device-group
// when ran against a Panorama device. If no device-group is specified, then all objects are returned.
func (p *PaloAlto) Addresses(devicegroup ...string) (*AddressObjects, error) {
	var addrs AddressObjects
	xpath := "/config/devices/entry//address"
	r := rested.NewRequest()

	if p.DeviceType != "panorama" && len(devicegroup) > 0 {
		return nil, errors.New("you must be connected to a Panorama device when specifying a device-group")
	}

	if p.DeviceType == "panos" && p.Panorama == true {
		xpath = "/config/panorama//address"
	}

	if p.DeviceType == "panos" && p.Panorama == false {
		xpath = "/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys1']/address"
	}

	if p.DeviceType == "panorama" && len(devicegroup) > 0 {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/device-group/entry[@name='%s']/address", devicegroup[0])
	}

	query := map[string]string{
		"type":   "config",
		"action": "get",
		"xpath":  xpath,
		"key":    p.Key,
	}
	addrData := r.Send("get", p.URI, nil, headers, query)

	if err := xml.Unmarshal(addrData.Body, &addrs); err != nil {
		return nil, err
	}

	if addrs.Status != "success" {
		return nil, fmt.Errorf("error code %s: %s", addrs.Code, errorCodes[addrs.Code])
	}

	return &addrs, nil
}

// AddressGroups returns information about all of the address groups. You can (optionally) specify a device-group
// when ran against a Panorama device. If no device-group is specified, then all address groups are returned.
func (p *PaloAlto) AddressGroups(devicegroup ...string) (*AddressGroups, error) {
	var parsedGroups xmlAddressGroups
	var groups AddressGroups
	xpath := "/config/devices/entry//address-group"
	r := rested.NewRequest()

	if p.DeviceType != "panorama" && len(devicegroup) > 0 {
		return nil, errors.New("you must be connected to a Panorama device when specifying a device-group")
	}

	if p.DeviceType == "panos" && p.Panorama == true {
		xpath = "/config/panorama//address-group"
	}

	if p.DeviceType == "panos" && p.Panorama == false {
		xpath = "/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys1']/address-group"
	}

	if p.DeviceType == "panorama" && len(devicegroup) > 0 {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/device-group/entry[@name='%s']/address-group", devicegroup[0])
	}

	query := map[string]string{
		"type":   "config",
		"action": "get",
		"xpath":  xpath,
		"key":    p.Key,
	}
	groupData := r.Send("get", p.URI, nil, headers, query)

	if err := xml.Unmarshal(groupData.Body, &parsedGroups); err != nil {
		return nil, err
	}

	if parsedGroups.Status != "success" {
		return nil, fmt.Errorf("error code %s: %s", parsedGroups.Code, errorCodes[parsedGroups.Code])
	}

	for _, g := range parsedGroups.Groups {
		gname := g.Name
		gtype := "Static"
		gmembers := g.Members
		gfilter := strings.TrimSpace(g.DynamicFilter)
		gdesc := g.Description

		if g.DynamicFilter != "" {
			gtype = "Dynamic"
		}

		groups.Groups = append(groups.Groups, AddressGroup{Name: gname, Type: gtype, Members: gmembers, DynamicFilter: gfilter, Description: gdesc})
	}

	return &groups, nil
}

// CreateAddress will add a new address object to the device. addrtype should be one of: ip, range, or fqdn. If creating
// an address object on a Panorama device, then specify the given device-group name as the last parameter.
func (p *PaloAlto) CreateAddress(name, addrtype, address, description string, devicegroup ...string) error {
	var xmlBody string
	var xpath string
	var reqError requestError
	r := rested.NewRequest()

	switch addrtype {
	case "ip":
		xmlBody = fmt.Sprintf("<ip-netmask>%s</ip-netmask>", address)
	case "range":
		xmlBody = fmt.Sprintf("<ip-range>%s</ip-range>", address)
	case "fqdn":
		xmlBody = fmt.Sprintf("<fqdn>%s</fqdn>", address)
	}

	if description != "" {
		xmlBody += fmt.Sprintf("<description>%s</description>", description)
	}

	if p.DeviceType == "panos" {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys1']/address/entry[@name='%s']", name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) > 0 {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/device-group/entry[@name='%s']/address/entry[@name='%s']", devicegroup[0], name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) <= 0 {
		return errors.New("you must specify a device-group when connected to a Panorama device")
	}

	query := map[string]string{
		"type":    "config",
		"action":  "set",
		"xpath":   xpath,
		"element": xmlBody,
		"key":     p.Key,
	}

	resp := r.Send("post", p.URI, nil, nil, query)
	if resp.Error != nil {
		return resp.Error
	}

	if err := xml.Unmarshal(resp.Body, &reqError); err != nil {
		return err
	}

	if reqError.Status != "success" {
		return fmt.Errorf("error code %s: %s", reqError.Code, errorCodes[reqError.Code])
	}

	return nil
}

// CreateStaticGroup will create a new static address group on the device. You can specify multiple members
// by separating them with a comma, i.e. "web-server1, web-server2". If creating an address group on
// a Panorama device, then specify the given device-group name as the last parameter.
func (p *PaloAlto) CreateStaticGroup(name, members, description string, devicegroup ...string) error {
	var xmlBody string
	var xpath string
	var reqError requestError
	r := rested.NewRequest()
	m := strings.Split(members, ",")

	if members == "" {
		return errors.New("you cannot create a static address group without any members")
	}

	xmlBody = "<static>"
	for _, member := range m {
		xmlBody += fmt.Sprintf("<member>%s</member>", strings.TrimSpace(member))
	}
	xmlBody += "</static>"

	if description != "" {
		xmlBody += fmt.Sprintf("<description>%s</description>", description)
	}

	if p.DeviceType == "panos" {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys1']/address-group/entry[@name='%s']", name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) > 0 {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/device-group/entry[@name='%s']/address-group/entry[@name='%s']", devicegroup[0], name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) <= 0 {
		return errors.New("you must specify a device-group when connected to a Panorama device")
	}

	query := map[string]string{
		"type":    "config",
		"action":  "set",
		"xpath":   xpath,
		"element": xmlBody,
		"key":     p.Key,
	}

	resp := r.Send("post", p.URI, nil, nil, query)
	if resp.Error != nil {
		return resp.Error
	}

	if err := xml.Unmarshal(resp.Body, &reqError); err != nil {
		return err
	}

	if reqError.Status != "success" {
		return fmt.Errorf("error code %s: %s", reqError.Code, errorCodes[reqError.Code])
	}

	return nil
}

// CreateDynamicGroup will create a new dynamic address group on the device. The filter must be written like so:
// 'vm-servers' and 'some tag' or 'pcs' - using the tags as the match criteria. If creating an address group on a
// Panorama device, then specify the given device-group name as the last parameter.
func (p *PaloAlto) CreateDynamicGroup(name, criteria, description string, devicegroup ...string) error {
	xmlBody := fmt.Sprintf("<dynamic><filter>%s</filter></dynamic>", criteria)
	var xpath string
	var reqError requestError
	r := rested.NewRequest()

	if criteria == "" {
		return errors.New("you cannot create a dynamic address group without any filter")
	}

	if description != "" {
		xmlBody += fmt.Sprintf("<description>%s</description>", description)
	}

	if p.DeviceType == "panos" {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys1']/address-group/entry[@name='%s']", name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) > 0 {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/device-group/entry[@name='%s']/address-group/entry[@name='%s']", devicegroup[0], name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) <= 0 {
		return errors.New("you must specify a device-group when connected to a Panorama device")
	}

	query := map[string]string{
		"type":    "config",
		"action":  "set",
		"xpath":   xpath,
		"element": xmlBody,
		"key":     p.Key,
	}

	resp := r.Send("post", p.URI, nil, nil, query)
	if resp.Error != nil {
		return resp.Error
	}

	if err := xml.Unmarshal(resp.Body, &reqError); err != nil {
		return err
	}

	if reqError.Status != "success" {
		return fmt.Errorf("error code %s: %s", reqError.Code, errorCodes[reqError.Code])
	}

	return nil
}

// DeleteAddress will remove an address object from the device. If deleting an address object on a
// Panorama device, then specify the given device-group name as the last parameter.
func (p *PaloAlto) DeleteAddress(name string, devicegroup ...string) error {
	var xpath string
	var reqError requestError
	r := rested.NewRequest()

	if p.DeviceType == "panos" {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys1']/address/entry[@name='%s']", name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) > 0 {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/device-group/entry[@name='%s']/address/entry[@name='%s']", devicegroup[0], name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) <= 0 {
		return errors.New("you must specify a device-group when connected to a Panorama device")
	}

	query := map[string]string{
		"type":   "config",
		"action": "delete",
		"xpath":  xpath,
		"key":    p.Key,
	}

	resp := r.Send("get", p.URI, nil, nil, query)
	if resp.Error != nil {
		return resp.Error
	}

	if err := xml.Unmarshal(resp.Body, &reqError); err != nil {
		return err
	}

	if reqError.Status != "success" {
		return fmt.Errorf("error code %s: %s", reqError.Code, errorCodes[reqError.Code])
	}

	return nil
}

// DeleteAddressGroup will remove an address group from the device. If deleting an address group on a
// Panorama device, then specify the given device-group name as the last parameter.
func (p *PaloAlto) DeleteAddressGroup(name string, devicegroup ...string) error {
	var xpath string
	var reqError requestError
	r := rested.NewRequest()

	if p.DeviceType == "panos" {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys1']/address-group/entry[@name='%s']", name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) > 0 {
		xpath = fmt.Sprintf("/config/devices/entry[@name='localhost.localdomain']/device-group/entry[@name='%s']/address-group/entry[@name='%s']", devicegroup[0], name)
	}

	if p.DeviceType == "panorama" && len(devicegroup) <= 0 {
		return errors.New("you must specify a device-group when connected to a Panorama device")
	}

	query := map[string]string{
		"type":   "config",
		"action": "delete",
		"xpath":  xpath,
		"key":    p.Key,
	}

	resp := r.Send("get", p.URI, nil, nil, query)
	if resp.Error != nil {
		return resp.Error
	}

	if err := xml.Unmarshal(resp.Body, &reqError); err != nil {
		return err
	}

	if reqError.Status != "success" {
		return fmt.Errorf("error code %s: %s", reqError.Code, errorCodes[reqError.Code])
	}

	return nil
}
<|endoftext|>


<commit_before>package btcplex

import (
    "encoding/json"
    "fmt"
    "strings"
    "strconv"
    "github.com/garyburd/redigo/redis"
)

type AddressData struct {
	Address       string `json:"address"`
	TxCnt         uint64 `json:"n_tx"`
	ReceivedCnt   uint64 `json:"-"`
	SentCnt       uint64 `json:"-"`
	TotalReceived uint64 `json:"total_received"`
	TotalSent     uint64 `json:"total_sent"`
	FinalBalance  uint64 `json:"final_balance"`
	Txs           []*Tx  `json:"txs"`
}

func GetAddress(rpool *redis.Pool, address string) (addressdata *AddressData, err error) {
    c := rpool.Get()
    defer c.Close()

    addressdata = new(AddressData)
    txsindex := map[string]interface{}{}

    txreceived := map[string]interface{}{}
    txsent := map[string]interface{}{}

    txs := []*Tx{}

    zkey := fmt.Sprintf("addr:%v", address)
    cnt, _ := redis.Int(c.Do("ZCARD", zkey))
    data, _ := redis.Strings(c.Do("ZREVRANGE", zkey, 0, cnt))

    for _, txd := range data {
        txds := strings.Split(txd, ":")
        chash := txds[1]
        _, inindex := txsindex[chash]
        if !inindex {
            tx, _ := GetTx(rpool, txds[1])
            txsindex[chash] = tx
        }
    }

    totalreceived := uint64(0)
    totalsent := uint64(0)

    for _, tx := range txsindex {
        ctx := tx.(*Tx)
        txaddressinfo := new(TxAddressInfo)
        for _, txi := range ctx.TxIns {
            if txi.PrevOut.Address == address {
                txaddressinfo.InTxIn = true
                txaddressinfo.Value -= int64(txi.PrevOut.Value)
                totalsent += txi.PrevOut.Value
                _, inindex := txsent[ctx.Hash]
                if !inindex {
                    txsent[ctx.Hash] = true
                }
            }
        }
        for _, txo := range ctx.TxOuts {
            if txo.Addr == address {
                txaddressinfo.InTxOut = true
                txaddressinfo.Value += int64(txo.Value)
                totalreceived += txo.Value
                _, inindex := txreceived[ctx.Hash]
                if !inindex {
                    txreceived[ctx.Hash] = true
                }
            }
        }
        ctx.TxAddressInfo = txaddressinfo
        txs = append(txs, ctx)
    }

    finalbalance := totalreceived - totalsent

    //By(TxBlockTime).Sort(txs)

    addressdata.Txs = txs
    addressdata.FinalBalance = finalbalance
    addressdata.TotalSent = totalsent
    addressdata.TotalReceived = totalreceived
    addressdata.TxCnt = uint64(len(txs))
    addressdata.Address = address
    addressdata.SentCnt = uint64(len(txsent))
    addressdata.ReceivedCnt = uint64(len(txreceived))

    return
}

// Return the block time at which the address first appeared
func AddressFirstSeen(rpool *redis.Pool, address string) (firstseen uint64, err error) {
    c := rpool.Get()
    defer c.Close()

    zkey := fmt.Sprintf("addr:%v", address)
    data, _ := redis.Strings(c.Do("ZREVRANGE", zkey, 0, 0, "withscores"))
    txoutblocktime, _ := strconv.Atoi(data[1])
    firstseen = uint64(txoutblocktime)
    return
}

func GetReceivedByAddress(rpool *redis.Pool, address string) (total uint64, err error) {
    c := rpool.Get()
    defer c.Close()

    zkey := fmt.Sprintf("addr:%v", address)
    cnt, _ := redis.Int(c.Do("ZCARD", zkey))
    data, _ := redis.Strings(c.Do("ZREVRANGE", zkey, 0, cnt))

    for _, txd := range data {
        if strings.Contains(txd, "txo:") {
            ctxo := new(TxOut)
            txoutjson, _ := redis.String(c.Do("GET", txd))
            err = json.Unmarshal([]byte(txoutjson), ctxo)
            total += uint64(ctxo.Value)
        }
    }

    return
}

func GetSentByAddress(rpool *redis.Pool, address string) (total uint64, err error) {
    c := rpool.Get()
    defer c.Close()

    zkey := fmt.Sprintf("addr:%v", address)
    cnt, _ := redis.Int(c.Do("ZCARD", zkey))
    data, _ := redis.Strings(c.Do("ZREVRANGE", zkey, 0, cnt))

    for _, txd := range data {
        if strings.Contains(txd, "txi:") {
            ctxi := new(TxIn)
            txinjson, _ := redis.String(c.Do("GET", txd))
            err = json.Unmarshal([]byte(txinjson), ctxi)
            total += uint64(ctxi.PrevOut.Value)
        }
    }

    return
}

func AddressBalance(rpool *redis.Pool, address string) (balance uint64, err error) {
    c := rpool.Get()
    defer c.Close()

    zkey := fmt.Sprintf("addr:%v", address)
    cnt, _ := redis.Int(c.Do("ZCARD", zkey))
    data, _ := redis.Strings(c.Do("ZREVRANGE", zkey, 0, cnt))

    for _, txd := range data {
        if strings.Contains(txd, "txo:") {
            ctxo := new(TxOut)
            txoutjson, _ := redis.String(c.Do("GET", txd))
            err = json.Unmarshal([]byte(txoutjson), ctxo)
            balance += uint64(ctxo.Value)
        }
        if strings.Contains(txd, "txi:") {
            ctxi := new(TxIn)
            txinjson, _ := redis.String(c.Do("GET", txd))
            err = json.Unmarshal([]byte(txinjson), ctxi)
            balance -= uint64(ctxi.PrevOut.Value)
        }
    }

    return
}
<commit_msg>Update GetAddress logic using zset<commit_after>package btcplex

import (
    "fmt"
    "strconv"
    "github.com/garyburd/redigo/redis"
)

type AddressData struct {
	Address       string `json:"address"`
	TxCnt         uint64 `json:"n_tx"`
	ReceivedCnt   uint64 `json:"-"`
	SentCnt       uint64 `json:"-"`
	TotalReceived uint64 `json:"total_received"`
	TotalSent     uint64 `json:"total_sent"`
	FinalBalance  uint64 `json:"final_balance"`
	Txs           []*Tx  `json:"txs"`
}

type AddressHash struct {
    TotalSent       int `redis:"ts"`
    TotalReceived   int `redis:"tr"`
}

func GetAddress(rpool *redis.Pool, address string) (addressdata *AddressData, err error) {
    c := rpool.Get()
    defer c.Close()

    addressdata = new(AddressData)
    txs1 := []*Tx{}
    txs := []*Tx{}

    zkey := fmt.Sprintf("addr:%v", address)
    txscnt, _ := redis.Int(c.Do("ZCARD", zkey))
    data, _ := redis.Strings(c.Do("ZREVRANGE", zkey, 0, 50))

    addressh := new(AddressHash)
    v, err := redis.Values(c.Do("HGETALL", fmt.Sprintf("addr:%v:h", address)))
    if err != nil {
        panic(err)
    }
    if err := redis.ScanStruct(v, addressh); err != nil {
        panic(err)
    }

    for _, txd := range data {
        tx, _ := GetTx(rpool, txd)
        txs1 = append(txs1, tx)
    }

    totalreceived := uint64(addressh.TotalReceived)
    totalsent := uint64(addressh.TotalSent)
    finalbalance := totalreceived - totalreceived
    sentcnt, _ := redis.Int(c.Do("ZCARD", fmt.Sprintf("addr:%v:sent", address)))
    receivedcnt, _ := redis.Int(c.Do("ZCARD", fmt.Sprintf("addr:%v:received", address)))

    for _, ctx := range txs1 {
        txaddressinfo := new(TxAddressInfo)
        for _, txi := range ctx.TxIns {
            if txi.PrevOut.Address == address {
                txaddressinfo.InTxIn = true
                txaddressinfo.Value -= int64(txi.PrevOut.Value)
            }
        }
        for _, txo := range ctx.TxOuts {
            if txo.Addr == address {
                txaddressinfo.InTxOut = true
                txaddressinfo.Value += int64(txo.Value)
            }
        }
        ctx.TxAddressInfo = txaddressinfo
        txs = append(txs, ctx)
    }

    //By(TxBlockTime).Sort(txs)

    addressdata.Txs = txs
    addressdata.FinalBalance = finalbalance
    addressdata.TotalSent = totalsent
    addressdata.TotalReceived = totalreceived
    addressdata.TxCnt = uint64(txscnt)
    addressdata.Address = address
    addressdata.SentCnt = uint64(sentcnt)
    addressdata.ReceivedCnt = uint64(receivedcnt)

    return
}

// Return the block time at which the address first appeared
func AddressFirstSeen(rpool *redis.Pool, address string) (firstseen uint64, err error) {
    c := rpool.Get()
    defer c.Close()

    zkey := fmt.Sprintf("addr:%v", address)
    data, _ := redis.Strings(c.Do("ZRANGE", zkey, 0, 0, "withscores"))
    txoutblocktime, _ := strconv.Atoi(data[1])
    firstseen = uint64(txoutblocktime)
    return
}

func GetReceivedByAddress(rpool *redis.Pool, address string) (total uint64, err error) {
    c := rpool.Get()
    defer c.Close()

    addressh := new(AddressHash)
    v, err := redis.Values(c.Do("HGETALL", fmt.Sprintf("addr:%v:h", address)))
    if err != nil {
        panic(err)
    }
    if err := redis.ScanStruct(v, addressh); err != nil {
        panic(err)
    }
    total = uint64(addressh.TotalReceived)
    return
}

func GetSentByAddress(rpool *redis.Pool, address string) (total uint64, err error) {
    c := rpool.Get()
    defer c.Close()

    addressh := new(AddressHash)
    v, err := redis.Values(c.Do("HGETALL", fmt.Sprintf("addr:%v:h", address)))
    if err != nil {
        panic(err)
    }
    if err := redis.ScanStruct(v, addressh); err != nil {
        panic(err)
    }
    total = uint64(addressh.TotalSent)
    return
}

func AddressBalance(rpool *redis.Pool, address string) (balance uint64, err error) {
    c := rpool.Get()
    defer c.Close()

    addressh := new(AddressHash)
    v, err := redis.Values(c.Do("HGETALL", fmt.Sprintf("addr:%v:h", address)))
    if err != nil {
        panic(err)
    }
    if err := redis.ScanStruct(v, addressh); err != nil {
        panic(err)
    }
    balance = uint64(addressh.TotalReceived - addressh.TotalSent)
    return
}
<|endoftext|>


<commit_before>package knowledge

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"nli-go/lib/common"
	"os"
	"path/filepath"
	"strconv"
)

func (factBase *SparqlFactBase) doCachedQuery(query string) sparqlResponse {

	sparqlResponse := sparqlResponse{}
	queryHash := fmt.Sprintf("%x", md5.Sum([]byte(query)))
	executable, _ := os.Executable()
	executablePath := filepath.Dir(executable)
	cacheDir := common.AbsolutePath(executablePath, "cache")
	queryCachePath := cacheDir + "/" + queryHash + ".json"

	_, err := os.Stat(cacheDir)
	if os.IsNotExist(err) {
		err = os.MkdirAll(cacheDir, 0777)
		if err != nil {
			factBase.log.AddError("Error creating cache dir " + cacheDir + " (" + err.Error() + ")")
			return sparqlResponse
		}
	}

	// check if cache exists
	_, err = os.Stat(queryCachePath)
	if os.IsNotExist(err) {
		sparqlResponse = factBase.populateCache(query, queryCachePath)
	} else {
		factBase.log.AddProduction("SPARQL", query + " (from cache)" + strconv.Itoa(len(sparqlResponse.Results.Bindings)) + " results)")

		sparqlResponse = factBase.readFromCache(queryCachePath)
	}

	return sparqlResponse
}

func (factBase *SparqlFactBase) populateCache(query string, queryCachePath string) sparqlResponse {

	sparqlResponse := factBase.callSparql(query)

	// put them in the cache
	jsonBytes, _ := json.Marshal(sparqlResponse)
	err := common.WriteFile(queryCachePath, string(jsonBytes))
	if err != nil {
		factBase.log.AddError("Error creating cache file: " + err.Error())
		return sparqlResponse
	}

	return sparqlResponse
}

func (factBase *SparqlFactBase) readFromCache(queryCachePath string) sparqlResponse {

	sparqlResponse := sparqlResponse{}
	jsonString, err := common.ReadFile(queryCachePath)

	if err != nil {
		factBase.log.AddError("Error reading cache file: " + err.Error())
		return sparqlResponse
	}

	err = json.Unmarshal([]byte(jsonString), &sparqlResponse)
	if err != nil {
		factBase.log.AddError("Error parsing SPARQL response cache: " + err.Error())
		return sparqlResponse
	}

	return sparqlResponse
}
<commit_msg>space<commit_after>package knowledge

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"nli-go/lib/common"
	"os"
	"path/filepath"
	"strconv"
)

func (factBase *SparqlFactBase) doCachedQuery(query string) sparqlResponse {

	sparqlResponse := sparqlResponse{}
	queryHash := fmt.Sprintf("%x", md5.Sum([]byte(query)))
	executable, _ := os.Executable()
	executablePath := filepath.Dir(executable)
	cacheDir := common.AbsolutePath(executablePath, "cache")
	queryCachePath := cacheDir + "/" + queryHash + ".json"

	_, err := os.Stat(cacheDir)
	if os.IsNotExist(err) {
		err = os.MkdirAll(cacheDir, 0777)
		if err != nil {
			factBase.log.AddError("Error creating cache dir " + cacheDir + " (" + err.Error() + ")")
			return sparqlResponse
		}
	}

	// check if cache exists
	_, err = os.Stat(queryCachePath)
	if os.IsNotExist(err) {
		sparqlResponse = factBase.populateCache(query, queryCachePath)
	} else {
		factBase.log.AddProduction("SPARQL", query + " (from cache) " + strconv.Itoa(len(sparqlResponse.Results.Bindings)) + " results)")

		sparqlResponse = factBase.readFromCache(queryCachePath)
	}

	return sparqlResponse
}

func (factBase *SparqlFactBase) populateCache(query string, queryCachePath string) sparqlResponse {

	sparqlResponse := factBase.callSparql(query)

	// put them in the cache
	jsonBytes, _ := json.Marshal(sparqlResponse)
	err := common.WriteFile(queryCachePath, string(jsonBytes))
	if err != nil {
		factBase.log.AddError("Error creating cache file: " + err.Error())
		return sparqlResponse
	}

	return sparqlResponse
}

func (factBase *SparqlFactBase) readFromCache(queryCachePath string) sparqlResponse {

	sparqlResponse := sparqlResponse{}
	jsonString, err := common.ReadFile(queryCachePath)

	if err != nil {
		factBase.log.AddError("Error reading cache file: " + err.Error())
		return sparqlResponse
	}

	err = json.Unmarshal([]byte(jsonString), &sparqlResponse)
	if err != nil {
		factBase.log.AddError("Error parsing SPARQL response cache: " + err.Error())
		return sparqlResponse
	}

	return sparqlResponse
}
<|endoftext|>


<commit_before>package core

// Arguments represents a structured set of arguments passed to a predicate.
// It allows destructive operations to internal properties because it is
// guaranteed by Thunks that arguments objects are never reused as a function
// call creates a Thunk.
type Arguments struct {
	positionals   []*Thunk
	expandedList  *Thunk
	keywords      []KeywordArgument
	expandedDicts []*Thunk
}

// NewArguments creates a new Arguments.
func NewArguments(
	ps []PositionalArgument,
	ks []KeywordArgument,
	ds []*Thunk) Arguments {
	ts := make([]*Thunk, 0, len(ps))
	l := (*Thunk)(nil)

	for i, p := range ps {
		if p.expanded {
			l = mergePositionalArguments(ps[i:])
			break
		}

		ts = append(ts, p.value)
	}

	return Arguments{ts, l, ks, ds}
}

// NewPositionalArguments creates an Arguments which consists of unexpanded
// positional arguments.
func NewPositionalArguments(ts ...*Thunk) Arguments {
	return Arguments{ts, nil, nil, nil}
}

func mergePositionalArguments(ps []PositionalArgument) *Thunk {
	t := EmptyList

	// Optimization for a common pattern of (func a b c ... ..xs).
	// Note that Merge is O(n) but Prepend is O(1).
	if last := len(ps) - 1; ps[last].expanded {
		t = ps[last].value
		ps = ps[:last]
	}

	for i := len(ps) - 1; i >= 0; i-- {
		p := ps[i]

		if p.expanded {
			t = PApp(Merge, p.value, t)
		} else {
			t = PApp(Prepend, p.value, t)
		}
	}

	return t
}

func (args *Arguments) nextPositional() *Thunk {
	if len(args.positionals) != 0 {
		t := args.positionals[0]
		args.positionals = args.positionals[1:]
		return t
	}

	if args.expandedList == nil {
		return nil
	}

	l := args.expandedList
	args.expandedList = PApp(Rest, l)
	return PApp(First, l)
}

func (args *Arguments) restPositionals() *Thunk {
	ts := args.positionals
	l := args.expandedList
	args.positionals = nil
	args.expandedList = nil

	if l == nil {
		return NewList(ts...)
	} else if len(ts) == 0 {
		return l
	}

	return PApp(Prepend, append(ts, l)...)
}

func (args *Arguments) searchKeyword(s string) *Thunk {
	for i, k := range args.keywords {
		if s == k.name {
			args.keywords = append(args.keywords[:i], args.keywords[i+1:]...)
			return k.value
		}
	}

	for i, t := range args.expandedDicts {
		v := t.Eval()
		d, ok := v.(DictionaryType)

		if !ok {
			return NotDictionaryError(v)
		}

		k := StringType(s)

		// Using DictionaryType.{Search,Remove} methods is safe here
		// because the key is always StringType.
		if v, ok := d.Search(k); ok {
			args.expandedDicts = append([]*Thunk{}, args.expandedDicts...)
			args.expandedDicts[i] = Normal(d.Remove(k))
			return v
		}
	}

	return nil
}

func (args *Arguments) restKeywords() *Thunk {
	ks := args.keywords
	ds := args.expandedDicts
	args.keywords = nil
	args.expandedDicts = nil

	d := emptyDictionary

	for _, k := range ks {
		v := d.insert(StringType(k.name), k.value)
		var ok bool
		d, ok = v.(DictionaryType)

		if !ok {
			return NotDictionaryError(v)
		}
	}

	return PApp(Merge, append([]*Thunk{Normal(d)}, ds...)...)
}

// Merge merges 2 sets of arguments into one.
func (args Arguments) Merge(old Arguments) Arguments {
	ks := append(args.keywords, old.keywords...)
	ds := append(args.expandedDicts, old.expandedDicts...)

	if args.expandedList == nil {
		return Arguments{append(args.positionals, old.positionals...), old.expandedList, ks, ds}
	}

	l := EmptyList

	if old.expandedList != nil {
		l = old.expandedList
	}

	return Arguments{
		args.positionals,
		PApp(Merge, args.expandedList, PApp(Prepend, append(old.positionals, l)...)),
		ks,
		ds,
	}
}

func (args Arguments) empty() *Thunk {
	if len(args.positionals) > 0 {
		return argumentError("%d positional arguments are left", len(args.positionals))
	}

	// Testing args.expandedList is impossible because we cannot know its length
	// without evaluating it.

	n := 0

	for _, t := range args.expandedDicts {
		v := t.Eval()
		d, ok := v.(DictionaryType)

		if !ok {
			return NotDictionaryError(v)
		}

		n += d.Size()
	}

	if n != 0 || args.keywords != nil && len(args.keywords) > 0 {
		return argumentError("%d keyword arguments are left", len(args.keywords)+n)
	}

	return nil
}
<commit_msg>Refactor Arguments.Merge<commit_after>package core

// Arguments represents a structured set of arguments passed to a predicate.
// It allows destructive operations to internal properties because it is
// guaranteed by Thunks that arguments objects are never reused as a function
// call creates a Thunk.
type Arguments struct {
	positionals   []*Thunk
	expandedList  *Thunk
	keywords      []KeywordArgument
	expandedDicts []*Thunk
}

// NewArguments creates a new Arguments.
func NewArguments(
	ps []PositionalArgument,
	ks []KeywordArgument,
	ds []*Thunk) Arguments {
	ts := make([]*Thunk, 0, len(ps))
	l := (*Thunk)(nil)

	for i, p := range ps {
		if p.expanded {
			l = mergePositionalArguments(ps[i:])
			break
		}

		ts = append(ts, p.value)
	}

	return Arguments{ts, l, ks, ds}
}

// NewPositionalArguments creates an Arguments which consists of unexpanded
// positional arguments.
func NewPositionalArguments(ts ...*Thunk) Arguments {
	return Arguments{ts, nil, nil, nil}
}

func mergePositionalArguments(ps []PositionalArgument) *Thunk {
	t := EmptyList

	// Optimization for a common pattern of (func a b c ... ..xs).
	// Note that Merge is O(n) but Prepend is O(1).
	if last := len(ps) - 1; ps[last].expanded {
		t = ps[last].value
		ps = ps[:last]
	}

	for i := len(ps) - 1; i >= 0; i-- {
		p := ps[i]

		if p.expanded {
			t = PApp(Merge, p.value, t)
		} else {
			t = Normal(cons(p.value, t))
		}
	}

	return t
}

func (args *Arguments) nextPositional() *Thunk {
	if len(args.positionals) != 0 {
		t := args.positionals[0]
		args.positionals = args.positionals[1:]
		return t
	}

	if args.expandedList == nil {
		return nil
	}

	l := args.expandedList
	args.expandedList = PApp(Rest, l)
	return PApp(First, l)
}

func (args *Arguments) restPositionals() *Thunk {
	ts := args.positionals
	l := args.expandedList
	args.positionals = nil
	args.expandedList = nil

	if l == nil {
		return NewList(ts...)
	} else if len(ts) == 0 {
		return l
	}

	return PApp(Prepend, append(ts, l)...)
}

func (args *Arguments) searchKeyword(s string) *Thunk {
	for i, k := range args.keywords {
		if s == k.name {
			args.keywords = append(args.keywords[:i], args.keywords[i+1:]...)
			return k.value
		}
	}

	for i, t := range args.expandedDicts {
		v := t.Eval()
		d, ok := v.(DictionaryType)

		if !ok {
			return NotDictionaryError(v)
		}

		k := StringType(s)

		// Using DictionaryType.{Search,Remove} methods is safe here
		// because the key is always StringType.
		if v, ok := d.Search(k); ok {
			args.expandedDicts = append([]*Thunk{}, args.expandedDicts...)
			args.expandedDicts[i] = Normal(d.Remove(k))
			return v
		}
	}

	return nil
}

func (args *Arguments) restKeywords() *Thunk {
	ks := args.keywords
	ds := args.expandedDicts
	args.keywords = nil
	args.expandedDicts = nil

	d := emptyDictionary

	for _, k := range ks {
		v := d.insert(StringType(k.name), k.value)
		var ok bool
		d, ok = v.(DictionaryType)

		if !ok {
			return NotDictionaryError(v)
		}
	}

	return PApp(Merge, append([]*Thunk{Normal(d)}, ds...)...)
}

// Merge merges 2 sets of arguments into one.
func (args Arguments) Merge(old Arguments) Arguments {
	ks := append(args.keywords, old.keywords...)
	ds := append(args.expandedDicts, old.expandedDicts...)

	if args.expandedList == nil {
		return Arguments{append(args.positionals, old.positionals...), old.expandedList, ks, ds}
	}

	l := EmptyList

	if old.expandedList != nil {
		l = old.expandedList
	}

	for i := len(old.positionals) - 1; i >= 0; i-- {
		l = Normal(cons(old.positionals[i], l))
	}

	return Arguments{
		args.positionals,
		PApp(Merge, args.expandedList, l),
		ks,
		ds,
	}
}

func (args Arguments) empty() *Thunk {
	if len(args.positionals) > 0 {
		return argumentError("%d positional arguments are left", len(args.positionals))
	}

	// Testing args.expandedList is impossible because we cannot know its length
	// without evaluating it.

	n := 0

	for _, t := range args.expandedDicts {
		v := t.Eval()
		d, ok := v.(DictionaryType)

		if !ok {
			return NotDictionaryError(v)
		}

		n += d.Size()
	}

	if n != 0 || args.keywords != nil && len(args.keywords) > 0 {
		return argumentError("%d keyword arguments are left", len(args.keywords)+n)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"math/rand"
	"testing"
)

// var testFileSizes = []int{0, 100, 200, 500, 1024, 7666, 1 << 20, 1 << 24}
var testFileSizes = []int{1 << 20}

func TestFile(t *testing.T) {
	rnd := rand.New(rand.NewSource(23))

	for i, filesize := range testFileSizes {
		f := NewFile(23, filesize, 23)
		buf, err := f.ReadAll()
		if err != nil {
			t.Errorf("test %d: error %v", i, err)
			continue
		}

		if len(buf) != filesize {
			t.Errorf("test %d: invalid number of bytes returned, want %d, got %d", i, filesize, len(buf))
			continue
		}

		buf2 := make([]byte, filesize)
		n, err := f.ReadAt(buf2, 0)

		if err != nil {
			t.Errorf("test %d: error %v", i, err)
			continue
		}

		if n != filesize {
			t.Errorf("test %d: invalid number of bytes returned, want %d, got %d", i, filesize, n)
			continue
		}

		if !bytes.Equal(buf, buf2) {
			t.Errorf("test %d: wrong bytes returned", i)
			fmt.Printf("test %d: wrong bytes returned\n", i)
			continue
		}

		if filesize == 0 {
			n, err = f.ReadAt(buf2, 23)
			if n != 0 {
				t.Errorf("test %d: to read 0 bytes from empty file, got %d", i, n)
			}

			if err == nil {
				t.Errorf("test %d: expected error for empty file at offset 23 not found", i)
			}

			continue
		}

		for j := 0; j < 10; j++ {
			fmt.Printf("\n=========================================\n")

			o := rnd.Intn(filesize)
			l := rnd.Intn(filesize - o)

			fmt.Printf("  filesize %v, len %v, off %v\n", filesize, l, o)

			readatBuf := make([]byte, l)
			n, err = f.ReadAt(readatBuf, int64(o))
			fmt.Printf("n %v, l %v, o %v\n", n, l, o)
			if err != nil {
				t.Errorf("test %d/%d: reading len %v bytes at offset %v failed: %v", i, j, l, o, err)
				continue
			}

			if n != l {
				t.Errorf("test %d/%d: want %d bytes, got %d", i, j, l, n)
				fmt.Printf("test %d/%d: want %d bytes, got %d\n", i, j, l, n)
				continue
			}

			if !bytes.Equal(readatBuf, buf[o:o+l]) {
				fmt.Printf("------------------ test %d/%d: wrong bytes returned at offset %v, len %v\n", i, j, o, l)
				// fmt.Printf("  want: %02x\n", buf[o:o+l])
				// fmt.Printf("   got: %02x\n", readatBuf)
				t.Errorf("test %d/%d: wrong bytes returned at offset %v, len %v", i, j, o, l)
			}
		}
	}
}

func TestRandReader(t *testing.T) {
	rd := newRandReader(rand.New(rand.NewSource(23)))

	buf1 := make([]byte, 200)
	n, err := rd.Read(buf1)
	if err != nil {
		t.Fatal(err)
	}

	if n != len(buf1) {
		t.Fatalf("not enough bytes returned, want %d, got %d", len(buf1), n)
	}

	for _, l := range []int{1, 2, 3, 4, 5, 6, 7, 20, 22} {
		buf2 := make([]byte, l)
		rd := newRandReader(rand.New(rand.NewSource(23)))

		for i := 0; i < 2; i++ {
			n, err := rd.Read(buf2)
			if err != nil {
				t.Error(err)
				continue
			}

			if n != len(buf2) {
				t.Errorf("not enough bytes returned, want %d, got %d", len(buf1), n)
				continue
			}

			if !bytes.Equal(buf1[i*l:(i+1)*l], buf2) {
				t.Errorf("bytes not equal: want %02x, got %02x", buf1[i*l:(i+1)*l], buf2)
			}
		}
	}
}
<commit_msg>Add single failing test<commit_after>package main

import (
	"bytes"
	"fmt"
	"math/rand"
	"testing"
)

// var testFileSizes = []int{0, 100, 200, 500, 1024, 7666, 1 << 20, 1 << 24}
var testFileSizes = []int{1 << 20}

func TestFile(t *testing.T) {
	rnd := rand.New(rand.NewSource(23))

	for i, filesize := range testFileSizes {
		f := NewFile(23, filesize, 23)
		buf, err := f.ReadAll()
		if err != nil {
			t.Errorf("test %d: error %v", i, err)
			continue
		}

		if len(buf) != filesize {
			t.Errorf("test %d: invalid number of bytes returned, want %d, got %d", i, filesize, len(buf))
			continue
		}

		buf2 := make([]byte, filesize)
		n, err := f.ReadAt(buf2, 0)

		if err != nil {
			t.Errorf("test %d: error %v", i, err)
			continue
		}

		if n != filesize {
			t.Errorf("test %d: invalid number of bytes returned, want %d, got %d", i, filesize, n)
			continue
		}

		if !bytes.Equal(buf, buf2) {
			t.Errorf("test %d: wrong bytes returned", i)
			fmt.Printf("test %d: wrong bytes returned\n", i)
			continue
		}

		if filesize == 0 {
			n, err = f.ReadAt(buf2, 23)
			if n != 0 {
				t.Errorf("test %d: to read 0 bytes from empty file, got %d", i, n)
			}

			if err == nil {
				t.Errorf("test %d: expected error for empty file at offset 23 not found", i)
			}

			continue
		}

		for j := 0; j < 10; j++ {
			fmt.Printf("\n=========================================\n")

			o := rnd.Intn(filesize)
			l := rnd.Intn(filesize - o)

			fmt.Printf("  filesize %v, len %v, off %v\n", filesize, l, o)

			readatBuf := make([]byte, l)
			n, err = f.ReadAt(readatBuf, int64(o))
			fmt.Printf("n %v, l %v, o %v\n", n, l, o)
			if err != nil {
				t.Errorf("test %d/%d: reading len %v bytes at offset %v failed: %v", i, j, l, o, err)
				continue
			}

			if n != l {
				t.Errorf("test %d/%d: want %d bytes, got %d", i, j, l, n)
				fmt.Printf("test %d/%d: want %d bytes, got %d\n", i, j, l, n)
				continue
			}

			if !bytes.Equal(readatBuf, buf[o:o+l]) {
				fmt.Printf("------------------ test %d/%d: wrong bytes returned at offset %v, len %v\n", i, j, o, l)
				// fmt.Printf("  want: %02x\n", buf[o:o+l])
				// fmt.Printf("   got: %02x\n", readatBuf)
				t.Errorf("test %d/%d: wrong bytes returned at offset %v, len %v", i, j, o, l)
			}
		}
	}
}

func TestSingleFile(t *testing.T) {
	rnd := rand.New(rand.NewSource(23))

	filesize := 1048576
	f := NewFile(23, filesize, 0)
	buf, err := f.ReadAll()
	if err != nil {
		t.Fatal(err)
	}

	if len(buf) != filesize {
		t.Fatalf("invalid number of bytes returned, want %d, got %d", filesize, len(buf))
	}

	buf2 := make([]byte, filesize)
	n, err := f.ReadAt(buf2, 0)

	if err != nil {
		t.Fatalf("error %v", err)
	}

	if n != filesize {
		t.Errorf("invalid number of bytes returned, want %d, got %d", filesize, n)
	}

	if !bytes.Equal(buf, buf2) {
		t.Fatalf("wrong bytes returned")
	}

	for j := 0; j < 10; j++ {
		fmt.Printf("\n=========================================\n")

		o := rnd.Intn(filesize)
		l := rnd.Intn(filesize - o)

		fmt.Printf("  filesize %v, len %v, off %v\n", filesize, l, o)

		readatBuf := make([]byte, l)
		n, err = f.ReadAt(readatBuf, int64(o))
		fmt.Printf("n %v, l %v, o %v\n", n, l, o)
		if err != nil {
			t.Fatalf("reading len %v bytes at offset %v failed: %v", l, o, err)
			continue
		}

		if n != l {
			t.Fatalf("want %d bytes, got %d", l, n)
			continue
		}

		if !bytes.Equal(readatBuf, buf[o:o+l]) {
			// fmt.Printf("  want: %02x\n", buf[o:o+l])
			// fmt.Printf("   got: %02x\n", readatBuf)
			t.Fatalf("wrong bytes returned at offset %v, len %v", o, l)
		}
	}
}

func TestRandReader(t *testing.T) {
	rd := newRandReader(rand.New(rand.NewSource(23)))

	buf1 := make([]byte, 200)
	n, err := rd.Read(buf1)
	if err != nil {
		t.Fatal(err)
	}

	if n != len(buf1) {
		t.Fatalf("not enough bytes returned, want %d, got %d", len(buf1), n)
	}

	for _, l := range []int{1, 2, 3, 4, 5, 6, 7, 20, 22} {
		buf2 := make([]byte, l)
		rd := newRandReader(rand.New(rand.NewSource(23)))

		for i := 0; i < 2; i++ {
			n, err := rd.Read(buf2)
			if err != nil {
				t.Error(err)
				continue
			}

			if n != len(buf2) {
				t.Errorf("not enough bytes returned, want %d, got %d", len(buf1), n)
				continue
			}

			if !bytes.Equal(buf1[i*l:(i+1)*l], buf2) {
				t.Errorf("bytes not equal: want %02x, got %02x", buf1[i*l:(i+1)*l], buf2)
			}
		}
	}
}
<|endoftext|>