<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"

	"github.com/kisielk/whisper-go/whisper"
	"github.com/raintank/dur"
	"github.com/raintank/metrictank/api"
	"github.com/raintank/metrictank/mdata/chunk"
	"github.com/raintank/metrictank/mdata/chunk/archive"
	"gopkg.in/raintank/schema.v1"
)

var (
	exitOnError = flag.Bool(
		"exit-on-error",
		true,
		"Exit with a message when there's an error",
	)
	verbose = flag.Bool(
		"verbose",
		false,
		"Write logs to terminal",
	)
	httpEndpoint = flag.String(
		"http-endpoint",
		"http://127.0.0.1:8080/chunks",
		"The http endpoint to send the data to",
	)
	chunkSpanStr = flag.String(
		"chunkspans",
		"10min",
		"List of chunk spans separated by ':'. The 1st whisper archive gets the 1st span, 2nd the 2nd, etc",
	)
	namePrefix = flag.String(
		"name-prefix",
		"",
		"Prefix to prepend before every metric name, should include the '.' if necessary",
	)
	threads = flag.Int(
		"threads",
		10,
		"Number of workers threads to process .wsp files",
	)
	skipUnfinishedChunks = flag.Bool(
		"skip-unfinished-chunks",
		true,
		"Chunks that have not completed their chunk span should be skipped",
	)
	orgId = flag.Int(
		"orgid",
		1,
		"Organization ID the data belongs to ",
	)
	whisperDirectory = flag.String(
		"whisper-directory",
		"/opt/graphite/storage/whisper",
		"The directory that contains the whisper file structure",
	)
	readArchivesStr = flag.String(
		"read-archives",
		"*",
		"Comma separated list of positive integers or '*' for all archives",
	)
	chunkSpans   []uint32
	readArchives map[int]struct{}
	printLock    sync.Mutex
)

func main() {
	flag.Parse()

	for _, chunkSpanStrSplit := range strings.Split(*chunkSpanStr, ":") {
		chunkSpans = append(chunkSpans, dur.MustParseUNsec("chunkspan", chunkSpanStrSplit))
	}

	if *readArchivesStr != "*" {
		readArchives = make(map[int]struct{})
		for _, archiveIdStr := range strings.Split(*readArchivesStr, ",") {
			archiveId, err := strconv.Atoi(archiveIdStr)
			if err != nil {
				throwError(fmt.Sprintf("Invalid archive id %s: %s", archiveIdStr, err))
			}
			readArchives[archiveId] = struct{}{}
		}
	}

	fileChan := make(chan string)

	wg := &sync.WaitGroup{}
	for i := 0; i < *threads; i++ {
		go processFromChan(fileChan, wg)
	}

	getFileListIntoChan(fileChan, wg)
	wg.Wait()
}

func throwError(msg string) {
	msg = fmt.Sprintf("%s\n", msg)
	if *exitOnError {
		panic(msg)
	} else {
		printLock.Lock()
		fmt.Fprintln(os.Stderr, msg)
		printLock.Unlock()
	}
}

func log(msg string) {
	if *verbose {
		printLock.Lock()
		fmt.Println(msg)
		printLock.Unlock()
	}
}

func processFromChan(files chan string, wg *sync.WaitGroup) {
	client := &http.Client{}

	for file := range files {
		fd, err := os.Open(file)
		if err != nil {
			throwError(fmt.Sprintf("ERROR: Failed to open whisper file '%s': %s\n", file, err))
			continue
		}
		w, err := whisper.OpenWhisper(fd)
		if err != nil {
			throwError(fmt.Sprintf("ERROR: Failed to open whisper file '%s': %s\n", file, err))
			continue
		}

		log(fmt.Sprintf("Processing file %s", file))
		b, err := getMetric(w, file).Encode()
		if err != nil {
			throwError(fmt.Sprintf("%s", err))
		}

		req, err := http.NewRequest("POST", *httpEndpoint, b)
		if err != nil {
			throwError(fmt.Sprintf("Cannot send request to http endpoint %s: %s", *httpEndpoint, err))
		}
		req.Header.Set("Content-Type", "application/json")
		req.Header.Set("Content-Encoding", "gzip")

		_, err = client.Do(req)
		if err != nil {
			throwError(fmt.Sprintf("Error sending request to http endpoint %s: %s", *httpEndpoint, err))
		}

		wg.Done()
	}
}

// generate the metric name based on the file name and given prefix
func getMetricName(file string) string {
	for {
		if file[0] == '/' {
			file = file[1:]
		} else {
			break
		}
	}
	splits := strings.Split(file, "/")
	leafNode := strings.Split(splits[len(splits)-1], ".")
	splits[len(splits)-1] = leafNode[0]
	return *namePrefix + strings.Join(splits, ".")
}

// pointSorter sorts points by timestamp
type pointSorter []whisper.Point

func (a pointSorter) Len() int           { return len(a) }
func (a pointSorter) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a pointSorter) Less(i, j int) bool { return a[i].Timestamp < a[j].Timestamp }

// the whisper archives are organized like a ringbuffer. since we need to
// insert the points into the chunks in order we first need to sort them
func sortPoints(points pointSorter) pointSorter {
	sort.Sort(points)
	return points
}

func getMetric(w *whisper.Whisper, file string) *archive.Metric {
	if len(w.Header.Archives) == 0 {
		throwError(fmt.Sprintf("ERROR: Whisper file contains no archives: %s", file))
	}

	archives := make([]archive.Archive, 0, len(w.Header.Archives))
	var chunkSpan uint32
	var rowKey string
	name := getMetricName(file)

	// md gets generated from the first archive in the whisper file
	md := getMetricData(name, int(w.Header.Archives[0].SecondsPerPoint))

	for archiveIdx, archiveInfo := range w.Header.Archives {
		if archiveIdx > 0 {
			rowKey = api.AggMetricKey(
				md.Id,
				w.Header.Metadata.AggregationMethod.String(),
				archiveInfo.SecondsPerPoint,
			)
		} else {
			rowKey = md.Id
		}

		// only read archive if archiveIdx is in readArchives
		if _, ok := readArchives[archiveIdx]; !ok && len(readArchives) > 0 {
			continue
		}

		encodedChunks := make(archive.ArchiveOfChunks)

		if len(chunkSpans)-1 < archiveIdx {
			// if we have more archives than chunk spans are specified, we simply use the last one
			chunkSpan = chunkSpans[len(chunkSpans)-1]
		} else {
			chunkSpan = chunkSpans[archiveIdx]
		}

		points, err := w.DumpArchive(archiveIdx)
		if err != nil {
			throwError(fmt.Sprintf("ERROR: Failed to read archive %d in '%s', skipping: %s", archiveIdx, file, err))
		}

		var point whisper.Point
		var t0, prevT0 uint32 = 0, 0
		var c *chunk.Chunk

		for _, point = range sortPoints(points) {
			if point.Timestamp == 0 {
				continue
			}

			t0 = point.Timestamp - (point.Timestamp % chunkSpan)
			if prevT0 == 0 {
				log(fmt.Sprintf("Create new chunk at t0: %d", t0))
				c = chunk.New(t0)
				prevT0 = t0
			} else if prevT0 != t0 {
				log(fmt.Sprintf("Mark chunk at t0 %d as finished", prevT0))
				c.Finish()

				encodedChunks[c.T0] = archive.MetricChunk{
					ChunkSpan: chunkSpan,
					Bytes:     c.Series.Bytes(),
				}

				log(fmt.Sprintf("Create new chunk at t0: %d", t0))
				c = chunk.New(t0)
				prevT0 = t0
			}

			err := c.Push(point.Timestamp, point.Value)
			if err != nil {
				throwError(fmt.Sprintf("ERROR: Failed to push value into chunk at t0 %d: %s", t0, err))
			}
		}

		if int64(point.Timestamp) > md.Time {
			md.Time = int64(point.Timestamp)
		}

		// if the last written point was also the last one of the current chunk,
		// or if skipUnfinishedChunks is on, we close the chunk and
		if point.Timestamp == t0+chunkSpan-archiveInfo.SecondsPerPoint || !*skipUnfinishedChunks {
			log(fmt.Sprintf("Mark current (last) chunk at t0 %d as finished", t0))
			c.Finish()
			encodedChunks[c.T0] = archive.MetricChunk{
				ChunkSpan: chunkSpan,
				Bytes:     c.Series.Bytes(),
			}
		}

		log(fmt.Sprintf("Whisper file %s archive %d (%s) gets %d chunks", file, archiveIdx, name, len(encodedChunks)))
		archives = append(archives, archive.Archive{
			ArchiveInfo: archiveInfo,
			Chunks:      encodedChunks,
			RowKey:      rowKey,
		})
	}

	return &archive.Metric{
		Metadata:   w.Header.Metadata,
		MetricData: *md,
		Archives:   archives,
	}
}

func getMetricData(name string, interval int) *schema.MetricData {
	md := &schema.MetricData{
		Name:     name,
		Metric:   name,
		Interval: interval,
		Value:    0,
		Unit:     "unknown",
		Time:     0,
		Mtype:    "gauge",
		Tags:     []string{},
		OrgId:    *orgId,
	}
	md.SetId()
	return md
}

// scan a directory and feed the list of whisper files relative to base into the given channel
func getFileListIntoChan(fileChan chan string, wg *sync.WaitGroup) {

	filepath.Walk(
		*whisperDirectory,
		func(path string, info os.FileInfo, err error) error {
			if path[len(path)-4:] == ".wsp" {
				wg.Add(1)
				fileChan <- path
			}
			return nil
		},
	)

	close(fileChan)
}
<commit_msg>remove unnecessary initialization<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"

	"github.com/kisielk/whisper-go/whisper"
	"github.com/raintank/dur"
	"github.com/raintank/metrictank/api"
	"github.com/raintank/metrictank/mdata/chunk"
	"github.com/raintank/metrictank/mdata/chunk/archive"
	"gopkg.in/raintank/schema.v1"
)

var (
	exitOnError = flag.Bool(
		"exit-on-error",
		true,
		"Exit with a message when there's an error",
	)
	verbose = flag.Bool(
		"verbose",
		false,
		"Write logs to terminal",
	)
	httpEndpoint = flag.String(
		"http-endpoint",
		"http://127.0.0.1:8080/chunks",
		"The http endpoint to send the data to",
	)
	chunkSpanStr = flag.String(
		"chunkspans",
		"10min",
		"List of chunk spans separated by ':'. The 1st whisper archive gets the 1st span, 2nd the 2nd, etc",
	)
	namePrefix = flag.String(
		"name-prefix",
		"",
		"Prefix to prepend before every metric name, should include the '.' if necessary",
	)
	threads = flag.Int(
		"threads",
		10,
		"Number of workers threads to process .wsp files",
	)
	skipUnfinishedChunks = flag.Bool(
		"skip-unfinished-chunks",
		true,
		"Chunks that have not completed their chunk span should be skipped",
	)
	orgId = flag.Int(
		"orgid",
		1,
		"Organization ID the data belongs to ",
	)
	whisperDirectory = flag.String(
		"whisper-directory",
		"/opt/graphite/storage/whisper",
		"The directory that contains the whisper file structure",
	)
	readArchivesStr = flag.String(
		"read-archives",
		"*",
		"Comma separated list of positive integers or '*' for all archives",
	)
	chunkSpans   []uint32
	readArchives map[int]struct{}
	printLock    sync.Mutex
)

func main() {
	flag.Parse()

	for _, chunkSpanStrSplit := range strings.Split(*chunkSpanStr, ":") {
		chunkSpans = append(chunkSpans, dur.MustParseUNsec("chunkspan", chunkSpanStrSplit))
	}

	if *readArchivesStr != "*" {
		readArchives = make(map[int]struct{})
		for _, archiveIdStr := range strings.Split(*readArchivesStr, ",") {
			archiveId, err := strconv.Atoi(archiveIdStr)
			if err != nil {
				throwError(fmt.Sprintf("Invalid archive id %s: %s", archiveIdStr, err))
			}
			readArchives[archiveId] = struct{}{}
		}
	}

	fileChan := make(chan string)

	wg := &sync.WaitGroup{}
	for i := 0; i < *threads; i++ {
		go processFromChan(fileChan, wg)
	}

	getFileListIntoChan(fileChan, wg)
	wg.Wait()
}

func throwError(msg string) {
	msg = fmt.Sprintf("%s\n", msg)
	if *exitOnError {
		panic(msg)
	} else {
		printLock.Lock()
		fmt.Fprintln(os.Stderr, msg)
		printLock.Unlock()
	}
}

func log(msg string) {
	if *verbose {
		printLock.Lock()
		fmt.Println(msg)
		printLock.Unlock()
	}
}

func processFromChan(files chan string, wg *sync.WaitGroup) {
	client := &http.Client{}

	for file := range files {
		fd, err := os.Open(file)
		if err != nil {
			throwError(fmt.Sprintf("ERROR: Failed to open whisper file '%s': %s\n", file, err))
			continue
		}
		w, err := whisper.OpenWhisper(fd)
		if err != nil {
			throwError(fmt.Sprintf("ERROR: Failed to open whisper file '%s': %s\n", file, err))
			continue
		}

		log(fmt.Sprintf("Processing file %s", file))
		b, err := getMetric(w, file).Encode()
		if err != nil {
			throwError(fmt.Sprintf("%s", err))
		}

		req, err := http.NewRequest("POST", *httpEndpoint, b)
		if err != nil {
			throwError(fmt.Sprintf("Cannot send request to http endpoint %s: %s", *httpEndpoint, err))
		}
		req.Header.Set("Content-Type", "application/json")
		req.Header.Set("Content-Encoding", "gzip")

		_, err = client.Do(req)
		if err != nil {
			throwError(fmt.Sprintf("Error sending request to http endpoint %s: %s", *httpEndpoint, err))
		}

		wg.Done()
	}
}

// generate the metric name based on the file name and given prefix
func getMetricName(file string) string {
	for {
		if file[0] == '/' {
			file = file[1:]
		} else {
			break
		}
	}
	splits := strings.Split(file, "/")
	leafNode := strings.Split(splits[len(splits)-1], ".")
	splits[len(splits)-1] = leafNode[0]
	return *namePrefix + strings.Join(splits, ".")
}

// pointSorter sorts points by timestamp
type pointSorter []whisper.Point

func (a pointSorter) Len() int           { return len(a) }
func (a pointSorter) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a pointSorter) Less(i, j int) bool { return a[i].Timestamp < a[j].Timestamp }

// the whisper archives are organized like a ringbuffer. since we need to
// insert the points into the chunks in order we first need to sort them
func sortPoints(points pointSorter) pointSorter {
	sort.Sort(points)
	return points
}

func getMetric(w *whisper.Whisper, file string) *archive.Metric {
	if len(w.Header.Archives) == 0 {
		throwError(fmt.Sprintf("ERROR: Whisper file contains no archives: %s", file))
	}

	archives := make([]archive.Archive, 0, len(w.Header.Archives))
	var chunkSpan uint32
	var rowKey string
	name := getMetricName(file)

	// md gets generated from the first archive in the whisper file
	md := getMetricData(name, int(w.Header.Archives[0].SecondsPerPoint))

	for archiveIdx, archiveInfo := range w.Header.Archives {
		if archiveIdx > 0 {
			rowKey = api.AggMetricKey(
				md.Id,
				w.Header.Metadata.AggregationMethod.String(),
				archiveInfo.SecondsPerPoint,
			)
		} else {
			rowKey = md.Id
		}

		// only read archive if archiveIdx is in readArchives
		if _, ok := readArchives[archiveIdx]; !ok && len(readArchives) > 0 {
			continue
		}

		encodedChunks := make(archive.ArchiveOfChunks)

		if len(chunkSpans)-1 < archiveIdx {
			// if we have more archives than chunk spans are specified, we simply use the last one
			chunkSpan = chunkSpans[len(chunkSpans)-1]
		} else {
			chunkSpan = chunkSpans[archiveIdx]
		}

		points, err := w.DumpArchive(archiveIdx)
		if err != nil {
			throwError(fmt.Sprintf("ERROR: Failed to read archive %d in '%s', skipping: %s", archiveIdx, file, err))
		}

		var point whisper.Point
		var t0, prevT0 uint32
		var c *chunk.Chunk

		for _, point = range sortPoints(points) {
			if point.Timestamp == 0 {
				continue
			}

			t0 = point.Timestamp - (point.Timestamp % chunkSpan)
			if prevT0 == 0 {
				log(fmt.Sprintf("Create new chunk at t0: %d", t0))
				c = chunk.New(t0)
				prevT0 = t0
			} else if prevT0 != t0 {
				log(fmt.Sprintf("Mark chunk at t0 %d as finished", prevT0))
				c.Finish()

				encodedChunks[c.T0] = archive.MetricChunk{
					ChunkSpan: chunkSpan,
					Bytes:     c.Series.Bytes(),
				}

				log(fmt.Sprintf("Create new chunk at t0: %d", t0))
				c = chunk.New(t0)
				prevT0 = t0
			}

			err := c.Push(point.Timestamp, point.Value)
			if err != nil {
				throwError(fmt.Sprintf("ERROR: Failed to push value into chunk at t0 %d: %s", t0, err))
			}
		}

		if int64(point.Timestamp) > md.Time {
			md.Time = int64(point.Timestamp)
		}

		// if the last written point was also the last one of the current chunk,
		// or if skipUnfinishedChunks is on, we close the chunk and
		if point.Timestamp == t0+chunkSpan-archiveInfo.SecondsPerPoint || !*skipUnfinishedChunks {
			log(fmt.Sprintf("Mark current (last) chunk at t0 %d as finished", t0))
			c.Finish()
			encodedChunks[c.T0] = archive.MetricChunk{
				ChunkSpan: chunkSpan,
				Bytes:     c.Series.Bytes(),
			}
		}

		log(fmt.Sprintf("Whisper file %s archive %d (%s) gets %d chunks", file, archiveIdx, name, len(encodedChunks)))
		archives = append(archives, archive.Archive{
			ArchiveInfo: archiveInfo,
			Chunks:      encodedChunks,
			RowKey:      rowKey,
		})
	}

	return &archive.Metric{
		Metadata:   w.Header.Metadata,
		MetricData: *md,
		Archives:   archives,
	}
}

func getMetricData(name string, interval int) *schema.MetricData {
	md := &schema.MetricData{
		Name:     name,
		Metric:   name,
		Interval: interval,
		Value:    0,
		Unit:     "unknown",
		Time:     0,
		Mtype:    "gauge",
		Tags:     []string{},
		OrgId:    *orgId,
	}
	md.SetId()
	return md
}

// scan a directory and feed the list of whisper files relative to base into the given channel
func getFileListIntoChan(fileChan chan string, wg *sync.WaitGroup) {

	filepath.Walk(
		*whisperDirectory,
		func(path string, info os.FileInfo, err error) error {
			if path[len(path)-4:] == ".wsp" {
				wg.Add(1)
				fileChan <- path
			}
			return nil
		},
	)

	close(fileChan)
}
<|endoftext|>


<commit_before>/*
 * This file is part of the libvirt-go-xml project
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 * Copyright (C) 2016 Red Hat, Inc.
 *
 */

package libvirtxml

import (
	"encoding/xml"
)

type CapsHostCPUTopology struct {
	Sockets int `xml:"sockets,attr"`
	Cores   int `xml:"cores,attr"`
	Threads int `xml:"threads,attr"`
}

type CapsHostCPUFeatureFlag struct {
	Name string `xml:"name,attr"`
}

type CapsHostCPUPageSize struct {
	Size int    `xml:"size,attr"`
	Unit string `xml:"unit,attr"`
}

type CapsHostCPU struct {
	Arch         string                   `xml:"arch,omitempty"`
	Model        string                   `xml:"model,omitempty"`
	Vendor       string                   `xml:"vendor,omitempty"`
	Topology     *CapsHostCPUTopology     `xml:"topology"`
	FeatureFlags []CapsHostCPUFeatureFlag `xml:"feature"`
	Features     *CapsHostCPUFeatures     `xml:"features"`
	PageSizes    []CapsHostCPUPageSize    `xml:"pages"`
}

type CapsHostCPUFeature struct {
}

type CapsHostCPUFeatures struct {
	PAE    *CapsHostCPUFeature `xml:"pae"`
	NonPAE *CapsHostCPUFeature `xml:"nonpae"`
	SVM    *CapsHostCPUFeature `xml:"svm"`
	VMX    *CapsHostCPUFeature `xml:"vmx"`
}

type CapsHostNUMAMemory struct {
	Size uint64 `xml:"size,attr"`
	Unit string `xml:"unit,attr"`
}

type CapsHostNUMAPageInfo struct {
	Size  int    `xml:"size,attr"`
	Unit  string `xml:"unit,attr"`
	Count uint64 `xml:",chardata"`
}

type CapsHostNUMACPU struct {
	ID       int    `xml:"id,attr"`
	SocketID *int   `xml:"socket_id,attr"`
	CoreID   *int   `xml:"core_id,attr"`
	Siblings string `xml:"siblings,attr,omitempty"`
}

type CapsHostNUMASibling struct {
	ID    int `xml:"id,attr"`
	Value int `xml:"value,attr"`
}

type CapsHostNUMACell struct {
	ID        int                    `xml:"id,attr"`
	Memory    *CapsHostNUMAMemory    `xml:"memory"`
	PageInfo  []CapsHostNUMAPageInfo `xml:"pages"`
	Distances *CapsHostNUMADistances `xml:"distances"`
	CPUS      *CapsHostNUMACPUs      `xml:"cpus"`
}

type CapsHostNUMADistances struct {
	Siblings []CapsHostNUMASibling `xml:"sibling"`
}

type CapsHostNUMACPUs struct {
	Num  uint              `xml:"num,attr,omitempty"`
	CPUs []CapsHostNUMACPU `xml:"cpu"`
}

type CapsHostNUMATopology struct {
	Cells *CapsHostNUMACells `xml:"cells"`
}

type CapsHostNUMACells struct {
	Num   uint               `xml:"num,attr,omitempty"`
	Cells []CapsHostNUMACell `xml:"cell"`
}

type CapsHostSecModelLabel struct {
	Type  string `xml:"type,attr"`
	Value string `xml:",chardata"`
}

type CapsHostSecModel struct {
	Name   string                  `xml:"model"`
	DOI    string                  `xml:"doi"`
	Labels []CapsHostSecModelLabel `xml:"baselabel"`
}

type CapsHostMigrationFeatures struct {
	Live          *CapsHostMigrationLive          `xml:"live"`
	URITransports *CapsHostMigrationURITransports `xml:"uri_transports"`
}

type CapsHostMigrationLive struct {
}

type CapsHostMigrationURITransports struct {
	URI []string `xml:"uri_transport"`
}

type CapsHost struct {
	UUID              string                     `xml:"uuid,omitempty"`
	CPU               *CapsHostCPU               `xml:"cpu"`
	MigrationFeatures *CapsHostMigrationFeatures `xml:"migration_features"`
	NUMA              *CapsHostNUMATopology      `xml:"topology"`
	SecModel          []CapsHostSecModel         `xml:"secmodel"`
}

type CapsGuestMachine struct {
	Name      string `xml:",chardata"`
	MaxCPUs   int    `xml:"maxCpus,attr,omitempty"`
	Canonical string `xml:"canonical,attr,omitempty"`
}

type CapsGuestDomain struct {
	Type     string             `xml:"type,attr"`
	Emulator string             `xml:"emulator,omitempty"`
	Machines []CapsGuestMachine `xml:"machine"`
}

type CapsGuestArch struct {
	Name     string             `xml:"name,attr"`
	WordSize string             `xml:"wordsize"`
	Emulator string             `xml:"emulator"`
	Machines []CapsGuestMachine `xml:"machine"`
	Domains  []CapsGuestDomain  `xml:"domain"`
}

type CapsGuestFeatureCPUSelection struct {
}

type CapsGuestFeatureDeviceBoot struct {
}

type CapsGuestFeaturePAE struct {
}

type CapsGuestFeatureNonPAE struct {
}

type CapsGuestFeatureAPIC struct {
	Default string `xml:"default,attr,omitempty"`
	Toggle  string `xml:"toggle,attr,omitempty"`
}

type CapsGuestFeatureACPI struct {
	Default string `xml:"default,attr,omitempty"`
	Toggle  string `xml:"toggle,attr,omitempty"`
}

type CapsGuestFeatures struct {
	CPUSelection *CapsGuestFeatureCPUSelection `xml:"cpuselection"`
	DeviceBoot   *CapsGuestFeatureDeviceBoot   `xml:"deviceboot"`
	PAE          *CapsGuestFeaturePAE          `xml:"pae"`
	NonPAE       *CapsGuestFeatureNonPAE       `xml:"nonpae"`
	APIC         *CapsGuestFeatureAPIC         `xml:"apic"`
	ACPI         *CapsGuestFeatureACPI         `xml:"acpi"`
}

type CapsGuest struct {
	OSType   string             `xml:"os_type"`
	Arch     CapsGuestArch      `xml:"arch"`
	Features *CapsGuestFeatures `xml:"features"`
}

type Caps struct {
	XMLName xml.Name    `xml:"capabilities"`
	Host    CapsHost    `xml:"host"`
	Guests  []CapsGuest `xml:"guest"`
}

func (c *Caps) Unmarshal(doc string) error {
	return xml.Unmarshal([]byte(doc), c)
}

func (c *Caps) Marshal() (string, error) {
	doc, err := xml.MarshalIndent(c, "", "  ")
	if err != nil {
		return "", err
	}
	return string(doc), nil
}
<commit_msg>Add capabilities host power mgmt<commit_after>/*
 * This file is part of the libvirt-go-xml project
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 * Copyright (C) 2016 Red Hat, Inc.
 *
 */

package libvirtxml

import (
	"encoding/xml"
)

type CapsHostCPUTopology struct {
	Sockets int `xml:"sockets,attr"`
	Cores   int `xml:"cores,attr"`
	Threads int `xml:"threads,attr"`
}

type CapsHostCPUFeatureFlag struct {
	Name string `xml:"name,attr"`
}

type CapsHostCPUPageSize struct {
	Size int    `xml:"size,attr"`
	Unit string `xml:"unit,attr"`
}

type CapsHostCPU struct {
	Arch         string                   `xml:"arch,omitempty"`
	Model        string                   `xml:"model,omitempty"`
	Vendor       string                   `xml:"vendor,omitempty"`
	Topology     *CapsHostCPUTopology     `xml:"topology"`
	FeatureFlags []CapsHostCPUFeatureFlag `xml:"feature"`
	Features     *CapsHostCPUFeatures     `xml:"features"`
	PageSizes    []CapsHostCPUPageSize    `xml:"pages"`
}

type CapsHostCPUFeature struct {
}

type CapsHostCPUFeatures struct {
	PAE    *CapsHostCPUFeature `xml:"pae"`
	NonPAE *CapsHostCPUFeature `xml:"nonpae"`
	SVM    *CapsHostCPUFeature `xml:"svm"`
	VMX    *CapsHostCPUFeature `xml:"vmx"`
}

type CapsHostNUMAMemory struct {
	Size uint64 `xml:"size,attr"`
	Unit string `xml:"unit,attr"`
}

type CapsHostNUMAPageInfo struct {
	Size  int    `xml:"size,attr"`
	Unit  string `xml:"unit,attr"`
	Count uint64 `xml:",chardata"`
}

type CapsHostNUMACPU struct {
	ID       int    `xml:"id,attr"`
	SocketID *int   `xml:"socket_id,attr"`
	CoreID   *int   `xml:"core_id,attr"`
	Siblings string `xml:"siblings,attr,omitempty"`
}

type CapsHostNUMASibling struct {
	ID    int `xml:"id,attr"`
	Value int `xml:"value,attr"`
}

type CapsHostNUMACell struct {
	ID        int                    `xml:"id,attr"`
	Memory    *CapsHostNUMAMemory    `xml:"memory"`
	PageInfo  []CapsHostNUMAPageInfo `xml:"pages"`
	Distances *CapsHostNUMADistances `xml:"distances"`
	CPUS      *CapsHostNUMACPUs      `xml:"cpus"`
}

type CapsHostNUMADistances struct {
	Siblings []CapsHostNUMASibling `xml:"sibling"`
}

type CapsHostNUMACPUs struct {
	Num  uint              `xml:"num,attr,omitempty"`
	CPUs []CapsHostNUMACPU `xml:"cpu"`
}

type CapsHostNUMATopology struct {
	Cells *CapsHostNUMACells `xml:"cells"`
}

type CapsHostNUMACells struct {
	Num   uint               `xml:"num,attr,omitempty"`
	Cells []CapsHostNUMACell `xml:"cell"`
}

type CapsHostSecModelLabel struct {
	Type  string `xml:"type,attr"`
	Value string `xml:",chardata"`
}

type CapsHostSecModel struct {
	Name   string                  `xml:"model"`
	DOI    string                  `xml:"doi"`
	Labels []CapsHostSecModelLabel `xml:"baselabel"`
}

type CapsHostMigrationFeatures struct {
	Live          *CapsHostMigrationLive          `xml:"live"`
	URITransports *CapsHostMigrationURITransports `xml:"uri_transports"`
}

type CapsHostMigrationLive struct {
}

type CapsHostMigrationURITransports struct {
	URI []string `xml:"uri_transport"`
}

type CapsHost struct {
	UUID              string                     `xml:"uuid,omitempty"`
	CPU               *CapsHostCPU               `xml:"cpu"`
	PowerManagement   *CapsHostPowerManagement   `xml:"power_management"`
	MigrationFeatures *CapsHostMigrationFeatures `xml:"migration_features"`
	NUMA              *CapsHostNUMATopology      `xml:"topology"`
	SecModel          []CapsHostSecModel         `xml:"secmodel"`
}

type CapsHostPowerManagement struct {
	SuspendMem    *CapsHostPowerManagementMode `xml:"suspend_mem"`
	SuspendDisk   *CapsHostPowerManagementMode `xml:"suspend_disk"`
	SuspendHybrid *CapsHostPowerManagementMode `xml:"suspend_hybrid"`
}

type CapsHostPowerManagementMode struct {
}

type CapsGuestMachine struct {
	Name      string `xml:",chardata"`
	MaxCPUs   int    `xml:"maxCpus,attr,omitempty"`
	Canonical string `xml:"canonical,attr,omitempty"`
}

type CapsGuestDomain struct {
	Type     string             `xml:"type,attr"`
	Emulator string             `xml:"emulator,omitempty"`
	Machines []CapsGuestMachine `xml:"machine"`
}

type CapsGuestArch struct {
	Name     string             `xml:"name,attr"`
	WordSize string             `xml:"wordsize"`
	Emulator string             `xml:"emulator"`
	Machines []CapsGuestMachine `xml:"machine"`
	Domains  []CapsGuestDomain  `xml:"domain"`
}

type CapsGuestFeatureCPUSelection struct {
}

type CapsGuestFeatureDeviceBoot struct {
}

type CapsGuestFeaturePAE struct {
}

type CapsGuestFeatureNonPAE struct {
}

type CapsGuestFeatureAPIC struct {
	Default string `xml:"default,attr,omitempty"`
	Toggle  string `xml:"toggle,attr,omitempty"`
}

type CapsGuestFeatureACPI struct {
	Default string `xml:"default,attr,omitempty"`
	Toggle  string `xml:"toggle,attr,omitempty"`
}

type CapsGuestFeatures struct {
	CPUSelection *CapsGuestFeatureCPUSelection `xml:"cpuselection"`
	DeviceBoot   *CapsGuestFeatureDeviceBoot   `xml:"deviceboot"`
	PAE          *CapsGuestFeaturePAE          `xml:"pae"`
	NonPAE       *CapsGuestFeatureNonPAE       `xml:"nonpae"`
	APIC         *CapsGuestFeatureAPIC         `xml:"apic"`
	ACPI         *CapsGuestFeatureACPI         `xml:"acpi"`
}

type CapsGuest struct {
	OSType   string             `xml:"os_type"`
	Arch     CapsGuestArch      `xml:"arch"`
	Features *CapsGuestFeatures `xml:"features"`
}

type Caps struct {
	XMLName xml.Name    `xml:"capabilities"`
	Host    CapsHost    `xml:"host"`
	Guests  []CapsGuest `xml:"guest"`
}

func (c *Caps) Unmarshal(doc string) error {
	return xml.Unmarshal([]byte(doc), c)
}

func (c *Caps) Marshal() (string, error) {
	doc, err := xml.MarshalIndent(c, "", "  ")
	if err != nil {
		return "", err
	}
	return string(doc), nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"math"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/gocql/gocql"
	"github.com/kisielk/whisper-go/whisper"
	"github.com/raintank/dur"
	"github.com/raintank/metrictank/api"
	"github.com/raintank/metrictank/cluster"
	"github.com/raintank/metrictank/cluster/partitioner"
	"github.com/raintank/metrictank/idx"
	"github.com/raintank/metrictank/idx/cassandra"
	"github.com/raintank/metrictank/mdata"
	"github.com/raintank/metrictank/mdata/chunk"
	"github.com/raintank/metrictank/mdata/chunk/archive"
)

var (
	exitOnError = flag.Bool(
		"exit-on-error",
		true,
		"Exit with a message when there's an error",
	)
	verbose = flag.Bool(
		"verbose",
		false,
		"Write logs to terminal",
	)
	fakeAvgAggregates = flag.Bool(
		"fake-avg-aggregates",
		true,
		"Generate sum/cnt series out of avg series to accomodate metrictank",
	)
	httpEndpoint = flag.String(
		"http-endpoint",
		"127.0.0.1:8080",
		"The http endpoint to listen on",
	)
	cassandraAddrs = flag.String(
		"cassandra-addrs",
		"localhost",
		"cassandra host (may be given multiple times as comma-separated list)",
	)
	cassandraKeyspace = flag.String(
		"cassandra-keyspace",
		"metrictank",
		"cassandra keyspace to use for storing the metric data table",
	)
	ttlsStr = flag.String(
		"ttls",
		"35d",
		"list of ttl strings used by MT separated by ','",
	)
	windowFactor = flag.Int(
		"window-factor",
		20,
		"the window factor be used when creating the metric table schema",
	)
	partitionScheme = flag.String(
		"partition-scheme",
		"bySeries",
		"method used for partitioning metrics. This should match the settings of tsdb-gw. (byOrg|bySeries)",
	)
	uriPath = flag.String(
		"uri-path",
		"/chunks",
		"the URI on which we expect chunks to get posted",
	)
	numPartitions = flag.Int(
		"num-partitions",
		1,
		"Number of Partitions",
	)
	GitHash   = "(none)"
	printLock sync.Mutex
)

type Server struct {
	Cluster     *gocql.ClusterConfig
	Session     *gocql.Session
	TTLTables   mdata.TTLTables
	Partitioner partitioner.Partitioner
	Index       idx.MetricIndex
}

func main() {
	cassandra.ConfigSetup()
	flag.Parse()

	cassCluster := gocql.NewCluster(strings.Split(*cassandraAddrs, ",")...)
	cassCluster.Consistency = gocql.ParseConsistency("one")
	cassCluster.Timeout = time.Second
	cassCluster.NumConns = 2
	cassCluster.ProtoVersion = 4
	cassCluster.Keyspace = *cassandraKeyspace

	session, err := cassCluster.CreateSession()
	if err != nil {
		panic(fmt.Sprintf("Failed to create cassandra session: %q", err))
	}

	splits := strings.Split(*ttlsStr, ",")
	ttls := make([]uint32, 0)
	for _, split := range splits {
		ttls = append(ttls, dur.MustParseUNsec("ttl", split))
	}
	ttlTables := mdata.GetTTLTables(ttls, *windowFactor, mdata.Table_name_format)

	p, err := partitioner.NewKafka(*partitionScheme)
	if err != nil {
		panic(fmt.Sprintf("Failed to instantiate partitioner: %q", err))
	}

	server := &Server{
		Cluster:     cassCluster,
		Session:     session,
		TTLTables:   ttlTables,
		Partitioner: p,
		Index:       cassandra.New(),
	}
	cluster.Init("mt-whisper-importer-writer", GitHash, time.Now(), "http", int(80))
	server.Index.Init()

	http.HandleFunc(*uriPath, server.chunksHandler)

	log(fmt.Sprintf("Listening on %q", *httpEndpoint))
	err = http.ListenAndServe(*httpEndpoint, nil)
	if err != nil {
		panic(fmt.Sprintf("Error creating listener: %q", err))
	}
}

func throwError(msg string) {
	msg = fmt.Sprintf("%s\n", msg)
	if *exitOnError {
		panic(msg)
	} else {
		printLock.Lock()
		fmt.Fprintln(os.Stderr, msg)
		printLock.Unlock()
	}
}

func log(msg string) {
	if *verbose {
		printLock.Lock()
		fmt.Println(msg)
		printLock.Unlock()
	}
}

func (s *Server) chunksHandler(w http.ResponseWriter, req *http.Request) {
	metric := &archive.Metric{}
	err := metric.UnmarshalCompressed(req.Body)
	if err != nil {
		throwError(fmt.Sprintf("Error decoding metric stream: %q", err))
		return
	}
	log("Handling new metric")

	if len(metric.Archives) == 0 {
		throwError("Metric has no archives")
		return
	}

	avg := (whisper.AggregationMethod(metric.AggregationMethod) == whisper.AggregationAverage)

	partition, err := s.Partitioner.Partition(&metric.MetricData, int32(*numPartitions))
	err = s.Index.AddOrUpdate(&metric.MetricData, partition)
	if err != nil {
		throwError(fmt.Sprintf("Error updating metric index: %q", err))
		return
	}

	for archiveIdx, a := range metric.Archives {
		archiveTTL := a.SecondsPerPoint * a.Points
		tableTTL, err := s.selectTableByTTL(archiveTTL)
		if err != nil {
			throwError(fmt.Sprintf("Failed to select table for ttl %d in %+v: %q", archiveTTL, s.TTLTables, err))
			return
		}
		entry, ok := s.TTLTables[tableTTL]
		if !ok {
			throwError(fmt.Sprintf("Failed to get selected table %d in %+v", tableTTL, s.TTLTables))
			return
		}
		tableName := entry.Table

		if !avg || archiveIdx == 0 || !*fakeAvgAggregates {
			log(fmt.Sprintf(
				"inserting %d chunks of archive %d with ttl %d into table %s with ttl %d and key %s",
				len(a.Chunks), archiveIdx, archiveTTL, tableName, tableTTL, a.RowKey,
			))
			s.insertChunks(tableName, a.RowKey, tableTTL, a.Chunks)
		} else {
			// averaged archives are a special case because mt doesn't store them as such.
			// mt reconstructs the averages on the fly from the sum and cnt series, so we need
			// to generate these two series out of raw averaged data by multiplying each point
			// with the aggregation span and storing the result as sum, cnt is the aggregation span.

			// aggCount is the aggregation span of this archive divided by the raw interval
			aggCount := a.SecondsPerPoint / metric.Archives[0].SecondsPerPoint
			aggSpan := a.SecondsPerPoint

			sumArchive := make([]chunk.IterGen, 0, len(a.Chunks))
			cntArchive := make([]chunk.IterGen, 0, len(a.Chunks))
			for _, ig := range a.Chunks {
				T0 := ig.Ts
				sum := chunk.New(T0)
				cnt := chunk.New(T0)

				it, err := ig.Get()
				if err != nil {
					throwError(fmt.Sprintf("failed to get iterator from itergen: %q", err))
					continue
				}

				for it.Next() {
					ts, val := it.Values()
					cnt.Push(ts, float64(aggCount))
					sum.Push(ts, val*float64(aggCount))
				}

				cnt.Finish()
				sum.Finish()

				cntArchive = append(cntArchive, *chunk.NewBareIterGen(cnt.Bytes(), T0, aggSpan))
				sumArchive = append(sumArchive, *chunk.NewBareIterGen(sum.Bytes(), T0, aggSpan))
			}

			cntId := api.AggMetricKey(metric.MetricData.Id, "cnt", aggSpan)
			sumId := api.AggMetricKey(metric.MetricData.Id, "sum", aggSpan)

			log(fmt.Sprintf(
				"inserting 2 archives of %d chunks per archive with ttl %d into table %s with ttl %d and keys %s/%s",
				len(a.Chunks), archiveTTL, tableName, tableTTL, cntId, sumId,
			))

			s.insertChunks(tableName, cntId, tableTTL, cntArchive)
			s.insertChunks(tableName, sumId, tableTTL, sumArchive)
		}
	}
}

func (s *Server) insertChunks(table, id string, ttl uint32, itergens []chunk.IterGen) {
	query := fmt.Sprintf("INSERT INTO %s (key, ts, data) values (?,?,?) USING TTL %d", table, ttl)
	for _, ig := range itergens {
		rowKey := fmt.Sprintf("%s_%d", id, ig.Ts/mdata.Month_sec)
		err := s.Session.Query(query, rowKey, ig.Ts, mdata.PrepareChunkData(ig.Span, ig.Bytes())).Exec()
		if err != nil {
			throwError(fmt.Sprintf("Error in query: %q", err))
		}
	}
}

func (s *Server) selectTableByTTL(ttl uint32) (uint32, error) {
	selectedTTL := uint32(math.MaxUint32)

	// find the table with the smallest TTL that is at least equal to archiveTTL
	for tableTTL := range s.TTLTables {
		if tableTTL >= ttl {
			if selectedTTL > tableTTL {
				selectedTTL = tableTTL
			}
		}
	}

	// we have not found a table that can accommodate the requested ttl
	if selectedTTL == math.MaxUint32 {
		return 0, errors.New(fmt.Sprintf("No Table found that can hold TTL %d", ttl))
	}

	return selectedTTL, nil
}
<commit_msg>fix merge conflict<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"math"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/gocql/gocql"
	"github.com/kisielk/whisper-go/whisper"
	"github.com/raintank/dur"
	"github.com/raintank/metrictank/api"
	"github.com/raintank/metrictank/cluster"
	"github.com/raintank/metrictank/cluster/partitioner"
	"github.com/raintank/metrictank/idx"
	"github.com/raintank/metrictank/idx/cassandra"
	"github.com/raintank/metrictank/mdata"
	"github.com/raintank/metrictank/mdata/chunk"
	"github.com/raintank/metrictank/mdata/chunk/archive"
)

var (
	exitOnError = flag.Bool(
		"exit-on-error",
		true,
		"Exit with a message when there's an error",
	)
	verbose = flag.Bool(
		"verbose",
		false,
		"Write logs to terminal",
	)
	fakeAvgAggregates = flag.Bool(
		"fake-avg-aggregates",
		true,
		"Generate sum/cnt series out of avg series to accomodate metrictank",
	)
	httpEndpoint = flag.String(
		"http-endpoint",
		"127.0.0.1:8080",
		"The http endpoint to listen on",
	)
	cassandraAddrs = flag.String(
		"cassandra-addrs",
		"localhost",
		"cassandra host (may be given multiple times as comma-separated list)",
	)
	cassandraKeyspace = flag.String(
		"cassandra-keyspace",
		"metrictank",
		"cassandra keyspace to use for storing the metric data table",
	)
	ttlsStr = flag.String(
		"ttls",
		"35d",
		"list of ttl strings used by MT separated by ','",
	)
	windowFactor = flag.Int(
		"window-factor",
		20,
		"the window factor be used when creating the metric table schema",
	)
	partitionScheme = flag.String(
		"partition-scheme",
		"bySeries",
		"method used for partitioning metrics. This should match the settings of tsdb-gw. (byOrg|bySeries)",
	)
	uriPath = flag.String(
		"uri-path",
		"/chunks",
		"the URI on which we expect chunks to get posted",
	)
	numPartitions = flag.Int(
		"num-partitions",
		1,
		"Number of Partitions",
	)
	GitHash   = "(none)"
	printLock sync.Mutex
)

type Server struct {
	Cluster     *gocql.ClusterConfig
	Session     *gocql.Session
	TTLTables   mdata.TTLTables
	Partitioner partitioner.Partitioner
	Index       idx.MetricIndex
}

func main() {
	cassandra.ConfigSetup()
	flag.Parse()

	cassCluster := gocql.NewCluster(strings.Split(*cassandraAddrs, ",")...)
	cassCluster.Consistency = gocql.ParseConsistency("one")
	cassCluster.Timeout = time.Second
	cassCluster.NumConns = 2
	cassCluster.ProtoVersion = 4
	cassCluster.Keyspace = *cassandraKeyspace

	session, err := cassCluster.CreateSession()
	if err != nil {
		panic(fmt.Sprintf("Failed to create cassandra session: %q", err))
	}

	splits := strings.Split(*ttlsStr, ",")
	ttls := make([]uint32, 0)
	for _, split := range splits {
		ttls = append(ttls, dur.MustParseUNsec("ttl", split))
	}
	ttlTables := mdata.GetTTLTables(ttls, *windowFactor, mdata.Table_name_format)

	p, err := partitioner.NewKafka(*partitionScheme)
	if err != nil {
		panic(fmt.Sprintf("Failed to instantiate partitioner: %q", err))
	}

	server := &Server{
		Cluster:     cassCluster,
		Session:     session,
		TTLTables:   ttlTables,
		Partitioner: p,
		Index:       cassandra.New(),
	}
	cluster.Init("mt-whisper-importer-writer", GitHash, time.Now(), "http", int(80))
	server.Index.Init()

	http.HandleFunc(*uriPath, server.chunksHandler)

	log(fmt.Sprintf("Listening on %q", *httpEndpoint))
	err = http.ListenAndServe(*httpEndpoint, nil)
	if err != nil {
		panic(fmt.Sprintf("Error creating listener: %q", err))
	}
}

func throwError(msg string) {
	msg = fmt.Sprintf("%s\n", msg)
	if *exitOnError {
		panic(msg)
	} else {
		printLock.Lock()
		fmt.Fprintln(os.Stderr, msg)
		printLock.Unlock()
	}
}

func log(msg string) {
	if *verbose {
		printLock.Lock()
		fmt.Println(msg)
		printLock.Unlock()
	}
}

func (s *Server) chunksHandler(w http.ResponseWriter, req *http.Request) {
	metric := &archive.Metric{}
	err := metric.UnmarshalCompressed(req.Body)
	if err != nil {
		throwError(fmt.Sprintf("Error decoding metric stream: %q", err))
		return
	}
	log("Handling new metric")

	if len(metric.Archives) == 0 {
		throwError("Metric has no archives")
		return
	}

	avg := (whisper.AggregationMethod(metric.AggregationMethod) == whisper.AggregationAverage)

	partition, err := s.Partitioner.Partition(&metric.MetricData, int32(*numPartitions))
	s.Index.AddOrUpdate(&metric.MetricData, partition)

	for archiveIdx, a := range metric.Archives {
		archiveTTL := a.SecondsPerPoint * a.Points
		tableTTL, err := s.selectTableByTTL(archiveTTL)
		if err != nil {
			throwError(fmt.Sprintf("Failed to select table for ttl %d in %+v: %q", archiveTTL, s.TTLTables, err))
			return
		}
		entry, ok := s.TTLTables[tableTTL]
		if !ok {
			throwError(fmt.Sprintf("Failed to get selected table %d in %+v", tableTTL, s.TTLTables))
			return
		}
		tableName := entry.Table

		if !avg || archiveIdx == 0 || !*fakeAvgAggregates {
			log(fmt.Sprintf(
				"inserting %d chunks of archive %d with ttl %d into table %s with ttl %d and key %s",
				len(a.Chunks), archiveIdx, archiveTTL, tableName, tableTTL, a.RowKey,
			))
			s.insertChunks(tableName, a.RowKey, tableTTL, a.Chunks)
		} else {
			// averaged archives are a special case because mt doesn't store them as such.
			// mt reconstructs the averages on the fly from the sum and cnt series, so we need
			// to generate these two series out of raw averaged data by multiplying each point
			// with the aggregation span and storing the result as sum, cnt is the aggregation span.

			// aggCount is the aggregation span of this archive divided by the raw interval
			aggCount := a.SecondsPerPoint / metric.Archives[0].SecondsPerPoint
			aggSpan := a.SecondsPerPoint

			sumArchive := make([]chunk.IterGen, 0, len(a.Chunks))
			cntArchive := make([]chunk.IterGen, 0, len(a.Chunks))
			for _, ig := range a.Chunks {
				T0 := ig.Ts
				sum := chunk.New(T0)
				cnt := chunk.New(T0)

				it, err := ig.Get()
				if err != nil {
					throwError(fmt.Sprintf("failed to get iterator from itergen: %q", err))
					continue
				}

				for it.Next() {
					ts, val := it.Values()
					cnt.Push(ts, float64(aggCount))
					sum.Push(ts, val*float64(aggCount))
				}

				cnt.Finish()
				sum.Finish()

				cntArchive = append(cntArchive, *chunk.NewBareIterGen(cnt.Bytes(), T0, aggSpan))
				sumArchive = append(sumArchive, *chunk.NewBareIterGen(sum.Bytes(), T0, aggSpan))
			}

			cntId := api.AggMetricKey(metric.MetricData.Id, "cnt", aggSpan)
			sumId := api.AggMetricKey(metric.MetricData.Id, "sum", aggSpan)

			log(fmt.Sprintf(
				"inserting 2 archives of %d chunks per archive with ttl %d into table %s with ttl %d and keys %s/%s",
				len(a.Chunks), archiveTTL, tableName, tableTTL, cntId, sumId,
			))

			s.insertChunks(tableName, cntId, tableTTL, cntArchive)
			s.insertChunks(tableName, sumId, tableTTL, sumArchive)
		}
	}
}

func (s *Server) insertChunks(table, id string, ttl uint32, itergens []chunk.IterGen) {
	query := fmt.Sprintf("INSERT INTO %s (key, ts, data) values (?,?,?) USING TTL %d", table, ttl)
	for _, ig := range itergens {
		rowKey := fmt.Sprintf("%s_%d", id, ig.Ts/mdata.Month_sec)
		err := s.Session.Query(query, rowKey, ig.Ts, mdata.PrepareChunkData(ig.Span, ig.Bytes())).Exec()
		if err != nil {
			throwError(fmt.Sprintf("Error in query: %q", err))
		}
	}
}

func (s *Server) selectTableByTTL(ttl uint32) (uint32, error) {
	selectedTTL := uint32(math.MaxUint32)

	// find the table with the smallest TTL that is at least equal to archiveTTL
	for tableTTL := range s.TTLTables {
		if tableTTL >= ttl {
			if selectedTTL > tableTTL {
				selectedTTL = tableTTL
			}
		}
	}

	// we have not found a table that can accommodate the requested ttl
	if selectedTTL == math.MaxUint32 {
		return 0, errors.New(fmt.Sprintf("No Table found that can hold TTL %d", ttl))
	}

	return selectedTTL, nil
}
<|endoftext|>


<commit_before>package capthis

import (
	"fmt"
	"testing"
	"./capthis"
)

func TestNew(t *testing.T) {
	caption := capthis.New()
	if (fmt.Sprintf("%T", caption) != "*capthis.Caption") {
		t.Error("Expected type *capthis.Caption")
	}

	if (fmt.Sprintf("%T", caption.MagicWand()) != "*imagick.MagickWand") {
		t.Error("Expected type *imagick.MagickWand")
	}

	if (fmt.Sprintf("%T", caption.PixelWand()) != "*imagick.PixelWand") {
		t.Error("Expected type *imagick.PixelWand")
	}

	if (fmt.Sprintf("%T", caption.DrawingWand()) != "*imagick.DrawingWand") {
		t.Error("Expected type *imagick.DrawingWand")
	}
}
<commit_msg>Tabs to spaces<commit_after>package capthis

import (
    "fmt"
    "testing"
    "./capthis"
)

func TestNew(t *testing.T) {
    caption := capthis.New()
    if (fmt.Sprintf("%T", caption) != "*capthis.Caption") {
        t.Error("Expected type *capthis.Caption")
    }

    if (fmt.Sprintf("%T", caption.MagicWand()) != "*imagick.MagickWand") {
        t.Error("Expected type *imagick.MagickWand")
    }

    if (fmt.Sprintf("%T", caption.PixelWand()) != "*imagick.PixelWand") {
        t.Error("Expected type *imagick.PixelWand")
    }

    if (fmt.Sprintf("%T", caption.DrawingWand()) != "*imagick.DrawingWand") {
        t.Error("Expected type *imagick.DrawingWand")
    }
}
<|endoftext|>


<commit_before>// The fuse package provides APIs to implement filesystems in
// userspace, using libfuse on Linux.  Typically, each call of the API
// happens in its own goroutine, so take care to make the file system
// thread-safe.

package fuse

import (
	"os"
)

// Types for users to implement.

// NodeFileSystem is a high level API that resembles the kernel's idea
// of what an FS looks like.  NodeFileSystems can have multiple
// hard-links to one file, for example. It is also suited if the data
// to represent fits in memory: you can construct FsNode at mount
// time, and the filesystem will be ready.
type NodeFileSystem interface {
	OnUnmount()
	OnMount(conn *FileSystemConnector)
	StatFs() *StatfsOut
	Root() FsNode
}

type FsNode interface {
	// The following are called by the FileSystemConnector
	Inode() *Inode
	SetInode(node *Inode)

	Lookup(name string, context *Context) (fi *os.FileInfo, node FsNode, code Status)
	OnForget()

	// Misc.
	Access(mode uint32, context *Context) (code Status)
	Readlink(c *Context) ([]byte, Status)

	// Namespace operations
	Mknod(name string, mode uint32, dev uint32, context *Context) (fi *os.FileInfo, newNode FsNode, code Status)
	Mkdir(name string, mode uint32, context *Context) (fi *os.FileInfo, newNode FsNode, code Status)
	Unlink(name string, context *Context) (code Status)
	Rmdir(name string, context *Context) (code Status)
	Symlink(name string, content string, context *Context) (fi *os.FileInfo, newNode FsNode, code Status)
	Rename(oldName string, newParent FsNode, newName string, context *Context) (code Status)
	Link(name string, existing FsNode, context *Context) (fi *os.FileInfo, newNode FsNode, code Status)

	// Files
	Create(name string, flags uint32, mode uint32, context *Context) (file File, fi *os.FileInfo, newNode FsNode, code Status)
	Open(flags uint32, context *Context) (file File, code Status)
	OpenDir(context *Context) (chan DirEntry, Status)

	// XAttrs
	GetXAttr(attribute string, context *Context) (data []byte, code Status)
	RemoveXAttr(attr string, context *Context) Status
	SetXAttr(attr string, data []byte, flags int, context *Context) Status
	ListXAttr(context *Context) (attrs []string, code Status)

	// Attributes
	GetAttr(file File, context *Context) (fi *os.FileInfo, code Status)
	Chmod(file File, perms uint32, context *Context) (code Status)
	Chown(file File, uid uint32, gid uint32, context *Context) (code Status)
	Truncate(file File, size uint64, context *Context) (code Status)
	Utimens(file File, atime uint64, mtime uint64, context *Context) (code Status)
}

// A filesystem API that uses paths rather than inodes.  A minimal
// file system should have at least a functional GetAttr method.
// Typically, each call happens in its own goroutine, so take care to
// make the file system thread-safe.
//
// Include DefaultFileSystem to provide a default null implementation of
// required methods.
type FileSystem interface {
	// Used for pretty printing.
	Name() string

	// Attributes.  This function is the main entry point, through
	// which FUSE discovers which files and directories exist.
	//
	// If the filesystem wants to implement hard-links, it should
	// return consistent non-zero FileInfo.Ino data.  Using
	// hardlinks incurs a performance hit.
	GetAttr(name string, context *Context) (*os.FileInfo, Status)

	// These should update the file's ctime too.
	Chmod(name string, mode uint32, context *Context) (code Status)
	Chown(name string, uid uint32, gid uint32, context *Context) (code Status)
	Utimens(name string, AtimeNs uint64, MtimeNs uint64, context *Context) (code Status)

	Truncate(name string, offset uint64, context *Context) (code Status)

	Access(name string, mode uint32, context *Context) (code Status)

	// Tree structure
	Link(oldName string, newName string, context *Context) (code Status)
	Mkdir(name string, mode uint32, context *Context) Status
	Mknod(name string, mode uint32, dev uint32, context *Context) Status
	Rename(oldName string, newName string, context *Context) (code Status)
	Rmdir(name string, context *Context) (code Status)
	Unlink(name string, context *Context) (code Status)

	// Extended attributes.
	GetXAttr(name string, attribute string, context *Context) (data []byte, code Status)
	ListXAttr(name string, context *Context) (attributes []string, code Status)
	RemoveXAttr(name string, attr string, context *Context) Status
	SetXAttr(name string, attr string, data []byte, flags int, context *Context) Status

	// Called after mount.
	OnMount(nodeFs *PathNodeFs)
	OnUnmount()

	// File handling.  If opening for writing, the file's mtime
	// should be updated too.
	Open(name string, flags uint32, context *Context) (file File, code Status)
	Create(name string, flags uint32, mode uint32, context *Context) (file File, code Status)

	// Directory handling
	OpenDir(name string, context *Context) (stream chan DirEntry, code Status)

	// Symlinks.
	Symlink(value string, linkName string, context *Context) (code Status)
	Readlink(name string, context *Context) (string, Status)

	StatFs() *StatfsOut
}

// A File object should be returned from FileSystem.Open and
// FileSystem.Create.  Include DefaultFile into the struct to inherit
// a default null implementation.
//
// TODO - should File be thread safe?
// TODO - should we pass a *Context argument?
type File interface {
	// Called upon registering the filehandle in the inode.
	SetInode(*Inode)

	// Wrappers around other File implementations, should return
	// the inner file here.
	InnerFile() File

	Read(*ReadIn, BufferPool) ([]byte, Status)
	Write(*WriteIn, []byte) (written uint32, code Status)
	Truncate(size uint64) Status

	GetAttr() (*os.FileInfo, Status)
	Chown(uid uint32, gid uint32) Status
	Chmod(perms uint32) Status
	Utimens(atimeNs uint64, mtimeNs uint64) Status
	Flush() Status
	Release()
	Fsync(*FsyncIn) (code Status)
}

// Wrap a File return in this to set FUSE flags.  Also used internally
// to store open file data.
type WithFlags struct {
	File

	// For debugging.
	Description string

	// Put FOPEN_* flags here.
	FuseFlags uint32

	// O_RDWR, O_TRUNCATE, etc.
	OpenFlags uint32
}

// MountOptions contains time out options for a (Node)FileSystem.  The
// default copied from libfuse and set in NewMountOptions() is
// (1s,1s,0s).
type FileSystemOptions struct {
	EntryTimeout    float64
	AttrTimeout     float64
	NegativeTimeout float64

	// If set, replace all uids with given UID.
	// NewFileSystemOptions() will set this to the daemon's
	// uid/gid.
	*Owner

	// If set, use a more portable, but slower inode number
	// generation scheme.  This will make inode numbers (exported
	// back to callers) stay within int32, which is necessary for
	// making stat() succeed in 32-bit programs.
	PortableInodes bool
}

type MountOptions struct {
	AllowOther bool

	// Options are passed as -o string to fusermount.
	Options []string

	// Default is _DEFAULT_BACKGROUND_TASKS, 12.  This numbers
	// controls the allowed number of requests that relate to
	// async I/O.  Concurrency for synchronous I/O is not limited.
	MaxBackground int
}

// DefaultFileSystem implements a FileSystem that returns ENOSYS for every operation.
type DefaultFileSystem struct{}

// DefaultFile returns ENOSYS for every operation.
type DefaultFile struct{}

// RawFileSystem is an interface close to the FUSE wire protocol.
//
// Unless you really know what you are doing, you should not implement
// this, but rather the FileSystem interface; the details of getting
// interactions with open files, renames, and threading right etc. are
// somewhat tricky and not very interesting.
//
// Include DefaultRawFileSystem to inherit a null implementation.
type RawFileSystem interface {
	Lookup(header *InHeader, name string) (out *EntryOut, status Status)
	Forget(header *InHeader, input *ForgetIn)

	// Attributes.
	GetAttr(header *InHeader, input *GetAttrIn) (out *AttrOut, code Status)
	SetAttr(header *InHeader, input *SetAttrIn) (out *AttrOut, code Status)

	// Modifying structure.
	Mknod(header *InHeader, input *MknodIn, name string) (out *EntryOut, code Status)
	Mkdir(header *InHeader, input *MkdirIn, name string) (out *EntryOut, code Status)
	Unlink(header *InHeader, name string) (code Status)
	Rmdir(header *InHeader, name string) (code Status)
	Rename(header *InHeader, input *RenameIn, oldName string, newName string) (code Status)
	Link(header *InHeader, input *LinkIn, filename string) (out *EntryOut, code Status)

	Symlink(header *InHeader, pointedTo string, linkName string) (out *EntryOut, code Status)
	Readlink(header *InHeader) (out []byte, code Status)
	Access(header *InHeader, input *AccessIn) (code Status)

	// Extended attributes.
	GetXAttr(header *InHeader, attr string) (data []byte, code Status)
	ListXAttr(header *InHeader) (attributes []byte, code Status)
	SetXAttr(header *InHeader, input *SetXAttrIn, attr string, data []byte) Status
	RemoveXAttr(header *InHeader, attr string) (code Status)

	// File handling.
	Create(header *InHeader, input *CreateIn, name string) (flags uint32, handle uint64, out *EntryOut, code Status)
	Open(header *InHeader, input *OpenIn) (flags uint32, handle uint64, status Status)
	Read(*InHeader, *ReadIn, BufferPool) ([]byte, Status)

	Release(header *InHeader, input *ReleaseIn)
	Write(*InHeader, *WriteIn, []byte) (written uint32, code Status)
	Flush(header *InHeader, input *FlushIn) Status
	Fsync(*InHeader, *FsyncIn) (code Status)

	// Directory handling
	OpenDir(header *InHeader, input *OpenIn) (flags uint32, handle uint64, status Status)
	ReadDir(header *InHeader, input *ReadIn) (*DirEntryList, Status)
	ReleaseDir(header *InHeader, input *ReleaseIn)
	FsyncDir(header *InHeader, input *FsyncIn) (code Status)

	//
	Ioctl(header *InHeader, input *IoctlIn) (output *IoctlOut, data []byte, code Status)
	StatFs() *StatfsOut

	// Provide callbacks for pushing notifications to the kernel.
	Init(params *RawFsInit)
}

// DefaultRawFileSystem returns ENOSYS for every operation.
type DefaultRawFileSystem struct{}

// Talk back to FUSE.
//
// InodeNotify invalidates the information associated with the inode
// (ie. data cache, attributes, etc.)
//
// EntryNotify should be used if the existence status of an entry changes,
// (ie. to notify of creation or deletion of the file).
//
// Somewhat confusingly, InodeNotify for a file that stopped to exist
// will give the correct result for Lstat (ENOENT), but the kernel
// will still issue file Open() on the inode.
type RawFsInit struct {
	InodeNotify func(*NotifyInvalInodeOut) Status
	EntryNotify func(parent uint64, name string) Status
}
<commit_msg>Fix naming in api.go.<commit_after>// The fuse package provides APIs to implement filesystems in
// userspace, using libfuse on Linux.  Typically, each call of the API
// happens in its own goroutine, so take care to make the file system
// thread-safe.

package fuse

import (
	"os"
)

// Types for users to implement.

// NodeFileSystem is a high level API that resembles the kernel's idea
// of what an FS looks like.  NodeFileSystems can have multiple
// hard-links to one file, for example. It is also suited if the data
// to represent fits in memory: you can construct FsNode at mount
// time, and the filesystem will be ready.
type NodeFileSystem interface {
	OnUnmount()
	OnMount(conn *FileSystemConnector)
	StatFs() *StatfsOut
	Root() FsNode
}

type FsNode interface {
	// The following are called by the FileSystemConnector
	Inode() *Inode
	SetInode(node *Inode)

	Lookup(name string, context *Context) (fi *os.FileInfo, node FsNode, code Status)
	OnForget()

	// Misc.
	Access(mode uint32, context *Context) (code Status)
	Readlink(c *Context) ([]byte, Status)

	// Namespace operations
	Mknod(name string, mode uint32, dev uint32, context *Context) (fi *os.FileInfo, newNode FsNode, code Status)
	Mkdir(name string, mode uint32, context *Context) (fi *os.FileInfo, newNode FsNode, code Status)
	Unlink(name string, context *Context) (code Status)
	Rmdir(name string, context *Context) (code Status)
	Symlink(name string, content string, context *Context) (fi *os.FileInfo, newNode FsNode, code Status)
	Rename(oldName string, newParent FsNode, newName string, context *Context) (code Status)
	Link(name string, existing FsNode, context *Context) (fi *os.FileInfo, newNode FsNode, code Status)

	// Files
	Create(name string, flags uint32, mode uint32, context *Context) (file File, fi *os.FileInfo, newNode FsNode, code Status)
	Open(flags uint32, context *Context) (file File, code Status)
	OpenDir(context *Context) (chan DirEntry, Status)

	// XAttrs
	GetXAttr(attribute string, context *Context) (data []byte, code Status)
	RemoveXAttr(attr string, context *Context) Status
	SetXAttr(attr string, data []byte, flags int, context *Context) Status
	ListXAttr(context *Context) (attrs []string, code Status)

	// Attributes
	GetAttr(file File, context *Context) (fi *os.FileInfo, code Status)
	Chmod(file File, perms uint32, context *Context) (code Status)
	Chown(file File, uid uint32, gid uint32, context *Context) (code Status)
	Truncate(file File, size uint64, context *Context) (code Status)
	Utimens(file File, atime uint64, mtime uint64, context *Context) (code Status)
}

// A filesystem API that uses paths rather than inodes.  A minimal
// file system should have at least a functional GetAttr method.
// Typically, each call happens in its own goroutine, so take care to
// make the file system thread-safe.
//
// Include DefaultFileSystem to provide a default null implementation of
// required methods.
type FileSystem interface {
	// Used for pretty printing.
	Name() string

	// Attributes.  This function is the main entry point, through
	// which FUSE discovers which files and directories exist.
	//
	// If the filesystem wants to implement hard-links, it should
	// return consistent non-zero FileInfo.Ino data.  Using
	// hardlinks incurs a performance hit.
	GetAttr(name string, context *Context) (*os.FileInfo, Status)

	// These should update the file's ctime too.
	Chmod(name string, mode uint32, context *Context) (code Status)
	Chown(name string, uid uint32, gid uint32, context *Context) (code Status)
	Utimens(name string, AtimeNs uint64, MtimeNs uint64, context *Context) (code Status)

	Truncate(name string, size uint64, context *Context) (code Status)

	Access(name string, mode uint32, context *Context) (code Status)

	// Tree structure
	Link(oldName string, newName string, context *Context) (code Status)
	Mkdir(name string, mode uint32, context *Context) Status
	Mknod(name string, mode uint32, dev uint32, context *Context) Status
	Rename(oldName string, newName string, context *Context) (code Status)
	Rmdir(name string, context *Context) (code Status)
	Unlink(name string, context *Context) (code Status)

	// Extended attributes.
	GetXAttr(name string, attribute string, context *Context) (data []byte, code Status)
	ListXAttr(name string, context *Context) (attributes []string, code Status)
	RemoveXAttr(name string, attr string, context *Context) Status
	SetXAttr(name string, attr string, data []byte, flags int, context *Context) Status

	// Called after mount.
	OnMount(nodeFs *PathNodeFs)
	OnUnmount()

	// File handling.  If opening for writing, the file's mtime
	// should be updated too.
	Open(name string, flags uint32, context *Context) (file File, code Status)
	Create(name string, flags uint32, mode uint32, context *Context) (file File, code Status)

	// Directory handling
	OpenDir(name string, context *Context) (stream chan DirEntry, code Status)

	// Symlinks.
	Symlink(value string, linkName string, context *Context) (code Status)
	Readlink(name string, context *Context) (string, Status)

	StatFs() *StatfsOut
}

// A File object should be returned from FileSystem.Open and
// FileSystem.Create.  Include DefaultFile into the struct to inherit
// a default null implementation.
//
// TODO - should File be thread safe?
// TODO - should we pass a *Context argument?
type File interface {
	// Called upon registering the filehandle in the inode.
	SetInode(*Inode)

	// Wrappers around other File implementations, should return
	// the inner file here.
	InnerFile() File

	Read(*ReadIn, BufferPool) ([]byte, Status)
	Write(*WriteIn, []byte) (written uint32, code Status)
	Truncate(size uint64) Status

	GetAttr() (*os.FileInfo, Status)
	Chown(uid uint32, gid uint32) Status
	Chmod(perms uint32) Status
	Utimens(atimeNs uint64, mtimeNs uint64) Status
	Flush() Status
	Release()
	Fsync(*FsyncIn) (code Status)
}

// Wrap a File return in this to set FUSE flags.  Also used internally
// to store open file data.
type WithFlags struct {
	File

	// For debugging.
	Description string

	// Put FOPEN_* flags here.
	FuseFlags uint32

	// O_RDWR, O_TRUNCATE, etc.
	OpenFlags uint32
}

// MountOptions contains time out options for a (Node)FileSystem.  The
// default copied from libfuse and set in NewMountOptions() is
// (1s,1s,0s).
type FileSystemOptions struct {
	EntryTimeout    float64
	AttrTimeout     float64
	NegativeTimeout float64

	// If set, replace all uids with given UID.
	// NewFileSystemOptions() will set this to the daemon's
	// uid/gid.
	*Owner

	// If set, use a more portable, but slower inode number
	// generation scheme.  This will make inode numbers (exported
	// back to callers) stay within int32, which is necessary for
	// making stat() succeed in 32-bit programs.
	PortableInodes bool
}

type MountOptions struct {
	AllowOther bool

	// Options are passed as -o string to fusermount.
	Options []string

	// Default is _DEFAULT_BACKGROUND_TASKS, 12.  This numbers
	// controls the allowed number of requests that relate to
	// async I/O.  Concurrency for synchronous I/O is not limited.
	MaxBackground int
}

// DefaultFileSystem implements a FileSystem that returns ENOSYS for every operation.
type DefaultFileSystem struct{}

// DefaultFile returns ENOSYS for every operation.
type DefaultFile struct{}

// RawFileSystem is an interface close to the FUSE wire protocol.
//
// Unless you really know what you are doing, you should not implement
// this, but rather the FileSystem interface; the details of getting
// interactions with open files, renames, and threading right etc. are
// somewhat tricky and not very interesting.
//
// Include DefaultRawFileSystem to inherit a null implementation.
type RawFileSystem interface {
	Lookup(header *InHeader, name string) (out *EntryOut, status Status)
	Forget(header *InHeader, input *ForgetIn)

	// Attributes.
	GetAttr(header *InHeader, input *GetAttrIn) (out *AttrOut, code Status)
	SetAttr(header *InHeader, input *SetAttrIn) (out *AttrOut, code Status)

	// Modifying structure.
	Mknod(header *InHeader, input *MknodIn, name string) (out *EntryOut, code Status)
	Mkdir(header *InHeader, input *MkdirIn, name string) (out *EntryOut, code Status)
	Unlink(header *InHeader, name string) (code Status)
	Rmdir(header *InHeader, name string) (code Status)
	Rename(header *InHeader, input *RenameIn, oldName string, newName string) (code Status)
	Link(header *InHeader, input *LinkIn, filename string) (out *EntryOut, code Status)

	Symlink(header *InHeader, pointedTo string, linkName string) (out *EntryOut, code Status)
	Readlink(header *InHeader) (out []byte, code Status)
	Access(header *InHeader, input *AccessIn) (code Status)

	// Extended attributes.
	GetXAttr(header *InHeader, attr string) (data []byte, code Status)
	ListXAttr(header *InHeader) (attributes []byte, code Status)
	SetXAttr(header *InHeader, input *SetXAttrIn, attr string, data []byte) Status
	RemoveXAttr(header *InHeader, attr string) (code Status)

	// File handling.
	Create(header *InHeader, input *CreateIn, name string) (flags uint32, handle uint64, out *EntryOut, code Status)
	Open(header *InHeader, input *OpenIn) (flags uint32, handle uint64, status Status)
	Read(*InHeader, *ReadIn, BufferPool) ([]byte, Status)

	Release(header *InHeader, input *ReleaseIn)
	Write(*InHeader, *WriteIn, []byte) (written uint32, code Status)
	Flush(header *InHeader, input *FlushIn) Status
	Fsync(*InHeader, *FsyncIn) (code Status)

	// Directory handling
	OpenDir(header *InHeader, input *OpenIn) (flags uint32, handle uint64, status Status)
	ReadDir(header *InHeader, input *ReadIn) (*DirEntryList, Status)
	ReleaseDir(header *InHeader, input *ReleaseIn)
	FsyncDir(header *InHeader, input *FsyncIn) (code Status)

	//
	Ioctl(header *InHeader, input *IoctlIn) (output *IoctlOut, data []byte, code Status)
	StatFs() *StatfsOut

	// Provide callbacks for pushing notifications to the kernel.
	Init(params *RawFsInit)
}

// DefaultRawFileSystem returns ENOSYS for every operation.
type DefaultRawFileSystem struct{}

// Talk back to FUSE.
//
// InodeNotify invalidates the information associated with the inode
// (ie. data cache, attributes, etc.)
//
// EntryNotify should be used if the existence status of an entry changes,
// (ie. to notify of creation or deletion of the file).
//
// Somewhat confusingly, InodeNotify for a file that stopped to exist
// will give the correct result for Lstat (ENOENT), but the kernel
// will still issue file Open() on the inode.
type RawFsInit struct {
	InodeNotify func(*NotifyInvalInodeOut) Status
	EntryNotify func(parent uint64, name string) Status
}
<|endoftext|>


<commit_before>package garchive

import (
	"bufio"
	"errors"
	"fmt"
	"log"
	"net"
	"os"
	"strings"
	"time"
        "os/user"
)

type IRCConnection struct {
	Location   string
	Nick       string
	Connection net.Conn
	Listeners  []func(*IRCCommand)
	Finished   chan bool
}

type IRCCommand struct {
	Source string
	Type   string
	Args   []string
}

type IRCListener func(*IRCCommand)

func MakeConnection(uri string, nick string) *IRCConnection {
	var connection IRCConnection

	connection.Location = uri
	connection.Nick = nick

	return &connection
}

/* go from a raw string form of a command to an IRCCommand */
func RawToIRCCommand(raw string) (*IRCCommand, error) {
	var command IRCCommand

	split_ver := strings.Split(raw, " ")
	/* first as a sanity check make sure that our array has at least
	   two entries, any less is not a valid command */
	if len(split_ver) < 2 {
		return &command, errors.New("invalid command (less than two entries in command)")
	}
	args_start := 2
	if strings.HasPrefix(split_ver[0], ":") {
		command.Source = strings.TrimPrefix(split_ver[0], ":")
		command.Type = split_ver[1]
	} else {
		command.Type = split_ver[0]
		args_start = 1
	}

	/* iterate over every element after the first two */
	multi_word_index := -1
	for index, arg := range split_ver[args_start:] {
		if strings.HasPrefix(arg, ":") {
			multi_word_index = index
			break
		}

		command.Args = append(command.Args, arg)
	}

	if multi_word_index != -1 {
		words := []string{}
		words = append(words, split_ver[args_start:][multi_word_index][1:len(split_ver[args_start:][multi_word_index])])
		words = append(words, split_ver[args_start:][multi_word_index+1:]...)
		command.Args = append(command.Args, strings.Join(words, " "))
	}

	command.Args[len(command.Args)-1] = strings.TrimSuffix(command.Args[len(command.Args)-1], "\r\n")

	return &command, nil
}

func MakeIRCCommand(cmdtype string, args ...string) *IRCCommand {
	var command IRCCommand

	command.Type = cmdtype
	command.Args = args

	return &command
}

func (command *IRCCommand) ToRaw() (string, error) {
	out := []string{}
	if command.Source != "" {
		out = append(out, command.Source)
	}
	out = append(out, command.Type)
	for _, arg := range command.Args[0 : len(command.Args)-1] {
		if strings.Contains(arg, " ") {
			return "", errors.New("nonfinal argument contains space")
		}
		out = append(out, arg)
	}

	if strings.Contains(command.Args[len(command.Args)-1], " ") {
		out = append(out, fmt.Sprint(":", command.Args[len(command.Args)-1]))
	} else {
		out = append(out, command.Args[len(command.Args)-1])
	}

	return fmt.Sprintf("%s\r\n", strings.Join(out, " ")), nil
}

func (connection *IRCConnection) Connect() error {
	log.Print("Connecting...")
	conn, err := net.Dial("tcp", connection.Location)
	connection.Connection = conn

	if err != nil {
		return err
	}

	log.Print("Connected!")

	go func() {
		//bio := bufio.NewReader(conn)
		for {
			line, err := bufio.NewReader(conn).ReadString('\n')
			log.Printf("Got line: %s\n", line)
			if err != nil {
				connection.Finished <- true
				log.Fatal(err)
			}
			command, err := RawToIRCCommand(line)
			if err != nil {
				log.Print(err)
			}
			for _, fn := range connection.Listeners {
				go fn(command)
			}
		}
	}()

	/* create a goroutine to send PONGs back when we receive PINGs */
	connection.AddListener(func(command *IRCCommand) {
		if command.Type == "PING" {
			fmt.Fprintf(conn, "PONG %s\r\n", command.Args[0])
		}
	})

	err = connection.SendCommand(MakeIRCCommand("NICK", "garchive"))
	if err != nil {
		return err
	}
        /* query the local system for a username. This isn't *really* necessary,
         * but it really isn't that big of a deal to do it aywa */
        user, err := user.Current()
        if err != nil {
          log.Print(err)
          user.Username = "unknown"
        }
        err = connection.SendCommand(MakeIRCCommand("USER", user.Username, "0", "*", "Garchive: An IRC archiver bot"))
	if err != nil {
		return err
	}

	return nil
}

func (connection *IRCConnection) AddListener(fn IRCListener) {
	connection.Listeners = append(connection.Listeners, fn)
}

func (connection *IRCConnection) SendCommand(command *IRCCommand) error {
	log.Printf("Sending command %v\n", command)
	raw_form, err := command.ToRaw()
	log.Printf("raw form: %v\n", raw_form)
	if err != nil {
		return err
	}

	fmt.Fprint(connection.Connection, raw_form)

	return nil
}

func MakeChannelListener(channel string, filename string) (func(*IRCCommand), error) {
	file, err := os.OpenFile(filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
	if err != nil {
		return nil, err
	}

	fn := func(command *IRCCommand) {
		line := ""
		switch command.Type {
		case "PRIVMSG":
			if command.Args[0] == channel {
				line = fmt.Sprintf("%s: %s\n", command.Source, command.Args[1])
			}
		case "JOIN":
			if command.Args[0] == channel {
				line = fmt.Sprintf("%s has joined %s\n", command.Source, channel)
			}
		case "PART":
			if command.Args[0] == channel {
				line = fmt.Sprintf("%s has left %s\n", command.Source, channel)
			}
		case "TOPIC":
			if command.Args[0] == channel {
				line = fmt.Sprintf("%s has set the topic to %s\n", command.Source, command.Args[1])
			}
		}

		if line != "" {
			fmt.Fprintf(file, "[%v] %s", time.Now(), line)
		}
	}

	return fn, nil
}

func Main() {
        if len(os.Args) < 4 {
          fmt.Printf("Usage: %s irc_uri nick channel [other_channels...]\n", os.Args[0])
          os.Exit(1)
        }
        uri := os.Args[1]
        nick := os.Args[2]
        channels := os.Args[3:len(os.Args)]
	connection := MakeConnection(uri, nick)

	connection.AddListener(func(command *IRCCommand) {
		fmt.Printf("%v\n", command)
		fmt.Printf("Args:\n")
		for index, arg := range command.Args {
			fmt.Printf("%d.  %s\n", index, arg)
		}
	})

        for _, c := range channels {
          fmt.Printf("Adding channel %v\n", c)
        }

	err := connection.Connect()
	if err != nil {
		log.Fatal(err)
	}

        for _, c := range channels {
          /* allow the user to specify the channel without the #
           * since bash requries it to be escaped */
           if !strings.HasPrefix(c, "#") {
             c = fmt.Sprintf("#%s", c)
           }
           err = connection.SendCommand(MakeIRCCommand("JOIN", c))
           if err != nil {
             log.Print(err)
           }
           channelListener, err := MakeChannelListener(c, strings.TrimPrefix(c, "#"))
           if err != nil {
             log.Print(err)
           }
           connection.AddListener(channelListener)
        }

	<-connection.Finished
}
<commit_msg>forgot to 'go fmt'<commit_after>package garchive

import (
	"bufio"
	"errors"
	"fmt"
	"log"
	"net"
	"os"
	"os/user"
	"strings"
	"time"
)

type IRCConnection struct {
	Location   string
	Nick       string
	Connection net.Conn
	Listeners  []func(*IRCCommand)
	Finished   chan bool
}

type IRCCommand struct {
	Source string
	Type   string
	Args   []string
}

type IRCListener func(*IRCCommand)

func MakeConnection(uri string, nick string) *IRCConnection {
	var connection IRCConnection

	connection.Location = uri
	connection.Nick = nick

	return &connection
}

/* go from a raw string form of a command to an IRCCommand */
func RawToIRCCommand(raw string) (*IRCCommand, error) {
	var command IRCCommand

	split_ver := strings.Split(raw, " ")
	/* first as a sanity check make sure that our array has at least
	   two entries, any less is not a valid command */
	if len(split_ver) < 2 {
		return &command, errors.New("invalid command (less than two entries in command)")
	}
	args_start := 2
	if strings.HasPrefix(split_ver[0], ":") {
		command.Source = strings.TrimPrefix(split_ver[0], ":")
		command.Type = split_ver[1]
	} else {
		command.Type = split_ver[0]
		args_start = 1
	}

	/* iterate over every element after the first two */
	multi_word_index := -1
	for index, arg := range split_ver[args_start:] {
		if strings.HasPrefix(arg, ":") {
			multi_word_index = index
			break
		}

		command.Args = append(command.Args, arg)
	}

	if multi_word_index != -1 {
		words := []string{}
		words = append(words, split_ver[args_start:][multi_word_index][1:len(split_ver[args_start:][multi_word_index])])
		words = append(words, split_ver[args_start:][multi_word_index+1:]...)
		command.Args = append(command.Args, strings.Join(words, " "))
	}

	command.Args[len(command.Args)-1] = strings.TrimSuffix(command.Args[len(command.Args)-1], "\r\n")

	return &command, nil
}

func MakeIRCCommand(cmdtype string, args ...string) *IRCCommand {
	var command IRCCommand

	command.Type = cmdtype
	command.Args = args

	return &command
}

func (command *IRCCommand) ToRaw() (string, error) {
	out := []string{}
	if command.Source != "" {
		out = append(out, command.Source)
	}
	out = append(out, command.Type)
	for _, arg := range command.Args[0 : len(command.Args)-1] {
		if strings.Contains(arg, " ") {
			return "", errors.New("nonfinal argument contains space")
		}
		out = append(out, arg)
	}

	if strings.Contains(command.Args[len(command.Args)-1], " ") {
		out = append(out, fmt.Sprint(":", command.Args[len(command.Args)-1]))
	} else {
		out = append(out, command.Args[len(command.Args)-1])
	}

	return fmt.Sprintf("%s\r\n", strings.Join(out, " ")), nil
}

func (connection *IRCConnection) Connect() error {
	log.Print("Connecting...")
	conn, err := net.Dial("tcp", connection.Location)
	connection.Connection = conn

	if err != nil {
		return err
	}

	log.Print("Connected!")

	go func() {
		//bio := bufio.NewReader(conn)
		for {
			line, err := bufio.NewReader(conn).ReadString('\n')
			log.Printf("Got line: %s\n", line)
			if err != nil {
				connection.Finished <- true
				log.Fatal(err)
			}
			command, err := RawToIRCCommand(line)
			if err != nil {
				log.Print(err)
			}
			for _, fn := range connection.Listeners {
				go fn(command)
			}
		}
	}()

	/* create a goroutine to send PONGs back when we receive PINGs */
	connection.AddListener(func(command *IRCCommand) {
		if command.Type == "PING" {
			fmt.Fprintf(conn, "PONG %s\r\n", command.Args[0])
		}
	})

	err = connection.SendCommand(MakeIRCCommand("NICK", "garchive"))
	if err != nil {
		return err
	}
	/* query the local system for a username. This isn't *really* necessary,
	 * but it really isn't that big of a deal to do it aywa */
	user, err := user.Current()
	if err != nil {
		log.Print(err)
		user.Username = "unknown"
	}
	err = connection.SendCommand(MakeIRCCommand("USER", user.Username, "0", "*", "Garchive: An IRC archiver bot"))
	if err != nil {
		return err
	}

	return nil
}

func (connection *IRCConnection) AddListener(fn IRCListener) {
	connection.Listeners = append(connection.Listeners, fn)
}

func (connection *IRCConnection) SendCommand(command *IRCCommand) error {
	log.Printf("Sending command %v\n", command)
	raw_form, err := command.ToRaw()
	log.Printf("raw form: %v\n", raw_form)
	if err != nil {
		return err
	}

	fmt.Fprint(connection.Connection, raw_form)

	return nil
}

func MakeChannelListener(channel string, filename string) (func(*IRCCommand), error) {
	file, err := os.OpenFile(filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
	if err != nil {
		return nil, err
	}

	fn := func(command *IRCCommand) {
		line := ""
		switch command.Type {
		case "PRIVMSG":
			if command.Args[0] == channel {
				line = fmt.Sprintf("%s: %s\n", command.Source, command.Args[1])
			}
		case "JOIN":
			if command.Args[0] == channel {
				line = fmt.Sprintf("%s has joined %s\n", command.Source, channel)
			}
		case "PART":
			if command.Args[0] == channel {
				line = fmt.Sprintf("%s has left %s\n", command.Source, channel)
			}
		case "TOPIC":
			if command.Args[0] == channel {
				line = fmt.Sprintf("%s has set the topic to %s\n", command.Source, command.Args[1])
			}
		}

		if line != "" {
			fmt.Fprintf(file, "[%v] %s", time.Now(), line)
		}
	}

	return fn, nil
}

func Main() {
	if len(os.Args) < 4 {
		fmt.Printf("Usage: %s irc_uri nick channel [other_channels...]\n", os.Args[0])
		os.Exit(1)
	}
	uri := os.Args[1]
	nick := os.Args[2]
	channels := os.Args[3:len(os.Args)]
	connection := MakeConnection(uri, nick)

	connection.AddListener(func(command *IRCCommand) {
		fmt.Printf("%v\n", command)
		fmt.Printf("Args:\n")
		for index, arg := range command.Args {
			fmt.Printf("%d.  %s\n", index, arg)
		}
	})

	for _, c := range channels {
		fmt.Printf("Adding channel %v\n", c)
	}

	err := connection.Connect()
	if err != nil {
		log.Fatal(err)
	}

	for _, c := range channels {
		/* allow the user to specify the channel without the #
		 * since bash requries it to be escaped */
		if !strings.HasPrefix(c, "#") {
			c = fmt.Sprintf("#%s", c)
		}
		err = connection.SendCommand(MakeIRCCommand("JOIN", c))
		if err != nil {
			log.Print(err)
		}
		channelListener, err := MakeChannelListener(c, strings.TrimPrefix(c, "#"))
		if err != nil {
			log.Print(err)
		}
		connection.AddListener(channelListener)
	}

	<-connection.Finished
}
<|endoftext|>


<commit_before>package gcfilter

import (
	"bufio"
	"fmt"
	"os"

	syscall "golang.org/x/sys/unix"
)

var current *os.File

type Wall struct {
	SweepTerm, Scan, Sync, Mark, MarkTerm float64
}

// Filter ..
func Filter() error {

	fmt.Println("filtering GC statements")

	current = os.Stderr
	r, w, err := os.Pipe()
	if err != nil {
		return err
	}

	rr, ww, err := os.Pipe()
	if err != nil {
		return err
	}

	// dup the Stderr File descriptor
	syscall.Dup2(int(w.Fd()), int(os.Stderr.Fd()))

	// move os.Stderr
	os.Stderr = ww

	// Listen on os.Stderr for messages and pipe them to the new files below
	go func(fwd *os.File, rr *os.File) {
		fmt.Println("running the goroutine")
		scanner := bufio.NewScanner(rr)
		for scanner.Scan() {
			line := scanner.Bytes()
			if len(line) != 0 {
				w.Write(line)
			}
		}
	}(w, rr)

	go func() {
		fmt.Println("running the goroutine")
		scanner := bufio.NewScanner(r)
		for scanner.Scan() {
			line := scanner.Text()

			var time, perc, wall, cpu string
			var number int

			if line[0:2] == "gc" {
				w := &Wall{}
				// gc 1 @0.248s 0%: 0.14+0.43+0.33+0.54+0.43 ms clock, 0.29+0.43+0+0.13/0.44/1.4+0.86 ms cpu, 4->4->1 MB, 4 MB goal, 4 P
				fmt.Sscanf(line, "gc %d %s %s %s ms clock, %s ms cpu,", &number, &time, &perc, &wall, &cpu)
				fmt.Sscanf(wall, "%f+%f+%f+%f+%f", &w.SweepTerm, &w.Scan, &w.Sync, &w.Mark, &w.MarkTerm)
				fmt.Println(w)
			} else {
				fmt.Println(line)
			}

		}
	}()

	return nil
}

// Reset returns Stderr to the original location
func Reset() {
	os.Stderr = current
}
<commit_msg>add some new elements<commit_after>package gcfilter

import (
	"bufio"
	"fmt"
	"os"
	"runtime"

	syscall "golang.org/x/sys/unix"
)

var current *os.File

// GCRun holds data about a GC run
type GCRun struct {
	N     int     // the GC number, incremented at each GC
	Time  float32 //time in seconds since program start
	Perc  int     // percentage of time spent in GC since program start
	Clock Timing
	CPU   Timing
	Heap  Heap
	Procs int
}

// Timing is the base struct for clock or cpu timing information
type Timing struct {
	SweepTerm, Scan, Sync, Mark, MarkTerm float64
}

// Heap tracks the size of the heap during a GC run
type Heap struct {
	start int
	end   int
	live  int
	goal  int
}

// Filter ..
func Filter() error {

	if GCOn() == false {
		return fmt.Errorf("GC Debug flag not set")
	}

	fmt.Println("filtering GC statements")

	current = os.Stderr
	r, w, err := os.Pipe()
	if err != nil {
		return err
	}

	rr, ww, err := os.Pipe()
	if err != nil {
		return err
	}

	// dup the Stderr File descriptor
	syscall.Dup2(int(w.Fd()), int(os.Stderr.Fd()))

	// move os.Stderr
	os.Stderr = ww

	// Listen on os.Stderr for messages and pipe them to the new files below
	go func(fwd *os.File, rr *os.File) {
		fmt.Println("running the goroutine")
		scanner := bufio.NewScanner(rr)
		for scanner.Scan() {
			line := scanner.Bytes()
			if len(line) != 0 {
				w.Write(line)
			}
		}
	}(w, rr)

	go func() {
		fmt.Println("running the goroutine")
		scanner := bufio.NewScanner(r)
		for scanner.Scan() {
			line := scanner.Text()

			var time, perc, wall, cpu string
			var number int

			if line[0:2] == "gc" {
				run := &GCRun{}
				// gc 1 @0.248s 0%: 0.14+0.43+0.33+0.54+0.43 ms clock, 0.29+0.43+0+0.13/0.44/1.4+0.86 ms cpu, 4->4->1 MB, 4 MB goal, 4 P
				fmt.Sscanf(line, "gc %d %s %s %s ms clock, %s ms cpu,", &number, &time, &perc, &wall, &cpu)
				fmt.Sscanf(wall, "%f+%f+%f+%f+%f", &w.SweepTerm, &w.Scan, &w.Sync, &w.Mark, &w.MarkTerm)
				fmt.Println(w)
			} else {
				fmt.Println(line)
			}

		}
	}()

	return nil
}

func GCOn() bool {
	var ms *runtime.MemStats
	runtime.ReadMemStats(ms)

	return ms.DebugGC
}

// Reset returns Stderr to the original location
func Reset() {
	os.Stderr = current
}
<|endoftext|>


<commit_before>/*
Copyright 2016 Tamás Gulácsi

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package oracall

import (
	"fmt"
	"hash/fnv"
	"io"
	"strings"
)

type PlsType struct {
	ora string
}

// NewArg returns a new argument to ease arument conversions.
func NewPlsType(ora string) PlsType {
	return PlsType{ora: ora}
}

// FromOra retrieves the value of the argument with arg type, from src variable to dst variable.
func (arg PlsType) FromOra(dst, src, varName string) string {
	if Gogo {
		if varName != "" {
			switch arg.ora {
			case "DATE":
				return fmt.Sprintf("%s.Set(%s)", dst, varName)
			}
		}
	}
	switch arg.ora {
	case "DATE":
		return fmt.Sprintf("%s = string(%s)", dst, src)
	case "PLS_INTEGER":
		return fmt.Sprintf("%s = %s.Value", dst, src)
	case "NUMBER":
		return fmt.Sprintf("%s = %s.Value", dst, src)
	}
	return fmt.Sprintf("%s = %s // %s", dst, src, arg.ora)
}

func (arg PlsType) GetOra(src, varName string) string {
	if Gogo {
		switch arg.ora {
		case "DATE":
			if varName != "" {
				return fmt.Sprintf("custom.NewDate(%s)", varName)
			}
			return fmt.Sprintf("custom.AsDate(%s)", src)
		}
	}
	return src
}

// ToOra adds the value of the argument with arg type, from src variable to dst variable.
func (arg PlsType) ToOra(dst, src string) (expr string, variable string) {
	dstVar := mkVarName(dst)
	if Gogo {
		switch arg.ora {
		case "DATE": // custom.Date
			var pointer string
			if src[0] == '&' {
				pointer = "&"
			}
			Log("dstVar", dstVar)
			return fmt.Sprintf(`%s := %s.Get() // toOra D
			%s = %s%s`,
					dstVar, strings.TrimPrefix(src, "&"),
					dst, pointer, dstVar,
				),
				dstVar
		}
	}
	switch arg.ora {
	case "PLS_INTEGER":
		if src[0] != '&' {
			return fmt.Sprintf("%s := ora.Int32{IsNull:true}; if %s != 0 { %s.Value, %s.IsNull = %s, false }; %s = %s", dstVar, src, dstVar, dstVar, src, dst, dstVar), dstVar
		}
	case "NUMBER":
		if src[0] != '&' {
			return fmt.Sprintf("%s := ora.Float64{IsNull:true}; if %s != 0 { %s.Value, %s.IsNull = %s, false }; %s = %s", dstVar, src, dstVar, dstVar, src, dst, dstVar), dstVar
		}
	}
	return fmt.Sprintf("%s = %s // %s", dst, src, arg.ora), ""
}

func mkVarName(dst string) string {
	h := fnv.New64()
	io.WriteString(h, dst)
	return fmt.Sprintf("var_%d", h.Sum64())
}
<commit_msg>temp var names use hex encoding<commit_after>/*
Copyright 2016 Tamás Gulácsi

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package oracall

import (
	"encoding/hex"
	"fmt"
	"hash/fnv"
	"io"
	"strings"
)

type PlsType struct {
	ora string
}

// NewArg returns a new argument to ease arument conversions.
func NewPlsType(ora string) PlsType {
	return PlsType{ora: ora}
}

// FromOra retrieves the value of the argument with arg type, from src variable to dst variable.
func (arg PlsType) FromOra(dst, src, varName string) string {
	if Gogo {
		if varName != "" {
			switch arg.ora {
			case "DATE":
				return fmt.Sprintf("%s.Set(%s)", dst, varName)
			}
		}
	}
	switch arg.ora {
	case "DATE":
		return fmt.Sprintf("%s = string(%s)", dst, src)
	case "PLS_INTEGER":
		return fmt.Sprintf("%s = %s.Value", dst, src)
	case "NUMBER":
		return fmt.Sprintf("%s = %s.Value", dst, src)
	}
	return fmt.Sprintf("%s = %s // %s", dst, src, arg.ora)
}

func (arg PlsType) GetOra(src, varName string) string {
	if Gogo {
		switch arg.ora {
		case "DATE":
			if varName != "" {
				return fmt.Sprintf("custom.NewDate(%s)", varName)
			}
			return fmt.Sprintf("custom.AsDate(%s)", src)
		}
	}
	return src
}

// ToOra adds the value of the argument with arg type, from src variable to dst variable.
func (arg PlsType) ToOra(dst, src string) (expr string, variable string) {
	dstVar := mkVarName(dst)
	if Gogo {
		switch arg.ora {
		case "DATE": // custom.Date
			var pointer string
			if src[0] == '&' {
				pointer = "&"
			}
			return fmt.Sprintf(`%s := %s.Get() // toOra D
			%s = %s%s`,
					dstVar, strings.TrimPrefix(src, "&"),
					dst, pointer, dstVar,
				),
				dstVar
		}
	}
	switch arg.ora {
	case "PLS_INTEGER":
		if src[0] != '&' {
			return fmt.Sprintf("%s := ora.Int32{IsNull:true}; if %s != 0 { %s.Value, %s.IsNull = %s, false }; %s = %s", dstVar, src, dstVar, dstVar, src, dst, dstVar), dstVar
		}
	case "NUMBER":
		if src[0] != '&' {
			return fmt.Sprintf("%s := ora.Float64{IsNull:true}; if %s != 0 { %s.Value, %s.IsNull = %s, false }; %s = %s", dstVar, src, dstVar, dstVar, src, dst, dstVar), dstVar
		}
	}
	return fmt.Sprintf("%s = %s // %s", dst, src, arg.ora), ""
}

func mkVarName(dst string) string {
	h := fnv.New64()
	io.WriteString(h, dst)
	var raw [8]byte
	var enc [8 * 2]byte
	hex.Encode(enc[:], h.Sum(raw[:0]))
	return fmt.Sprintf("var_%s", enc[:])
}
<|endoftext|>


<commit_before>package i18n

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	"github.com/cloudfoundry/cli/cf/i18n/detection"
	resources "github.com/cloudfoundry/cli/cf/resources"
	go_i18n "github.com/nicksnyder/go-i18n/i18n"
)

const (
	DEFAULT_LOCALE   = "en_US"
	DEFAULT_LANGUAGE = "en"
)

var T go_i18n.TranslateFunc

var SUPPORTED_LOCALES = map[string]string{
	"de": "de_DE",
	"en": "en_US",
	"es": "es_ES",
	"fr": "fr_FR",
	"it": "it_IT",
	"ja": "ja_JA",
	//"ko": "ko_KO", - Will add support for Korean when nicksnyder/go-i18n supports Korean
	"pt": "pt_BR",
	//"ru": "ru_RU", - Will add support for Russian when nicksnyder/go-i18n supports Russian
	"zh": "zh_Hans",
}
var Resources_path = filepath.Join("cf", "i18n", "resources")

func GetResourcesPath() string {
	return Resources_path
}

func Init(config core_config.ReadWriter, detector detection.Detector) go_i18n.TranslateFunc {
	var T go_i18n.TranslateFunc
	var err error

	locale := config.Locale()
	if locale != "" {
		err = loadFromAsset(locale)
		if err == nil {
			T, err = go_i18n.Tfunc(config.Locale(), DEFAULT_LOCALE)
		}
	} else {
		var userLocale string
		userLocale, err = initWithUserLocale(detector)
		if err != nil {
			userLocale = mustLoadDefaultLocale()
		}

		T, err = go_i18n.Tfunc(userLocale, DEFAULT_LOCALE)
	}

	if err != nil {
		panic(err)
	}

	return T
}

func initWithUserLocale(detector detection.Detector) (string, error) {
	userLocale, err := detector.DetectIETF()
	if err != nil {
		userLocale = DEFAULT_LOCALE
	}

	language, err := detector.DetectLanguage()
	if err != nil {
		language = DEFAULT_LANGUAGE
	}

	userLocale = strings.Replace(userLocale, "-", "_", 1)
	if strings.HasPrefix(userLocale, "zh_TW") || strings.HasPrefix(userLocale, "zh_HK") {
		userLocale = "zh_Hant"
		language = "zh"
	}

	err = loadFromAsset(userLocale)
	if err != nil {
		locale := SUPPORTED_LOCALES[language]
		if locale == "" {
			userLocale = DEFAULT_LOCALE
		} else {
			userLocale = locale
		}
		err = loadFromAsset(userLocale)
	}

	return userLocale, err
}

func mustLoadDefaultLocale() string {
	userLocale := DEFAULT_LOCALE

	err := loadFromAsset(DEFAULT_LOCALE)
	if err != nil {
		panic("Could not load en_US language files. God save the queen. " + err.Error())
	}

	return userLocale
}

func loadFromAsset(locale string) error {
	assetName := locale + ".all.json"
	assetKey := filepath.Join(GetResourcesPath(), assetName)

	byteArray, err := resources.Asset(assetKey)
	if err != nil {
		return err
	}

	if len(byteArray) == 0 {
		return errors.New(fmt.Sprintf("Could not load i18n asset: %v", assetKey))
	}

	tmpDir, err := ioutil.TempDir("", "cloudfoundry_cli_i18n_res")
	if err != nil {
		return err
	}
	defer func() {
		os.RemoveAll(tmpDir)
	}()

	fileName, err := saveLanguageFileToDisk(tmpDir, assetName, byteArray)
	if err != nil {
		return err
	}

	go_i18n.MustLoadTranslationFile(fileName)

	os.RemoveAll(fileName)

	return nil
}

func saveLanguageFileToDisk(tmpDir, assetName string, byteArray []byte) (fileName string, err error) {
	fileName = filepath.Join(tmpDir, assetName)
	file, err := os.Create(fileName)
	if err != nil {
		return
	}
	defer file.Close()

	_, err = file.Write(byteArray)
	if err != nil {
		return
	}

	return
}
<commit_msg>better error message when tmp dir does not exist while not load language files<commit_after>package i18n

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	"github.com/cloudfoundry/cli/cf/i18n/detection"
	resources "github.com/cloudfoundry/cli/cf/resources"
	go_i18n "github.com/nicksnyder/go-i18n/i18n"
)

const (
	DEFAULT_LOCALE   = "en_US"
	DEFAULT_LANGUAGE = "en"
)

var T go_i18n.TranslateFunc

var SUPPORTED_LOCALES = map[string]string{
	"de": "de_DE",
	"en": "en_US",
	"es": "es_ES",
	"fr": "fr_FR",
	"it": "it_IT",
	"ja": "ja_JA",
	//"ko": "ko_KO", - Will add support for Korean when nicksnyder/go-i18n supports Korean
	"pt": "pt_BR",
	//"ru": "ru_RU", - Will add support for Russian when nicksnyder/go-i18n supports Russian
	"zh": "zh_Hans",
}
var Resources_path = filepath.Join("cf", "i18n", "resources")

func GetResourcesPath() string {
	return Resources_path
}

func Init(config core_config.ReadWriter, detector detection.Detector) go_i18n.TranslateFunc {
	var T go_i18n.TranslateFunc
	var err error

	locale := config.Locale()
	if locale != "" {
		err = loadFromAsset(locale)
		if err == nil {
			T, err = go_i18n.Tfunc(config.Locale(), DEFAULT_LOCALE)
		}
	} else {
		var userLocale string
		userLocale, err = initWithUserLocale(detector)
		if err != nil {
			userLocale = mustLoadDefaultLocale()
		}

		T, err = go_i18n.Tfunc(userLocale, DEFAULT_LOCALE)
	}

	if err != nil {
		panic(err)
	}

	return T
}

func initWithUserLocale(detector detection.Detector) (string, error) {
	userLocale, err := detector.DetectIETF()
	if err != nil {
		userLocale = DEFAULT_LOCALE
	}

	language, err := detector.DetectLanguage()
	if err != nil {
		language = DEFAULT_LANGUAGE
	}

	userLocale = strings.Replace(userLocale, "-", "_", 1)
	if strings.HasPrefix(userLocale, "zh_TW") || strings.HasPrefix(userLocale, "zh_HK") {
		userLocale = "zh_Hant"
		language = "zh"
	}

	err = loadFromAsset(userLocale)
	if err != nil {
		locale := SUPPORTED_LOCALES[language]
		if locale == "" {
			userLocale = DEFAULT_LOCALE
		} else {
			userLocale = locale
		}
		err = loadFromAsset(userLocale)
	}

	return userLocale, err
}

func mustLoadDefaultLocale() string {
	userLocale := DEFAULT_LOCALE

	err := loadFromAsset(DEFAULT_LOCALE)
	if err != nil {
		panic("Could not load en_US language files. God save the queen. \n" + err.Error() + "\n\n")
	}

	return userLocale
}

func loadFromAsset(locale string) error {
	assetName := locale + ".all.json"
	assetKey := filepath.Join(GetResourcesPath(), assetName)

	byteArray, err := resources.Asset(assetKey)
	if err != nil {
		return err
	}

	if len(byteArray) == 0 {
		return errors.New(fmt.Sprintf("Could not load i18n asset: %v", assetKey))
	}

	_, err = os.Stat(os.TempDir())
	if err != nil {
		if !os.IsExist(err) {
			return errors.New("Please make sure Temp dir exist - " + os.TempDir())
		} else {
			return err
		}
	}

	tmpDir, err := ioutil.TempDir("", "cloudfoundry_cli_i18n_res")
	if err != nil {
		return err
	}
	defer func() {
		os.RemoveAll(tmpDir)
	}()

	fileName, err := saveLanguageFileToDisk(tmpDir, assetName, byteArray)
	if err != nil {
		return err
	}

	go_i18n.MustLoadTranslationFile(fileName)

	os.RemoveAll(fileName)

	return nil
}

func saveLanguageFileToDisk(tmpDir, assetName string, byteArray []byte) (fileName string, err error) {
	fileName = filepath.Join(tmpDir, assetName)
	file, err := os.Create(fileName)
	if err != nil {
		return
	}
	defer file.Close()

	_, err = file.Write(byteArray)
	if err != nil {
		return
	}

	return
}
<|endoftext|>