<commit_before>package lytics

import (
	"fmt"
	"net/url"
	"strings"
	"time"
)

const (
	segmentEndpoint               = "segment/:id"
	segmentListEndpoint           = "segment"
	segmentSizeEndpoint           = "segment/:id/sizes"
	segmentSizesEndpoint          = "segment/sizes"       // ids
	segmentAttributionEndpoint    = "segment/attribution" // ids
	segmentScanEndpoint           = "segment/:id/scan"
	segmentCollectionListEndpoint = "segmentcollection"
	segmentCollectionEndpoint     = "segmentcollection/:id"
)

type Segment struct {
	Id          string    `json:"id"`
	Aid         int       `json:"aid"` // Deprecated; use AccountId
	AccountId   string    `json:"account_id"`
	ShortId     string    `json:"short_id,omitempty"`
	Name        string    `json:"name"`
	IsPublic    bool      `json:"is_public"`
	PublicName  string    `json:"public_name,omitempty"`
	SlugName    string    `json:"slug_name"`
	Description string    `json:"description,omitempty"`
	Table       string    `json:"table,omitempty"`
	AuthorId    string    `json:"author_id"`
	Updated     time.Time `json:"updated" bson:"updated"`
	Created     time.Time `json:"created" bson:"created"`
	SegType     string    `json:"op"`
	Negate      bool      `json:"negate"`
	Tags        []string  `json:"tags"`
}

type SegmentSize struct {
	Id       string  `json:"id"`
	Name     string  `json:"name"`
	SlugName string  `json:"slug_name"`
	Size     float64 `json:"size"`
}

type SegmentAttribution struct {
	Id      string                      `json:"id"`
	Metrics []SegmentAttributionMetrics `json:"metrics"`
}

type SegmentAttributionMetrics struct {
	Value   int64   `json:"value"`
	Ts      string  `json:"ts"`
	Anomaly float64 `json:"anomaly"`
}

type SegmentCollection struct {
	Aid           int               `json:"aid"`
	AccountId     string            `json:"account_id"`
	Id            string            `json:"id"`
	Name          string            `json:"name"`
	Slug          string            `json:"slug_name"`
	Description   string            `json:"description,omitempty"`
	Table         string            `json:"table,omitempty"`
	AuthorId      string            `json:"author_id"`
	Updated       time.Time         `json:"updated"`
	Created       time.Time         `json:"created"`
	Internal      bool              `json:"internal"`
	Collection    []*SegColRelation `json:"collection""`
	ParentSegment string            `json:"parent_segment"`
}

type SegColRelation struct {
	Id    string `json:"id"`
	Order int    `json:"order"`
}

type SegmentScanner struct {
	SegmentID string
	Next      string
	Previous  string
	Loader    chan []Entity
	Shutdown  chan bool
	Total     int
	Batches   []int
}

// Created is a helper method to convert the timestamp into human readable format for metrics
func (s *SegmentAttributionMetrics) Created() (time.Time, error) {
	return parseLyticsTime(s.Ts)
}

// GetSegment returns the details for a single segment based on id
// https://www.getlytics.com/developers/rest-api#segment
func (l *Client) GetSegment(id string) (Segment, error) {
	res := ApiResp{}
	data := Segment{}

	// make the request
	err := l.Get(parseLyticsURL(segmentEndpoint, map[string]string{"id": id}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegments returns a list of all segments for an account
// https://www.getlytics.com/developers/rest-api#segment-list
func (l *Client) GetSegments() ([]Segment, error) {
	res := ApiResp{}
	data := []Segment{}

	// make the request
	err := l.Get(segmentListEndpoint, nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentSize returns the segment size information for a single segment
// https://www.getlytics.com/developers/rest-api#segment-sizes
func (l *Client) GetSegmentSize(id string) (SegmentSize, error) {
	res := ApiResp{}
	data := SegmentSize{}

	// make the request
	err := l.Get(parseLyticsURL(segmentSizeEndpoint, map[string]string{"id": id}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentSizes returns the segment sizes for all segments on an account
// https://www.getlytics.com/developers/rest-api#segment-sizes
func (l *Client) GetSegmentSizes(segments []string) ([]SegmentSize, error) {
	params := url.Values{}
	res := ApiResp{}
	data := []SegmentSize{}

	// if we have specific segments to filter by add those to the params as comma separated string
	if len(segments) > 0 {
		params.Add("ids", strings.Join(segments, ","))
	}

	// make the request
	err := l.Get(segmentSizesEndpoint, params, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentAttribution returns the attribution (change over time) for segments
// method accepts a string slice of 1 or more segments to query.
// NOT CURRENTLY DOCUMENTED
func (l *Client) GetSegmentAttribution(segments []string) ([]SegmentAttribution, error) {
	params := url.Values{}

	res := ApiResp{}
	data := []SegmentAttribution{}

	// if the request is for a specific set of segments add that as comma separated param
	if len(segments) > 0 {
		params.Add("ids", strings.Join(segments, ","))
	}

	// make the request
	err := l.Get(parseLyticsURL(segmentAttributionEndpoint, nil), params, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentCollection returns a single collection of segments
// (a grouped/named lists of segments)
func (l *Client) GetSegmentCollection(id string) (SegmentCollection, error) {
	res := ApiResp{}
	data := SegmentCollection{}

	err := l.Get(parseLyticsURL(segmentCollectionEndpoint, map[string]string{"id": id}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentCollectionList returns a list of all segment
// collections for an account
func (l *Client) GetSegmentCollectionList() ([]SegmentCollection, error) {
	res := ApiResp{}
	data := []SegmentCollection{}

	err := l.Get(parseLyticsURL(segmentCollectionListEndpoint, nil), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// Other Available Endpoints
// * POST    create segment
// * PUT     update segment
// * DELETE  remove segment

// **************************** START OF SEGMENT SCAN METHODS ****************************

// GetSegmentEntities returns a single page of entities (20max) for the given segment
// also returns the next value if there are more than 20 entities in the segment
// https://www.getlytics.com/developers/rest-api#segment-scan
func (l *Client) GetSegmentEntities(segment, next string) (interface{}, string, []Entity, error) {
	res := ApiResp{}
	data := []Entity{}
	params := url.Values{}

	params.Add("start", next)

	// make the request
	err := l.Get(parseLyticsURL(segmentScanEndpoint, map[string]string{"id": segment}), params, nil, &res, &data)
	if err != nil {
		return "", "", data, err
	}

	return res.Status, res.Next, data, nil
}

// CreateScanner generates a segment scanner so that we can process entities as they are loaded
// reduces the load as some segments can have hundreds of thousands of users and it is impractical
// to return a single result
func (l *Client) CreateScanner() error {
	scanner := SegmentScanner{}

	// create loader and shutdown
	loader := make(chan []Entity)
	shutdown := make(chan bool)

	// save to scanner
	scanner.Loader = loader
	scanner.Shutdown = shutdown

	// add the scanner to the client
	l.Scan = &scanner

	return nil
}

// LoadEntity does the heavy lifting when it comes to paging. It loops through all available pages
// and emits the batch of entities to be processed along the way. Also maintains a slice of batch
// counts and total count to help with debugging and reporting.
func (l *Client) LoadEntity() {
	var (
		entities []Entity
		err      error
		fails    int
		maxTries int
	)

	maxTries = 10

	// make calls for next batch of segments until we run out of next params
	for {
		_, l.Scan.Next, entities, err = l.GetSegmentEntities(l.Scan.SegmentID, l.Scan.Next)
		if err != nil {
			fails++

			if fails > maxTries {
				panic(fmt.Sprintf("Failed to get entities, exceeded max tries(%d): %v", maxTries, err))
			}

			// if we fail and have not exceeded the limit, try again
			continue
		}

		// for logging add the batch details to the scanner
		l.Scan.Batches = append(l.Scan.Batches, len(entities))

		// for logging add the total entites returned to the scanner
		l.Scan.Total = l.Scan.Total + len(entities)

		// emit the entities to the loader for processing
		l.Scan.Loader <- entities

		// if there are no more pages we will have a blank next, just break and return
		if l.Scan.Next == "" {
			break
		}
	}

	// everything worked, shutdown safely
	l.Scan.Shutdown <- true
}

// PageMembers sets the segment on the master scanner and initiates the main go routine
// for paging
func (l *Client) PageMembers(segment string) error {
	// save the target segment on the scanner
	l.Scan.SegmentID = segment

	// fire up the go routine for paging entities
	go l.LoadEntity()
	return nil
}
<commit_msg>Fix merge conflicts.<commit_after>package lytics

import (
	"fmt"
	"net/url"
	"strconv"
	"strings"
	"time"
)

const (
	segmentEndpoint               = "segment/:id"
	segmentListEndpoint           = "segment"
	segmentSizeEndpoint           = "segment/:id/sizes"
	segmentSizesEndpoint          = "segment/sizes"       // ids
	segmentAttributionEndpoint    = "segment/attribution" // ids
	segmentScanEndpoint           = "segment/:id/scan"
	segmentCollectionListEndpoint = "segmentcollection"
	segmentCollectionEndpoint     = "segmentcollection/:id"
)

type Segment struct {
	Id          string    `json:"id"`
	Aid         int       `json:"aid"` // Deprecated; use AccountId
	AccountId   string    `json:"account_id"`
	ShortId     string    `json:"short_id,omitempty"`
	Name        string    `json:"name"`
	IsPublic    bool      `json:"is_public"`
	PublicName  string    `json:"public_name,omitempty"`
	SlugName    string    `json:"slug_name"`
	Description string    `json:"description,omitempty"`
	Table       string    `json:"table,omitempty"`
	AuthorId    string    `json:"author_id"`
	Updated     time.Time `json:"updated" bson:"updated"`
	Created     time.Time `json:"created" bson:"created"`
	SegType     string    `json:"op"`
	Negate      bool      `json:"negate"`
	Tags        []string  `json:"tags"`
}

type SegmentSize struct {
	Id       string  `json:"id"`
	Name     string  `json:"name"`
	SlugName string  `json:"slug_name"`
	Size     float64 `json:"size"`
}

type SegmentAttribution struct {
	Id      string                      `json:"id"`
	Metrics []SegmentAttributionMetrics `json:"metrics"`
}

type SegmentAttributionMetrics struct {
	Value   int64   `json:"value"`
	Ts      string  `json:"ts"`
	Anomaly float64 `json:"anomaly"`
}

type SegmentCollection struct {
	Aid           int               `json:"aid"`
	AccountId     string            `json:"account_id"`
	Id            string            `json:"id"`
	Name          string            `json:"name"`
	Slug          string            `json:"slug_name"`
	Description   string            `json:"description,omitempty"`
	Table         string            `json:"table,omitempty"`
	AuthorId      string            `json:"author_id"`
	Updated       time.Time         `json:"updated"`
	Created       time.Time         `json:"created"`
	Internal      bool              `json:"internal"`
	Collection    []*SegColRelation `json:"collection""`
	ParentSegment string            `json:"parent_segment"`
}

type SegColRelation struct {
	Id    string `json:"id"`
	Order int    `json:"order"`
}

type SegmentScanner struct {
	SegmentID string
	Next      string
	Previous  string
	Loader    chan []Entity
	Shutdown  chan bool
	Total     int
	Batches   []int
}

// Created is a helper method to convert the timestamp into human readable format for metrics
func (s *SegmentAttributionMetrics) Created() (time.Time, error) {
	return parseLyticsTime(s.Ts)
}

// GetSegment returns the details for a single segment based on id
// https://www.getlytics.com/developers/rest-api#segment
func (l *Client) GetSegment(id string) (Segment, error) {
	res := ApiResp{}
	data := Segment{}

	// make the request
	err := l.Get(parseLyticsURL(segmentEndpoint, map[string]string{"id": id}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegments returns a list of all segments for an account
// https://www.getlytics.com/developers/rest-api#segment-list
func (l *Client) GetSegments() ([]Segment, error) {
	res := ApiResp{}
	data := []Segment{}

	// make the request
	err := l.Get(segmentListEndpoint, nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentSize returns the segment size information for a single segment
// https://www.getlytics.com/developers/rest-api#segment-sizes
func (l *Client) GetSegmentSize(id string) (SegmentSize, error) {
	res := ApiResp{}
	data := SegmentSize{}

	// make the request
	err := l.Get(parseLyticsURL(segmentSizeEndpoint, map[string]string{"id": id}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentSizes returns the segment sizes for all segments on an account
// https://www.getlytics.com/developers/rest-api#segment-sizes
func (l *Client) GetSegmentSizes(segments []string) ([]SegmentSize, error) {
	params := url.Values{}
	res := ApiResp{}
	data := []SegmentSize{}

	// if we have specific segments to filter by add those to the params as comma separated string
	if len(segments) > 0 {
		params.Add("ids", strings.Join(segments, ","))
	}

	// make the request
	err := l.Get(segmentSizesEndpoint, params, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentAttribution returns the attribution (change over time) for segments
// method accepts a string slice of 1 or more segments to query.
// NOT CURRENTLY DOCUMENTED
func (l *Client) GetSegmentAttribution(segments []string) ([]SegmentAttribution, error) {
	params := url.Values{}

	res := ApiResp{}
	data := []SegmentAttribution{}

	// if the request is for a specific set of segments add that as comma separated param
	if len(segments) > 0 {
		params.Add("ids", strings.Join(segments, ","))
	}

	// make the request
	err := l.Get(parseLyticsURL(segmentAttributionEndpoint, nil), params, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentCollection returns a single collection of segments
// (a grouped/named lists of segments)
func (l *Client) GetSegmentCollection(id string) (SegmentCollection, error) {
	res := ApiResp{}
	data := SegmentCollection{}

	err := l.Get(parseLyticsURL(segmentCollectionEndpoint, map[string]string{"id": id}), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// GetSegmentCollectionList returns a list of all segment
// collections for an account
func (l *Client) GetSegmentCollectionList() ([]SegmentCollection, error) {
	res := ApiResp{}
	data := []SegmentCollection{}

	err := l.Get(parseLyticsURL(segmentCollectionListEndpoint, nil), nil, nil, &res, &data)
	if err != nil {
		return data, err
	}

	return data, nil
}

// Other Available Endpoints
// * POST    create segment
// * PUT     update segment
// * DELETE  remove segment

// **************************** START OF SEGMENT SCAN METHODS ****************************

// GetSegmentEntities returns a single page of entities (20max) for the given segment
// also returns the next value if there are more than 20 entities in the segment
// https://www.getlytics.com/developers/rest-api#segment-scan
func (l *Client) GetSegmentEntities(segment, next string, limit int) (interface{}, string, []Entity, error) {
	res := ApiResp{}
	data := []Entity{}
	params := url.Values{}

	params.Add("start", next)
	params.Add("limit", strconv.Itoa(limit))

	// make the request
	err := l.Get(parseLyticsURL(segmentScanEndpoint, map[string]string{"id": segment}), params, nil, &res, &data)
	if err != nil {
		return "", "", data, err
	}

	return res.Status, res.Next, data, nil
}

// CreateScanner generates a segment scanner so that we can process entities as they are loaded
// reduces the load as some segments can have hundreds of thousands of users and it is impractical
// to return a single result
func (l *Client) CreateScanner() error {
	scanner := SegmentScanner{}

	// create loader and shutdown
	loader := make(chan []Entity)
	shutdown := make(chan bool)

	// save to scanner
	scanner.Loader = loader
	scanner.Shutdown = shutdown

	// add the scanner to the client
	l.Scan = &scanner

	return nil
}

// LoadEntity does the heavy lifting when it comes to paging. It loops through all available pages
// and emits the batch of entities to be processed along the way. Also maintains a slice of batch
// counts and total count to help with debugging and reporting.
func (l *Client) LoadEntity() {
	var (
		entities []Entity
		err      error
		fails    int
		maxTries int
	)

	maxTries = 10

	// make calls for next batch of segments until we run out of next params
	for {
		_, l.Scan.Next, entities, err = l.GetSegmentEntities(l.Scan.SegmentID, l.Scan.Next, 100)
		if err != nil {
			fails++

			if fails > maxTries {
				panic(fmt.Sprintf("Failed to get entities, exceeded max tries(%d): %v", maxTries, err))
			}

			// if we fail and have not exceeded the limit, try again
			continue
		}

		// for logging add the batch details to the scanner
		l.Scan.Batches = append(l.Scan.Batches, len(entities))

		// for logging add the total entites returned to the scanner
		l.Scan.Total = l.Scan.Total + len(entities)

		// emit the entities to the loader for processing
		l.Scan.Loader <- entities

		// if there are no more pages we will have a blank next, just break and return
		if l.Scan.Next == "" {
			break
		}
	}

	// everything worked, shutdown safely
	l.Scan.Shutdown <- true
}

// PageMembers sets the segment on the master scanner and initiates the main go routine
// for paging
func (l *Client) PageMembers(segment string) error {
	// save the target segment on the scanner
	l.Scan.SegmentID = segment

	// fire up the go routine for paging entities
	go l.LoadEntity()
	return nil
}
<|endoftext|>


<commit_before>package sel

import (
	"encoding/json"
	"os"
	"flag"
	"fmt"
	"link-select/types"
)

func SelectLink(arg *flag.Flag) {
	fmt.Fprintf(os.Stdout, "Selecting %s from json file\n", arg.Value)

	readFile, err := os.Open("files/read.json")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error while opening read.json")
		os.Exit(-1)
	}

	var article types.Article

	jsonParser := json.NewDecoder(readFile)
	if err = jsonParser.Decode(&article); err != nil {
		fmt.Fprintf(os.Stderr, "Error while parsing read.json")
		os.Exit(-1)
	}

	//for i, v := range articles {
	//fmt.Fprintf("title: %s, link: %s\n", article.Title, article.Link)
}<commit_msg>Implement read.json parsing and proper error messages<commit_after>package sel

import (
	"os"
	"flag"
	"fmt"
	"log"
	"encoding/json"
	"link-select/types"
)

func SelectLink(arg *flag.Flag) {
	fmt.Fprintf(os.Stdout, "Selecting %s from json file\n", arg.Value)

	readFile, err := os.Open("files/read.json")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error while opening read JSON file\n")
		os.Exit(-1)
	}

	var readList types.ReadList

	jsonParser := json.NewDecoder(readFile)
	if err = jsonParser.Decode(&readList); err != nil {
		fmt.Fprintf(os.Stderr, "Error while parsing read JSON file\n")
		log.Fatal(err)
		os.Exit(-1)
	}

	//for i, v := range articles {
	//fmt.Fprintf("title: %s, link: %s\n", article.Title, article.Link)
}<|endoftext|>


<commit_before>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"context"
	"fmt"

	envoy_corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
	xdsapi "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
	"github.com/spf13/cobra"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"istio.io/istio/istioctl/pkg/clioptions"
	"istio.io/istio/istioctl/pkg/multixds"
	"istio.io/istio/istioctl/pkg/util/handlers"
	"istio.io/istio/istioctl/pkg/writer/pilot"
)

const (
	// TypeDebug requests debug info from istio, a secured implementation for istio debug interface
	TypeDebug = "istio.io/debug"
)

func debugCommand() *cobra.Command {
	var opts clioptions.ControlPlaneOptions
	var centralOpts clioptions.CentralControlPlaneOptions

	debugCommand := &cobra.Command{
		Use:   "debug [<type>/]<name>[.<namespace>]",
		Short: "Retrieves the debug information of istio",
		Long: `
Retrieves the debug information from Istiod or Pods in the mesh using the service account from the pod if --cert-dir is empty.
By default it will use the default serviceAccount from (istio-system) namespace if the pod is not specified.
`,
		Example: `  # Retrieve sync status for all Envoys in a mesh
  istioctl x debug syncz

  # Retrieve sync diff for a single Envoy and Istiod
  istioctl x debug syncz istio-egressgateway-59585c5b9c-ndc59.istio-system

  # SECURITY OPTIONS

  # Retrieve syncz debug information directly from the control plane, using token security
  # (This is the usual way to get the debug information with an out-of-cluster control plane.)
  istioctl x debug syncz --xds-address istio.cloudprovider.example.com:15012

  # Retrieve syncz debug information via Kubernetes config, using token security
  # (This is the usual way to get the debug information with an in-cluster control plane.)
  istioctl x debug syncz

  # Retrieve syncz debug information directly from the control plane, using RSA certificate security
  # (Certificates must be obtained before this step.  The --cert-dir flag lets istioctl bypass the Kubernetes API server.)
  istioctl x debug syncz --xds-address istio.example.com:15012 --cert-dir ~/.istio-certs

  # Retrieve syncz information via XDS from specific control plane in multi-control plane in-cluster configuration
  # (Select a specific control plane in an in-cluster canary Istio configuration.)
  istioctl x debug syncz --xds-label istio.io/rev=default
`,
		RunE: func(c *cobra.Command, args []string) error {
			kubeClient, err := kubeClientWithRevision(kubeconfig, configContext, opts.Revision)
			if err != nil {
				return err
			}
			if len(args) == 0 {
				return CommandParseError{
					e: fmt.Errorf("debug type is required"),
				}
			}
			var xdsRequest xdsapi.DiscoveryRequest
			var namespace, serviceAccount string
			if len(args) > 1 {
				podName, ns, err := handlers.InferPodInfoFromTypedResource(args[1],
					handlers.HandleNamespace(namespace, defaultNamespace),
					kubeClient.UtilFactory())
				if err != nil {
					return err
				}
				pod, err := kubeClient.CoreV1().Pods(ns).Get(context.TODO(), podName, metav1.GetOptions{})
				if err != nil {
					return err
				}
				namespace = ns
				serviceAccount = pod.Spec.ServiceAccountName
				xdsRequest = xdsapi.DiscoveryRequest{
					ResourceNames: []string{fmt.Sprintf("%s?proxyID=%s.%s", args[0], podName, ns)},
					Node: &envoy_corev3.Node{
						Id: "debug~0.0.0.0~istioctl~cluster.local",
					},
					TypeUrl: TypeDebug,
				}
			} else {
				xdsRequest = xdsapi.DiscoveryRequest{
					ResourceNames: []string{args[0]},
					Node: &envoy_corev3.Node{
						Id: "debug~0.0.0.0~istioctl~cluster.local",
					},
					TypeUrl: TypeDebug,
				}
			}
			xdsResponses, err := multixds.AllRequestAndProcessXds(&xdsRequest, &centralOpts, istioNamespace,
				namespace, serviceAccount, kubeClient)
			if err != nil {
				return err
			}
			sw := pilot.XdsStatusWriter{Writer: c.OutOrStdout()}
			return sw.PrintAll(xdsResponses)
		},
	}

	opts.AttachControlPlaneFlags(debugCommand)
	centralOpts.AttachControlPlaneFlags(debugCommand)
	debugCommand.Long += "\n\n" + ExperimentalMsg
	return debugCommand
}
<commit_msg>support default in-cluster access for debug (#31606)<commit_after>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"context"
	"fmt"

	envoy_corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
	xdsapi "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
	"github.com/spf13/cobra"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/kubectl/pkg/polymorphichelpers"

	"istio.io/istio/istioctl/pkg/clioptions"
	"istio.io/istio/istioctl/pkg/multixds"
	"istio.io/istio/istioctl/pkg/util/handlers"
	"istio.io/istio/istioctl/pkg/writer/pilot"
)

const (
	// TypeDebug requests debug info from istio, a secured implementation for istio debug interface
	TypeDebug         = "istio.io/debug"
	istiodServiceName = "istiod"
	xdsPortName       = "https-dns"
)

func debugCommand() *cobra.Command {
	var opts clioptions.ControlPlaneOptions
	var centralOpts clioptions.CentralControlPlaneOptions

	debugCommand := &cobra.Command{
		Use:   "debug [<type>/]<name>[.<namespace>]",
		Short: "Retrieves the debug information of istio",
		Long: `
Retrieves the debug information from Istiod or Pods in the mesh using the service account from the pod if --cert-dir is empty.
By default it will use the default serviceAccount from (istio-system) namespace if the pod is not specified.
`,
		Example: `  # Retrieve sync status for all Envoys in a mesh
  istioctl x debug syncz

  # Retrieve sync diff for a single Envoy and Istiod
  istioctl x debug syncz istio-egressgateway-59585c5b9c-ndc59.istio-system

  # SECURITY OPTIONS

  # Retrieve syncz debug information directly from the control plane, using token security
  # (This is the usual way to get the debug information with an out-of-cluster control plane.)
  istioctl x debug syncz --xds-address istio.cloudprovider.example.com:15012

  # Retrieve syncz debug information via Kubernetes config, using token security
  # (This is the usual way to get the debug information with an in-cluster control plane.)
  istioctl x debug syncz

  # Retrieve syncz debug information directly from the control plane, using RSA certificate security
  # (Certificates must be obtained before this step.  The --cert-dir flag lets istioctl bypass the Kubernetes API server.)
  istioctl x debug syncz --xds-address istio.example.com:15012 --cert-dir ~/.istio-certs

  # Retrieve syncz information via XDS from specific control plane in multi-control plane in-cluster configuration
  # (Select a specific control plane in an in-cluster canary Istio configuration.)
  istioctl x debug syncz --xds-label istio.io/rev=default
`,
		RunE: func(c *cobra.Command, args []string) error {
			kubeClient, err := kubeClientWithRevision(kubeconfig, configContext, opts.Revision)
			if err != nil {
				return err
			}
			if len(args) == 0 {
				return CommandParseError{
					e: fmt.Errorf("debug type is required"),
				}
			}
			var xdsRequest xdsapi.DiscoveryRequest
			var namespace, serviceAccount string
			if len(args) > 1 {
				podName, ns, err := handlers.InferPodInfoFromTypedResource(args[1],
					handlers.HandleNamespace(namespace, defaultNamespace),
					kubeClient.UtilFactory())
				if err != nil {
					return err
				}
				pod, err := kubeClient.CoreV1().Pods(ns).Get(context.TODO(), podName, metav1.GetOptions{})
				if err != nil {
					return err
				}
				namespace = ns
				serviceAccount = pod.Spec.ServiceAccountName
				xdsRequest = xdsapi.DiscoveryRequest{
					ResourceNames: []string{fmt.Sprintf("%s?proxyID=%s.%s", args[0], podName, ns)},
					Node: &envoy_corev3.Node{
						Id: "debug~0.0.0.0~istioctl~cluster.local",
					},
					TypeUrl: TypeDebug,
				}
			} else {
				xdsRequest = xdsapi.DiscoveryRequest{
					ResourceNames: []string{args[0]},
					Node: &envoy_corev3.Node{
						Id: "debug~0.0.0.0~istioctl~cluster.local",
					},
					TypeUrl: TypeDebug,
				}
			}
			if centralOpts.Xds == "" {
				svc, err := kubeClient.CoreV1().Services(istioNamespace).Get(context.Background(), istiodServiceName, metav1.GetOptions{})
				if err != nil {
					return fmt.Errorf("please specify %q as %v", "--xds-address", err)
				}
				namespace, selector, err := polymorphichelpers.SelectorsForObject(svc)
				if err != nil {
					return fmt.Errorf("please specify %q as we cannot attach to %T: %v", "--xds-address", svc, err)
				}

				options := metav1.ListOptions{LabelSelector: selector.String()}

				podList, err := kubeClient.CoreV1().Pods(namespace).List(context.TODO(), options)
				if err != nil {
					return fmt.Errorf("please specify %q as %v", "--xds-address", err)
				}
				//  select a pod randomly to simulate current debug behavior
				pod := podList.Items[0]
				if err != nil {
					return fmt.Errorf("please specify %q as %v", "--xds-address", err)
				}
				podPort := 15012
				for _, v := range svc.Spec.Ports {
					if v.Name == xdsPortName {
						podPort = v.TargetPort.IntValue()
					}
				}
				f, err := kubeClient.NewPortForwarder(pod.Name, pod.Namespace, "", 0, podPort)
				if err != nil {
					return fmt.Errorf("please specify %q as %v", "--xds-address", err)
				}
				if err := f.Start(); err != nil {
					return fmt.Errorf("please specify %q as %v", "--xds-address", err)
				}
				centralOpts.Xds = f.Address()
				defer func() {
					f.Close()
					f.WaitForStop()
				}()
			}
			xdsResponses, err := multixds.AllRequestAndProcessXds(&xdsRequest, &centralOpts, istioNamespace,
				namespace, serviceAccount, kubeClient)
			if err != nil {
				return err
			}
			sw := pilot.XdsStatusWriter{Writer: c.OutOrStdout()}
			return sw.PrintAll(xdsResponses)
		},
	}

	opts.AttachControlPlaneFlags(debugCommand)
	centralOpts.AttachControlPlaneFlags(debugCommand)
	debugCommand.Long += "\n\n" + ExperimentalMsg
	return debugCommand
}
<|endoftext|>


<commit_before>package ca

import (
	"fmt"

	"github.com/cloudflare/cfssl/certdb"
	"github.com/ericyan/lorica/internal/certsql"
	"github.com/jmoiron/sqlx"
	_ "github.com/mattn/go-sqlite3"
)

var (
	// Adpated from https://github.com/cloudflare/cfssl/blob/master/certdb/sqlite/migrations/001_CreateCertificates.sql
	createSQL = `
CREATE TABLE IF NOT EXISTS metadata (
	key   blob NOT NULL,
	value blob NOT NULL,
	PRIMARY KEY(key)
);

CREATE TABLE IF NOT EXISTS certificates (
	serial_number            blob NOT NULL,
	authority_key_identifier blob NOT NULL,
	ca_label                 blob,
	status                   blob NOT NULL,
	reason                   int,
	expiry                   timestamp,
	revoked_at               timestamp,
	pem                      blob NOT NULL,
	PRIMARY KEY(serial_number, authority_key_identifier)
);

CREATE TABLE IF NOT EXISTS ocsp_responses (
	serial_number            blob NOT NULL,
	authority_key_identifier blob NOT NULL,
	body                     blob NOT NULL,
	expiry                   timestamp,
	PRIMARY KEY(serial_number, authority_key_identifier),
	FOREIGN KEY(serial_number, authority_key_identifier) REFERENCES certificates(serial_number, authority_key_identifier)
);
`

	insertMetadataSQL = `INSERT INTO metadata (key, value) VALUES (?, ?);`

	selectMetadataSQL = `SELECT value FROM metadata WHERE key = ?;`
)

type database struct {
	*sqlx.DB
}

func openDB(dsn string) (*database, error) {
	db, err := sqlx.Open("sqlite3", dsn)
	if err != nil {
		return nil, err
	}

	_, err = db.Exec(createSQL)
	if err != nil {
		return nil, err
	}

	return &database{db}, nil
}

func (db *database) Accessor() certdb.Accessor {
	return certsql.NewAccessor(db.DB)
}

func (db *database) GetMetadata(key []byte) ([]byte, error) {
	var values [][]byte
	err := db.DB.Select(&values, selectMetadataSQL, key)
	if err != nil {
		return nil, err
	}

	return values[0], nil
}

func (db *database) SetMetadata(key, value []byte) error {
	res, err := db.DB.Exec(insertMetadataSQL, key, value)
	if err != nil {
		return err
	}

	numRowsAffected, err := res.RowsAffected()

	if numRowsAffected == 0 {
		return fmt.Errorf("failed to insert the certificate record")
	}

	if numRowsAffected != 1 {
		return fmt.Errorf("%d rows are affected, should be 1 row", numRowsAffected)
	}

	return err
}
<commit_msg>Do not panic when getting nil metadata<commit_after>package ca

import (
	"errors"
	"fmt"

	"github.com/cloudflare/cfssl/certdb"
	"github.com/ericyan/lorica/internal/certsql"
	"github.com/jmoiron/sqlx"
	_ "github.com/mattn/go-sqlite3"
)

var (
	// Adpated from https://github.com/cloudflare/cfssl/blob/master/certdb/sqlite/migrations/001_CreateCertificates.sql
	createSQL = `
CREATE TABLE IF NOT EXISTS metadata (
	key   blob NOT NULL,
	value blob NOT NULL,
	PRIMARY KEY(key)
);

CREATE TABLE IF NOT EXISTS certificates (
	serial_number            blob NOT NULL,
	authority_key_identifier blob NOT NULL,
	ca_label                 blob,
	status                   blob NOT NULL,
	reason                   int,
	expiry                   timestamp,
	revoked_at               timestamp,
	pem                      blob NOT NULL,
	PRIMARY KEY(serial_number, authority_key_identifier)
);

CREATE TABLE IF NOT EXISTS ocsp_responses (
	serial_number            blob NOT NULL,
	authority_key_identifier blob NOT NULL,
	body                     blob NOT NULL,
	expiry                   timestamp,
	PRIMARY KEY(serial_number, authority_key_identifier),
	FOREIGN KEY(serial_number, authority_key_identifier) REFERENCES certificates(serial_number, authority_key_identifier)
);
`

	insertMetadataSQL = `INSERT INTO metadata (key, value) VALUES (?, ?);`

	selectMetadataSQL = `SELECT value FROM metadata WHERE key = ?;`
)

type database struct {
	*sqlx.DB
}

func openDB(dsn string) (*database, error) {
	db, err := sqlx.Open("sqlite3", dsn)
	if err != nil {
		return nil, err
	}

	_, err = db.Exec(createSQL)
	if err != nil {
		return nil, err
	}

	return &database{db}, nil
}

func (db *database) Accessor() certdb.Accessor {
	return certsql.NewAccessor(db.DB)
}

func (db *database) GetMetadata(key []byte) ([]byte, error) {
	var values [][]byte
	err := db.DB.Select(&values, selectMetadataSQL, key)
	if err != nil {
		return nil, err
	}

	if len(values) == 0 {
		return nil, errors.New("empty value")
	}

	return values[0], nil
}

func (db *database) SetMetadata(key, value []byte) error {
	res, err := db.DB.Exec(insertMetadataSQL, key, value)
	if err != nil {
		return err
	}

	numRowsAffected, err := res.RowsAffected()

	if numRowsAffected == 0 {
		return fmt.Errorf("failed to insert the certificate record")
	}

	if numRowsAffected != 1 {
		return fmt.Errorf("%d rows are affected, should be 1 row", numRowsAffected)
	}

	return err
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tls

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/des"
	"crypto/hmac"
	"crypto/rc4"
	"crypto/sha1"
	"crypto/x509"
	"hash"
)

// a keyAgreement implements the client and server side of a TLS key agreement
// protocol by generating and processing key exchange messages.
type keyAgreement interface {
	// On the server side, the first two methods are called in order.

	// In the case that the key agreement protocol doesn't use a
	// ServerKeyExchange message, generateServerKeyExchange can return nil,
	// nil.
	generateServerKeyExchange(*Config, *Certificate, *clientHelloMsg, *serverHelloMsg) (*serverKeyExchangeMsg, error)
	processClientKeyExchange(*Config, *Certificate, *clientKeyExchangeMsg, uint16) ([]byte, error)

	// On the client side, the next two methods are called in order.

	// This method may not be called if the server doesn't send a
	// ServerKeyExchange message.
	processServerKeyExchange(*Config, *clientHelloMsg, *serverHelloMsg, *x509.Certificate, *serverKeyExchangeMsg) error
	generateClientKeyExchange(*Config, *clientHelloMsg, *x509.Certificate) ([]byte, *clientKeyExchangeMsg, error)
}

const (
	// suiteECDH indicates that the cipher suite involves elliptic curve
	// Diffie-Hellman. This means that it should only be selected when the
	// client indicates that it supports ECC with a curve and point format
	// that we're happy with.
	suiteECDHE = 1 << iota
	// suiteECDSA indicates that the cipher suite involves an ECDSA
	// signature and therefore may only be selected when the server's
	// certificate is ECDSA. If this is not set then the cipher suite is
	// RSA based.
	suiteECDSA
	// suiteTLS12 indicates that the cipher suite should only be advertised
	// and accepted when using TLS 1.2.
	suiteTLS12
)

// A cipherSuite is a specific combination of key agreement, cipher and MAC
// function. All cipher suites currently assume RSA key agreement.
type cipherSuite struct {
	id uint16
	// the lengths, in bytes, of the key material needed for each component.
	keyLen int
	macLen int
	ivLen  int
	ka     func(version uint16) keyAgreement
	// flags is a bitmask of the suite* values, above.
	flags  int
	cipher func(key, iv []byte, isRead bool) interface{}
	mac    func(version uint16, macKey []byte) macFunction
	aead   func(key, fixedNonce []byte) cipher.AEAD
}

var cipherSuites = []*cipherSuite{
	// Ciphersuite order is chosen so that ECDHE comes before plain RSA
	// and RC4 comes before AES (because of the Lucky13 attack).
	{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, 16, 0, 4, ecdheRSAKA, suiteECDHE | suiteTLS12, nil, nil, aeadAESGCM},
	{TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, 16, 0, 4, ecdheECDSAKA, suiteECDHE | suiteECDSA | suiteTLS12, nil, nil, aeadAESGCM},
	{TLS_ECDHE_RSA_WITH_RC4_128_SHA, 16, 20, 0, ecdheRSAKA, suiteECDHE, cipherRC4, macSHA1, nil},
	{TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, 16, 20, 0, ecdheECDSAKA, suiteECDHE | suiteECDSA, cipherRC4, macSHA1, nil},
	{TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, 16, 20, 16, ecdheRSAKA, suiteECDHE, cipherAES, macSHA1, nil},
	{TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, 16, 20, 16, ecdheECDSAKA, suiteECDHE | suiteECDSA, cipherAES, macSHA1, nil},
	{TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, 32, 20, 16, ecdheRSAKA, suiteECDHE, cipherAES, macSHA1, nil},
	{TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, 32, 20, 16, ecdheECDSAKA, suiteECDHE | suiteECDSA, cipherAES, macSHA1, nil},
	{TLS_RSA_WITH_RC4_128_SHA, 16, 20, 0, rsaKA, 0, cipherRC4, macSHA1, nil},
	{TLS_RSA_WITH_AES_128_CBC_SHA, 16, 20, 16, rsaKA, 0, cipherAES, macSHA1, nil},
	{TLS_RSA_WITH_AES_256_CBC_SHA, 32, 20, 16, rsaKA, 0, cipherAES, macSHA1, nil},
	{TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, 24, 20, 8, ecdheRSAKA, suiteECDHE, cipher3DES, macSHA1, nil},
	{TLS_RSA_WITH_3DES_EDE_CBC_SHA, 24, 20, 8, rsaKA, 0, cipher3DES, macSHA1, nil},
}

func cipherRC4(key, iv []byte, isRead bool) interface{} {
	cipher, _ := rc4.NewCipher(key)
	return cipher
}

func cipher3DES(key, iv []byte, isRead bool) interface{} {
	block, _ := des.NewTripleDESCipher(key)
	if isRead {
		return cipher.NewCBCDecrypter(block, iv)
	}
	return cipher.NewCBCEncrypter(block, iv)
}

func cipherAES(key, iv []byte, isRead bool) interface{} {
	block, _ := aes.NewCipher(key)
	if isRead {
		return cipher.NewCBCDecrypter(block, iv)
	}
	return cipher.NewCBCEncrypter(block, iv)
}

// macSHA1 returns a macFunction for the given protocol version.
func macSHA1(version uint16, key []byte) macFunction {
	if version == VersionSSL30 {
		mac := ssl30MAC{
			h:   sha1.New(),
			key: make([]byte, len(key)),
		}
		copy(mac.key, key)
		return mac
	}
	return tls10MAC{hmac.New(sha1.New, key)}
}

type macFunction interface {
	Size() int
	MAC(digestBuf, seq, header, data []byte) []byte
}

// fixedNonceAEAD wraps an AEAD and prefixes a fixed portion of the nonce to
// each call.
type fixedNonceAEAD struct {
	// sealNonce and openNonce are buffers where the larger nonce will be
	// constructed. Since a seal and open operation may be running
	// concurrently, there is a separate buffer for each.
	sealNonce, openNonce []byte
	aead                 cipher.AEAD
}

func (f *fixedNonceAEAD) NonceSize() int { return 8 }
func (f *fixedNonceAEAD) Overhead() int  { return f.aead.Overhead() }

func (f *fixedNonceAEAD) Seal(out, nonce, plaintext, additionalData []byte) []byte {
	copy(f.sealNonce[len(f.sealNonce)-8:], nonce)
	return f.aead.Seal(out, f.sealNonce, plaintext, additionalData)
}

func (f *fixedNonceAEAD) Open(out, nonce, plaintext, additionalData []byte) ([]byte, error) {
	copy(f.openNonce[len(f.openNonce)-8:], nonce)
	return f.aead.Open(out, f.openNonce, plaintext, additionalData)
}

func aeadAESGCM(key, fixedNonce []byte) cipher.AEAD {
	aes, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}
	aead, err := cipher.NewGCM(aes)
	if err != nil {
		panic(err)
	}

	nonce1, nonce2 := make([]byte, 12), make([]byte, 12)
	copy(nonce1, fixedNonce)
	copy(nonce2, fixedNonce)

	return &fixedNonceAEAD{nonce1, nonce2, aead}
}

// ssl30MAC implements the SSLv3 MAC function, as defined in
// www.mozilla.org/projects/security/pki/nss/ssl/draft302.txt section 5.2.3.1
type ssl30MAC struct {
	h   hash.Hash
	key []byte
}

func (s ssl30MAC) Size() int {
	return s.h.Size()
}

var ssl30Pad1 = [48]byte{0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36}

var ssl30Pad2 = [48]byte{0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c}

func (s ssl30MAC) MAC(digestBuf, seq, header, data []byte) []byte {
	padLength := 48
	if s.h.Size() == 20 {
		padLength = 40
	}

	s.h.Reset()
	s.h.Write(s.key)
	s.h.Write(ssl30Pad1[:padLength])
	s.h.Write(seq)
	s.h.Write(header[:1])
	s.h.Write(header[3:5])
	s.h.Write(data)
	digestBuf = s.h.Sum(digestBuf[:0])

	s.h.Reset()
	s.h.Write(s.key)
	s.h.Write(ssl30Pad2[:padLength])
	s.h.Write(digestBuf)
	return s.h.Sum(digestBuf[:0])
}

// tls10MAC implements the TLS 1.0 MAC function. RFC 2246, section 6.2.3.
type tls10MAC struct {
	h hash.Hash
}

func (s tls10MAC) Size() int {
	return s.h.Size()
}

func (s tls10MAC) MAC(digestBuf, seq, header, data []byte) []byte {
	s.h.Reset()
	s.h.Write(seq)
	s.h.Write(header)
	s.h.Write(data)
	return s.h.Sum(digestBuf[:0])
}

func rsaKA(version uint16) keyAgreement {
	return rsaKeyAgreement{}
}

func ecdheECDSAKA(version uint16) keyAgreement {
	return &ecdheKeyAgreement{
		sigType: signatureECDSA,
		version: version,
	}
}

func ecdheRSAKA(version uint16) keyAgreement {
	return &ecdheKeyAgreement{
		sigType: signatureRSA,
		version: version,
	}
}

// mutualCipherSuite returns a cipherSuite given a list of supported
// ciphersuites and the id requested by the peer.
func mutualCipherSuite(have []uint16, want uint16) *cipherSuite {
	for _, id := range have {
		if id == want {
			for _, suite := range cipherSuites {
				if suite.id == want {
					return suite
				}
			}
			return nil
		}
	}
	return nil
}

// A list of the possible cipher suite ids. Taken from
// http://www.iana.org/assignments/tls-parameters/tls-parameters.xml
const (
	TLS_RSA_WITH_RC4_128_SHA                uint16 = 0x0005
	TLS_RSA_WITH_3DES_EDE_CBC_SHA           uint16 = 0x000a
	TLS_RSA_WITH_AES_128_CBC_SHA            uint16 = 0x002f
	TLS_RSA_WITH_AES_256_CBC_SHA            uint16 = 0x0035
	TLS_ECDHE_ECDSA_WITH_RC4_128_SHA        uint16 = 0xc007
	TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA    uint16 = 0xc009
	TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA    uint16 = 0xc00a
	TLS_ECDHE_RSA_WITH_RC4_128_SHA          uint16 = 0xc011
	TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA     uint16 = 0xc012
	TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA      uint16 = 0xc013
	TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA      uint16 = 0xc014
	TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256   uint16 = 0xc02f
	TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 uint16 = 0xc02b

	// TLS_FALLBACK_SCSV isn't a standard cipher suite but an indicator
	// that the client is doing version fallback. See
	// https://tools.ietf.org/html/draft-ietf-tls-downgrade-scsv-00.
	TLS_FALLBACK_SCSV uint16 = 0x5600
)
<commit_msg>always return a cipher suite<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tls

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/des"
	"crypto/hmac"
	"crypto/rc4"
	"crypto/sha1"
	"crypto/x509"
	"hash"
)

// a keyAgreement implements the client and server side of a TLS key agreement
// protocol by generating and processing key exchange messages.
type keyAgreement interface {
	// On the server side, the first two methods are called in order.

	// In the case that the key agreement protocol doesn't use a
	// ServerKeyExchange message, generateServerKeyExchange can return nil,
	// nil.
	generateServerKeyExchange(*Config, *Certificate, *clientHelloMsg, *serverHelloMsg) (*serverKeyExchangeMsg, error)
	processClientKeyExchange(*Config, *Certificate, *clientKeyExchangeMsg, uint16) ([]byte, error)

	// On the client side, the next two methods are called in order.

	// This method may not be called if the server doesn't send a
	// ServerKeyExchange message.
	processServerKeyExchange(*Config, *clientHelloMsg, *serverHelloMsg, *x509.Certificate, *serverKeyExchangeMsg) error
	generateClientKeyExchange(*Config, *clientHelloMsg, *x509.Certificate) ([]byte, *clientKeyExchangeMsg, error)
}

const (
	// suiteECDH indicates that the cipher suite involves elliptic curve
	// Diffie-Hellman. This means that it should only be selected when the
	// client indicates that it supports ECC with a curve and point format
	// that we're happy with.
	suiteECDHE = 1 << iota
	// suiteECDSA indicates that the cipher suite involves an ECDSA
	// signature and therefore may only be selected when the server's
	// certificate is ECDSA. If this is not set then the cipher suite is
	// RSA based.
	suiteECDSA
	// suiteTLS12 indicates that the cipher suite should only be advertised
	// and accepted when using TLS 1.2.
	suiteTLS12
)

// A cipherSuite is a specific combination of key agreement, cipher and MAC
// function. All cipher suites currently assume RSA key agreement.
type cipherSuite struct {
	id uint16
	// the lengths, in bytes, of the key material needed for each component.
	keyLen int
	macLen int
	ivLen  int
	ka     func(version uint16) keyAgreement
	// flags is a bitmask of the suite* values, above.
	flags  int
	cipher func(key, iv []byte, isRead bool) interface{}
	mac    func(version uint16, macKey []byte) macFunction
	aead   func(key, fixedNonce []byte) cipher.AEAD
}

var cipherSuites = []*cipherSuite{
	// Ciphersuite order is chosen so that ECDHE comes before plain RSA
	// and RC4 comes before AES (because of the Lucky13 attack).
	{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, 16, 0, 4, ecdheRSAKA, suiteECDHE | suiteTLS12, nil, nil, aeadAESGCM},
	{TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, 16, 0, 4, ecdheECDSAKA, suiteECDHE | suiteECDSA | suiteTLS12, nil, nil, aeadAESGCM},
	{TLS_ECDHE_RSA_WITH_RC4_128_SHA, 16, 20, 0, ecdheRSAKA, suiteECDHE, cipherRC4, macSHA1, nil},
	{TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, 16, 20, 0, ecdheECDSAKA, suiteECDHE | suiteECDSA, cipherRC4, macSHA1, nil},
	{TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, 16, 20, 16, ecdheRSAKA, suiteECDHE, cipherAES, macSHA1, nil},
	{TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, 16, 20, 16, ecdheECDSAKA, suiteECDHE | suiteECDSA, cipherAES, macSHA1, nil},
	{TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, 32, 20, 16, ecdheRSAKA, suiteECDHE, cipherAES, macSHA1, nil},
	{TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, 32, 20, 16, ecdheECDSAKA, suiteECDHE | suiteECDSA, cipherAES, macSHA1, nil},
	{TLS_RSA_WITH_RC4_128_SHA, 16, 20, 0, rsaKA, 0, cipherRC4, macSHA1, nil},
	{TLS_RSA_WITH_AES_128_CBC_SHA, 16, 20, 16, rsaKA, 0, cipherAES, macSHA1, nil},
	{TLS_RSA_WITH_AES_256_CBC_SHA, 32, 20, 16, rsaKA, 0, cipherAES, macSHA1, nil},
	{TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, 24, 20, 8, ecdheRSAKA, suiteECDHE, cipher3DES, macSHA1, nil},
	{TLS_RSA_WITH_3DES_EDE_CBC_SHA, 24, 20, 8, rsaKA, 0, cipher3DES, macSHA1, nil},
}

func cipherRC4(key, iv []byte, isRead bool) interface{} {
	cipher, _ := rc4.NewCipher(key)
	return cipher
}

func cipher3DES(key, iv []byte, isRead bool) interface{} {
	block, _ := des.NewTripleDESCipher(key)
	if isRead {
		return cipher.NewCBCDecrypter(block, iv)
	}
	return cipher.NewCBCEncrypter(block, iv)
}

func cipherAES(key, iv []byte, isRead bool) interface{} {
	block, _ := aes.NewCipher(key)
	if isRead {
		return cipher.NewCBCDecrypter(block, iv)
	}
	return cipher.NewCBCEncrypter(block, iv)
}

// macSHA1 returns a macFunction for the given protocol version.
func macSHA1(version uint16, key []byte) macFunction {
	if version == VersionSSL30 {
		mac := ssl30MAC{
			h:   sha1.New(),
			key: make([]byte, len(key)),
		}
		copy(mac.key, key)
		return mac
	}
	return tls10MAC{hmac.New(sha1.New, key)}
}

type macFunction interface {
	Size() int
	MAC(digestBuf, seq, header, data []byte) []byte
}

// fixedNonceAEAD wraps an AEAD and prefixes a fixed portion of the nonce to
// each call.
type fixedNonceAEAD struct {
	// sealNonce and openNonce are buffers where the larger nonce will be
	// constructed. Since a seal and open operation may be running
	// concurrently, there is a separate buffer for each.
	sealNonce, openNonce []byte
	aead                 cipher.AEAD
}

func (f *fixedNonceAEAD) NonceSize() int { return 8 }
func (f *fixedNonceAEAD) Overhead() int  { return f.aead.Overhead() }

func (f *fixedNonceAEAD) Seal(out, nonce, plaintext, additionalData []byte) []byte {
	copy(f.sealNonce[len(f.sealNonce)-8:], nonce)
	return f.aead.Seal(out, f.sealNonce, plaintext, additionalData)
}

func (f *fixedNonceAEAD) Open(out, nonce, plaintext, additionalData []byte) ([]byte, error) {
	copy(f.openNonce[len(f.openNonce)-8:], nonce)
	return f.aead.Open(out, f.openNonce, plaintext, additionalData)
}

func aeadAESGCM(key, fixedNonce []byte) cipher.AEAD {
	aes, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}
	aead, err := cipher.NewGCM(aes)
	if err != nil {
		panic(err)
	}

	nonce1, nonce2 := make([]byte, 12), make([]byte, 12)
	copy(nonce1, fixedNonce)
	copy(nonce2, fixedNonce)

	return &fixedNonceAEAD{nonce1, nonce2, aead}
}

// ssl30MAC implements the SSLv3 MAC function, as defined in
// www.mozilla.org/projects/security/pki/nss/ssl/draft302.txt section 5.2.3.1
type ssl30MAC struct {
	h   hash.Hash
	key []byte
}

func (s ssl30MAC) Size() int {
	return s.h.Size()
}

var ssl30Pad1 = [48]byte{0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36}

var ssl30Pad2 = [48]byte{0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c, 0x5c}

func (s ssl30MAC) MAC(digestBuf, seq, header, data []byte) []byte {
	padLength := 48
	if s.h.Size() == 20 {
		padLength = 40
	}

	s.h.Reset()
	s.h.Write(s.key)
	s.h.Write(ssl30Pad1[:padLength])
	s.h.Write(seq)
	s.h.Write(header[:1])
	s.h.Write(header[3:5])
	s.h.Write(data)
	digestBuf = s.h.Sum(digestBuf[:0])

	s.h.Reset()
	s.h.Write(s.key)
	s.h.Write(ssl30Pad2[:padLength])
	s.h.Write(digestBuf)
	return s.h.Sum(digestBuf[:0])
}

// tls10MAC implements the TLS 1.0 MAC function. RFC 2246, section 6.2.3.
type tls10MAC struct {
	h hash.Hash
}

func (s tls10MAC) Size() int {
	return s.h.Size()
}

func (s tls10MAC) MAC(digestBuf, seq, header, data []byte) []byte {
	s.h.Reset()
	s.h.Write(seq)
	s.h.Write(header)
	s.h.Write(data)
	return s.h.Sum(digestBuf[:0])
}

func rsaKA(version uint16) keyAgreement {
	return rsaKeyAgreement{}
}

func ecdheECDSAKA(version uint16) keyAgreement {
	return &ecdheKeyAgreement{
		sigType: signatureECDSA,
		version: version,
	}
}

func ecdheRSAKA(version uint16) keyAgreement {
	return &ecdheKeyAgreement{
		sigType: signatureRSA,
		version: version,
	}
}

// mutualCipherSuite returns a cipherSuite given a list of supported
// ciphersuites and the id requested by the peer.
func mutualCipherSuite(have []uint16, want uint16) *cipherSuite {
	for _, id := range have {
		if id == want {
			for _, suite := range cipherSuites {
				if suite.id == want {
					return suite
				}
			}
			//return nil
			return cipherSuites[0]
		}
	}
	//return nil
	return cipherSuites[0]
}

// A list of the possible cipher suite ids. Taken from
// http://www.iana.org/assignments/tls-parameters/tls-parameters.xml
const (
	TLS_RSA_WITH_RC4_128_SHA                uint16 = 0x0005
	TLS_RSA_WITH_3DES_EDE_CBC_SHA           uint16 = 0x000a
	TLS_RSA_WITH_AES_128_CBC_SHA            uint16 = 0x002f
	TLS_RSA_WITH_AES_256_CBC_SHA            uint16 = 0x0035
	TLS_ECDHE_ECDSA_WITH_RC4_128_SHA        uint16 = 0xc007
	TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA    uint16 = 0xc009
	TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA    uint16 = 0xc00a
	TLS_ECDHE_RSA_WITH_RC4_128_SHA          uint16 = 0xc011
	TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA     uint16 = 0xc012
	TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA      uint16 = 0xc013
	TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA      uint16 = 0xc014
	TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256   uint16 = 0xc02f
	TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 uint16 = 0xc02b

	// TLS_FALLBACK_SCSV isn't a standard cipher suite but an indicator
	// that the client is doing version fallback. See
	// https://tools.ietf.org/html/draft-ietf-tls-downgrade-scsv-00.
	TLS_FALLBACK_SCSV uint16 = 0x5600
)
<|endoftext|>


<commit_before>package tokenbucket

import (
	"time"
)

const (
	token byte = 00
)

// NewTokenBucket returns a new token-bucket.
func NewTokenBucket(rate time.Duration, size uint16) chan byte {
	sizeInt := int(size)
	bucket := make(chan byte, sizeInt)
	fillBucket(bucket, sizeInt)

	go func(bucket chan<- byte) {
		for _ = range time.NewTicker(rate).C {
			fillBucket(bucket, sizeInt)
		}
	}(bucket)

	return bucket
}

func fillBucket(bucket chan<- byte, size int) {
	for i := size - len(bucket); i > 0; i-- {
		bucket <- token
	}
}
<commit_msg>refactor<commit_after>package tokenbucket

import (
	"time"
)

// NewTokenBucket returns a new token-bucket.
func NewTokenBucket(rate time.Duration, size uint16) chan struct{} {
	sizeInt := int(size)
	bucket := make(chan struct{}, sizeInt)
	fillBucket(bucket, sizeInt)

	go func(bucket chan<- struct{}) {
		for _ = range time.NewTicker(rate).C {
			fillBucket(bucket, sizeInt)
		}
	}(bucket)

	return bucket
}

func fillBucket(bucket chan<- struct{}, size int) {
	for i := size - len(bucket); i > 0; i-- {
		bucket <- struct{}{}
	}
}
<|endoftext|>


<commit_before>package play

import (
  "appengine"
  "appengine/datastore"
  "bytes"
  "crypto/sha1"
  "encoding/json"
  "fmt"
  "net/http"
  "github.com/lestrrat/go-xslate"
)

type SavedTemplate struct {
  Template string
  Variables string
}

var TX *xslate.Xslate
func init() {
  http.HandleFunc("/api/render", serveApiRender)
  http.HandleFunc("/api/save", serveApiSave)
  http.HandleFunc("/p/", serveLoad)
  http.HandleFunc("/", serveIndex)
}

func createXslate() *xslate.Xslate {
  if TX != nil {
    return TX
  }
  TX, err := xslate.New(xslate.Args {
    "Loader": xslate.Args {
      "LoadPaths": []string { "./templates" },
    },
  })

  if err != nil {
    panic("Could not instantiate xslate!")
  }

  return TX
}

func serveIndex(w http.ResponseWriter, r *http.Request) {
  tx := createXslate()
  err := tx.RenderInto(w, "index.html.tx", xslate.Vars {
    "Template": `[%- # This is where your template go -%]
Hello World [% name %]!
[%- # Variables can be specified in the box below, in JSON format -%]`,
    "Variables": `{
"name": "Bob"
}`,
  })
  if err != nil {
    http.Error(w, err.Error(), http.StatusInternalServerError)
    return
  }
}

func serveLoad(w http.ResponseWriter, r *http.Request) {
  path := r.URL.Path
  // path starts with /p/, we need everything after that
  id := path[3:]

  c := appengine.NewContext(r)
  var st SavedTemplate
  err := datastore.Get(
    c,
    datastore.NewKey(c, "SavedTemplate", id, 0, nil),
    &st,
  )
  if err != nil {
    http.Error(w, err.Error(), http.StatusNotFound)
    return
  }

  tx := createXslate()
  err = tx.RenderInto(w, "index.html.tx", xslate.Vars {
    "Template": st.Template,
    "Variables": st.Variables,
  })
  if err != nil {
    http.Error(w, err.Error(), http.StatusInternalServerError)
    return
  }
}

func createSavedTemplate(r *http.Request) (*SavedTemplate, error) {
  template := r.FormValue("template")
  variables := make(map[string]interface {})

  varString := r.FormValue("variables")
  if varString != "" {
    err := json.Unmarshal([]byte(varString), &variables)
    if err != nil {
        return nil, err
    }
  }

  // Normalize variables into a string
  jsonStr, err := json.Marshal(&variables)
  if err != nil {
    return nil, err
  }
  buf := &bytes.Buffer {}
  json.Compact(buf, jsonStr)

  varString = buf.String()

  st := &SavedTemplate { template, varString }

  return st, nil
}

// StringID creates a new string based on the contents of the template
// that is, if two people try to save identical templates,
// they both refer to the same ID
func (st *SavedTemplate) StringID() string {
  h := sha1.New()
  fmt.Fprintf(h, "%s", st.Template)
  fmt.Fprintf(h, "%s", st.Variables)
  return fmt.Sprintf("%x", h.Sum(nil))
}

func serveApiSave(w http.ResponseWriter, r *http.Request) {
  st, err := createSavedTemplate(r)
  if err != nil {
    http.Error(w, err.Error(), http.StatusInternalServerError)
    return
  }

  c := appengine.NewContext(r)
  key := datastore.NewKey(c, "SavedTemplate", st.StringID(), 0, nil)
  _, err = datastore.Put(
    c,
    key,
    st,
  )
  if err != nil {
    http.Error(w, err.Error(), http.StatusInternalServerError)
    return
  }

  fmt.Fprintf(w, "%s", key.StringID())

}

func serveApiRender(w http.ResponseWriter, r *http.Request) {
  tx := createXslate()

  var vars map[string]interface {}
  if varString := r.FormValue("variables"); varString != "" {
    err := json.Unmarshal([]byte(varString), &vars)
    if err != nil {
      w.WriteHeader(500)
      fmt.Fprintf(w, "Failed to decode variables: %s", err)
      return
    }
  }

  output, err := tx.RenderString(r.FormValue("template"), xslate.Vars(vars))
  if err != nil {
    w.WriteHeader(500)
    fmt.Fprintf(w, err.Error())
    return
  }

  fmt.Fprint(w, output)
}
<commit_msg>I'm a born again go-hard-tabber<commit_after>package play

import (
	"appengine"
	"appengine/datastore"
	"bytes"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"github.com/lestrrat/go-xslate"
	"net/http"
)

type SavedTemplate struct {
	Template  string
	Variables string
}

var TX *xslate.Xslate

func init() {
	http.HandleFunc("/api/render", serveApiRender)
	http.HandleFunc("/api/save", serveApiSave)
	http.HandleFunc("/p/", serveLoad)
	http.HandleFunc("/", serveIndex)
}

func createXslate() *xslate.Xslate {
	if TX != nil {
		return TX
	}
	TX, err := xslate.New(xslate.Args{
		"Loader": xslate.Args{
			"LoadPaths": []string{"./templates"},
		},
	})

	if err != nil {
		panic("Could not instantiate xslate!")
	}

	return TX
}

func serveIndex(w http.ResponseWriter, r *http.Request) {
	tx := createXslate()
	err := tx.RenderInto(w, "index.html.tx", xslate.Vars{
		"Template": `[%- # This is where your template go -%]
Hello World [% name %]!
[%- # Variables can be specified in the box below, in JSON format -%]`,
		"Variables": `{
"name": "Bob"
}`,
	})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

func serveLoad(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path
	// path starts with /p/, we need everything after that
	id := path[3:]

	c := appengine.NewContext(r)
	var st SavedTemplate
	err := datastore.Get(
		c,
		datastore.NewKey(c, "SavedTemplate", id, 0, nil),
		&st,
	)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}

	tx := createXslate()
	err = tx.RenderInto(w, "index.html.tx", xslate.Vars{
		"Template":  st.Template,
		"Variables": st.Variables,
	})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

func createSavedTemplate(r *http.Request) (*SavedTemplate, error) {
	template := r.FormValue("template")
	variables := make(map[string]interface{})

	varString := r.FormValue("variables")
	if varString != "" {
		err := json.Unmarshal([]byte(varString), &variables)
		if err != nil {
			return nil, err
		}
	}

	// Normalize variables into a string
	jsonStr, err := json.Marshal(&variables)
	if err != nil {
		return nil, err
	}
	buf := &bytes.Buffer{}
	json.Compact(buf, jsonStr)

	varString = buf.String()

	st := &SavedTemplate{template, varString}

	return st, nil
}

// StringID creates a new string based on the contents of the template
// that is, if two people try to save identical templates,
// they both refer to the same ID
func (st *SavedTemplate) StringID() string {
	h := sha1.New()
	fmt.Fprintf(h, "%s", st.Template)
	fmt.Fprintf(h, "%s", st.Variables)
	return fmt.Sprintf("%x", h.Sum(nil))
}

func serveApiSave(w http.ResponseWriter, r *http.Request) {
	st, err := createSavedTemplate(r)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	c := appengine.NewContext(r)
	key := datastore.NewKey(c, "SavedTemplate", st.StringID(), 0, nil)
	_, err = datastore.Put(
		c,
		key,
		st,
	)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	fmt.Fprintf(w, "%s", key.StringID())

}

func serveApiRender(w http.ResponseWriter, r *http.Request) {
	tx := createXslate()

	var vars map[string]interface{}
	if varString := r.FormValue("variables"); varString != "" {
		err := json.Unmarshal([]byte(varString), &vars)
		if err != nil {
			w.WriteHeader(500)
			fmt.Fprintf(w, "Failed to decode variables: %s", err)
			return
		}
	}

	output, err := tx.RenderString(r.FormValue("template"), xslate.Vars(vars))
	if err != nil {
		w.WriteHeader(500)
		fmt.Fprintf(w, err.Error())
		return
	}

	fmt.Fprint(w, output)
}
<|endoftext|>


<commit_before>package models

import (
	"io"
	"log"
	"net/http"
	"os"
	"strings"

	"github.com/Xe/uuid"
	"github.com/jinzhu/gorm"
	. "stevenbooru.cf/globals"
)

// Image is an image that has been uploaded to the booru.
type Image struct {
	gorm.Model
	UUID        string `sql:"unique;size:36" json:"uuid"`
	Poster      *User
	PosterID    uint
	Tags        []*Tag
	Hash        string `sql:"unique;size:128"`
	Filename    string `sql:"size:512"`
	Description string `sql:"size:2048"`
	Mime        string
}

func NewImage(r *http.Request, user *User) (i *Image, err error) {
	iuuid := uuid.NewUUID()

	fin, header, err := r.FormFile("file")
	if err != nil {
		return nil, err
	}

	tags := r.FormValue("tags")

	log.Printf("%#v", header.Header)
	log.Printf("%s", strings.Split(tags, ","))

	mime := header.Header.Get("Content-Type")

	i = &Image{
		UUID:     iuuid.String(),
		Poster:   user,
		PosterID: user.ID,
		Mime:     mime,
		Filename: header.Filename,
	}

	err = os.Mkdir(Config.Storage.Path+"/"+i.UUID, os.ModePerm)
	if err != nil {
		return nil, err
	}

	fout, err := os.Create(Config.Storage.Path + "/" + i.UUID + "/" + i.Filename)
	if err != nil {
		return nil, err
	}

	io.Copy(fout, fin)

	conn := Redis.Get()
	defer conn.Close()

	_, err = conn.Do("RPUSH", "uploads", i.UUID)
	if err != nil {
		return nil, err
	}

	query := Db.Create(i)
	if query.Error != nil {
		return nil, query.Error
	}

	return
}
<commit_msg>Refine image uploading<commit_after>package models

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"

	"github.com/Xe/uuid"
	"github.com/dchest/blake2b"
	"github.com/jinzhu/gorm"
	. "stevenbooru.cf/globals"
)

// Image is an image that has been uploaded to the booru.
type Image struct {
	gorm.Model
	UUID        string `sql:"unique;size:36" json:"uuid"`
	Poster      *User
	PosterID    uint
	Tags        []*Tag
	Hash        string `sql:"unique;size:128"`
	Filename    string `sql:"size:512"`
	Description string `sql:"size:2048"`
	Mime        string
}

func NewImage(r *http.Request, user *User) (i *Image, err error) {
	iuuid := uuid.NewUUID()

	userUpload, header, err := r.FormFile("file")
	if err != nil {
		return nil, err
	}

	data, err := ioutil.ReadAll(userUpload)
	if err != nil {
		return nil, err
	}

	c := &blake2b.Config{
		Salt: []byte("lol as if"),
		Size: blake2b.Size,
	}

	b2b, err := blake2b.New(c)
	if err != nil {
		panic(err)
	}

	b2b.Reset()
	io.Copy(b2b, bytes.NewBuffer(data))
	hash := fmt.Sprintf("%x", b2b.Sum(nil))

	tags := r.FormValue("tags")

	log.Printf("%#v", header.Header)
	log.Printf("%s", strings.Split(tags, ","))

	mime := header.Header.Get("Content-Type")

	i = &Image{
		UUID:        iuuid.String(),
		Poster:      user,
		PosterID:    user.ID,
		Mime:        mime,
		Filename:    header.Filename,
		Hash:        hash,
		Description: r.Form.Get("description"),
	}

	err = os.Mkdir(Config.Storage.Path+"/"+i.UUID, os.ModePerm)
	if err != nil {
		return nil, err
	}

	fout, err := os.Create(Config.Storage.Path + "/" + i.UUID + "/" + i.Filename)
	if err != nil {
		return nil, err
	}

	io.Copy(fout, bytes.NewBuffer(data))

	conn := Redis.Get()
	defer conn.Close()

	_, err = conn.Do("RPUSH", "uploads", i.UUID)
	if err != nil {
		return nil, err
	}

	query := Db.Create(i)
	if query.Error != nil {
		return nil, query.Error
	}

	return
}
<|endoftext|>


<commit_before>package zeroconf

import (
	"strings"
	"net"

	dns "github.com/miekg/godns"
)

type Service struct {
        Type      string
        Name      string
        Domain    string
        Interface *net.Interface
        Address   net.Addr
}

// s.unmarshal may not be complete, return false if so
func (s *Service) valid() bool {
	return len(s.Name) > 0
}

func (s *Service) unmarshal(msg *dns.Msg) {
	for i := range msg.Answer {
		switch rr := msg.Answer[i].(type) {
		case *dns.RR_SRV:
			x := strings.Split(rr.Hdr.Name, ".")
			s.Name = x[0]
			s.Type = strings.Join(x[1:3], ".")
			s.Domain = strings.Join(x[3:], ".")
		}
	}
}
<commit_msg>Parse additional TXT records into the *Service<commit_after>package zeroconf

import (
	"log"
	"strings"
	"net"

	dns "github.com/miekg/godns"
)

type Service struct {
        Type      string
        Name      string
        Domain    string
        Interface *net.Interface
        Host   	Host
	Additional	[]string
}

type Host struct {
	Name	string
	Port	uint16
}

// s.unmarshal may not be complete, return false if so
func (s *Service) valid() bool {
	return len(s.Name) > 0
}

func (s *Service) unmarshal(msg *dns.Msg) {
	for i := range msg.Answer {
		switch rr := msg.Answer[i].(type) {
		case *dns.RR_SRV:
			x := strings.Split(rr.Hdr.Name, ".")
			s.Name = x[0]
			s.Type = strings.Join(x[1:3], ".")
			s.Domain = strings.Join(x[3:], ".")
			s.Host.Name = rr.Target
			s.Host.Port = rr.Port

		case *dns.RR_TXT:
			s.Additional = append(s.Additional, strings.Split(rr.Txt, ",")...)
		default:
			log.Printf("%#v", rr)
		}
	}
}
<|endoftext|>


<commit_before>package elastic_journald

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os/exec"
	"strings"
	"time"
	"unsafe"

	"github.com/liquidm/elastigo/lib"
)

// #include <stdio.h>
// #include <string.h>
// #include <systemd/sd-journal.h>
// #cgo LDFLAGS: -lsystemd
import "C"

type Config struct {
	Hosts       elasticHostsType
	IndexPrefix string
}

type Service struct {
	Config  *Config
	Journal *C.sd_journal
	Cursor  string
	Elastic *elastigo.Conn
	Indexer *elastigo.BulkIndexer
}

func NewService() *Service {
	config := &Config{
		Hosts:       elasticHosts,
		IndexPrefix: *elasticPrefix,
	}

	elastic := elastigo.NewConn()
	indexer := elastic.NewBulkIndexerErrors(2, 30)
	indexer.BufferDelayMax = time.Duration(30) * time.Second
	indexer.BulkMaxDocs = 1000
	indexer.BulkMaxBuffer = 65536
	indexer.Sender = func(buf *bytes.Buffer) error {
		respJson, err := elastic.DoCommand("POST", "/_bulk", nil, buf)
		if err != nil {
			// TODO
			panic(fmt.Sprintf("Bulk error: \n%v", err))
		} else {
			response := struct {
				Took   int64 `json:"took"`
				Errors bool  `json:"errors"`
				Items  []struct {
					Index struct {
						Id string `json:"_id"`
					} `json:"index"`
				} `json:"items"`
			}{}

			jsonErr := json.Unmarshal(respJson, &response)
			if jsonErr != nil {
				// TODO
				panic(jsonErr)
			}
			if response.Errors {
				// TODO
				fmt.Println(string(respJson))
				panic("elasticsearch reported errors on intake")
			}

			messagesStored := len(response.Items)
			lastStoredCursor := response.Items[messagesStored-1].Index.Id
			ioutil.WriteFile(*elasticCursorFile, []byte(lastStoredCursor), 0644)

			// this will cause a busy loop, don't do it
			// fmt.Printf("Sent %v entries\n", messagesStored)
		}
		return err
	}

	service := &Service{
		Config:  config,
		Elastic: elastic,
		Indexer: indexer,
	}
	return service
}

func (s *Service) Run() {
	s.Elastic.SetHosts(s.Config.Hosts)

	s.InitJournal()
	s.ProcessStream(GetFQDN())
}

func (s *Service) ProcessStream(hostname *string) {
	s.Indexer.Start()
	defer s.Indexer.Stop()

	for {
		r := C.sd_journal_next(s.Journal)
		if r < 0 {
			panic(fmt.Sprintf("failed to iterate to next entry: %s", C.strerror(-r)))
		}
		if r == 0 {
			r = C.sd_journal_wait(s.Journal, 1000000)
			if r < 0 {
				panic(fmt.Sprintf("failed to wait for changes: %s", C.strerror(-r)))
			}
			continue
		}
		s.ProcessEntry(hostname)
	}
}

func (s *Service) ProcessEntry(hostname *string) {
	var realtime C.uint64_t
	r := C.sd_journal_get_realtime_usec(s.Journal, &realtime)
	if r < 0 {
		panic(fmt.Sprintf("failed to get realtime timestamp: %s", C.strerror(-r)))
	}

	var cursor *C.char
	r = C.sd_journal_get_cursor(s.Journal, &cursor)
	if r < 0 {
		panic(fmt.Sprintf("failed to get cursor: %s", C.strerror(-r)))
	}

	row := make(map[string]interface{})

	timestamp := time.Unix(int64(realtime/1000000), int64(realtime%1000000)).UTC()

	row["ts"] = timestamp.Format("2006-01-02T15:04:05Z")
	row["host"] = hostname
	s.ProcessEntryFields(row)

	message, _ := json.Marshal(row)
	indexName := fmt.Sprintf("%v-%v", s.Config.IndexPrefix, timestamp.Format("2006-01-02"))
	cursorId := C.GoString(cursor)

	s.Indexer.Index(
		indexName,       // index
		"journal",       // type
		cursorId,        // id
		"",              // parent
		"",              // ttl
		nil,             // date
		string(message), // content
	)
}

func (s *Service) ProcessEntryFields(row map[string]interface{}) {
	var length C.size_t
	var cData *C.char

	for C.sd_journal_restart_data(s.Journal); C.sd_journal_enumerate_data(s.Journal, (*unsafe.Pointer)(unsafe.Pointer(&cData)), &length) > 0; {
		data := C.GoString(cData)

		parts := strings.SplitN(data, "=", 2)

		key := strings.ToLower(parts[0])
		value := parts[1]

		switch key {
		// don't index bloat
		case "_cap_effective":
		case "_cmdline":
		case "_exe":
		case "_hostname":
		case "_systemd_cgroup":
		case "_systemd_slice":
		case "_transport":
		case "syslog_facility":
		case "syslog_identifier":
			continue
		default:
			row[strings.TrimPrefix(key, "_")] = value
		}
	}
}

func (s *Service) InitJournal() {
	r := C.sd_journal_open(&s.Journal, C.SD_JOURNAL_LOCAL_ONLY)
	if r < 0 {
		panic(fmt.Sprintf("failed to open journal: %s", C.strerror(-r)))
	}

	bytes, err := ioutil.ReadFile(*elasticCursorFile)
	if err == nil {
		s.Cursor = string(bytes)
	}

	if s.Cursor != "" {
		r = C.sd_journal_seek_cursor(s.Journal, C.CString(s.Cursor))
		if r < 0 {
			panic(fmt.Sprintf("failed to seek journal: %s", C.strerror(-r)))
		}
		r = C.sd_journal_next_skip(s.Journal, 1)
		if r < 0 {
			panic(fmt.Sprintf("failed to skip current journal entry: %s", C.strerror(-r)))
		}
	}
}

func GetFQDN() *string {
	cmd := exec.Command("hostname", "-f")
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		return nil
	}
	fqdn := string(bytes.TrimSpace(out.Bytes()))
	return &fqdn
}

type elasticHostsType []string

func (e *elasticHostsType) String() string {
	return strings.Join(*e, ",")
}

func (e *elasticHostsType) Set(value string) error {
	for _, host := range strings.Split(value, ",") {
		*e = append(*e, host)
	}
	return nil
}

var elasticCursorFile = flag.String("cursor", ".elastic_journal_cursor", "The file to keep cursor state between runs")
var elasticHosts elasticHostsType
var elasticPrefix = flag.String("prefix", "journald", "The index prefix to use")

func init() {
	flag.Var(&elasticHosts, "hosts", "comma-separated list of elastic (target) hosts")
}
<commit_msg>Extract fields from messages, take 2<commit_after>package elastic_journald

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os/exec"
	"regexp"
	"strings"
	"time"
	"unsafe"

	"github.com/liquidm/elastigo/lib"
)

// #include <stdio.h>
// #include <string.h>
// #include <systemd/sd-journal.h>
// #cgo LDFLAGS: -lsystemd
import "C"

type Config struct {
	Hosts       elasticHostsType
	IndexPrefix string
}

type Service struct {
	Config  *Config
	Journal *C.sd_journal
	Cursor  string
	Elastic *elastigo.Conn
	Indexer *elastigo.BulkIndexer

	messageFieldsRe *regexp.Regexp
}

func NewService() *Service {
	config := &Config{
		Hosts:       elasticHosts,
		IndexPrefix: *elasticPrefix,
	}

	elastic := elastigo.NewConn()
	indexer := elastic.NewBulkIndexerErrors(2, 30)
	indexer.BufferDelayMax = time.Duration(30) * time.Second
	indexer.BulkMaxDocs = 1000
	indexer.BulkMaxBuffer = 65536
	indexer.Sender = func(buf *bytes.Buffer) error {
		respJson, err := elastic.DoCommand("POST", "/_bulk", nil, buf)
		if err != nil {
			// TODO
			panic(fmt.Sprintf("Bulk error: \n%v", err))
		} else {
			response := struct {
				Took   int64 `json:"took"`
				Errors bool  `json:"errors"`
				Items  []struct {
					Index struct {
						Id string `json:"_id"`
					} `json:"index"`
				} `json:"items"`
			}{}

			jsonErr := json.Unmarshal(respJson, &response)
			if jsonErr != nil {
				// TODO
				panic(jsonErr)
			}
			if response.Errors {
				// TODO
				fmt.Println(string(respJson))
				panic("elasticsearch reported errors on intake")
			}

			messagesStored := len(response.Items)
			lastStoredCursor := response.Items[messagesStored-1].Index.Id
			ioutil.WriteFile(*elasticCursorFile, []byte(lastStoredCursor), 0644)

			// this will cause a busy loop, don't do it
			// fmt.Printf("Sent %v entries\n", messagesStored)
		}
		return err
	}

	service := &Service{
		Config:  config,
		Elastic: elastic,
		Indexer: indexer,

		messageFieldsRe: regexp.MustCompile("indexed:([[:graph:]]+)=([[:graph:]]+)"),
	}
	return service
}

func (s *Service) Run() {
	s.Elastic.SetHosts(s.Config.Hosts)

	s.InitJournal()
	s.ProcessStream(GetFQDN())
}

func (s *Service) ProcessStream(hostname *string) {
	s.Indexer.Start()
	defer s.Indexer.Stop()

	for {
		r := C.sd_journal_next(s.Journal)
		if r < 0 {
			panic(fmt.Sprintf("failed to iterate to next entry: %s", C.strerror(-r)))
		}
		if r == 0 {
			r = C.sd_journal_wait(s.Journal, 1000000)
			if r < 0 {
				panic(fmt.Sprintf("failed to wait for changes: %s", C.strerror(-r)))
			}
			continue
		}
		s.ProcessEntry(hostname)
	}
}

func (s *Service) ProcessEntry(hostname *string) {
	var realtime C.uint64_t
	r := C.sd_journal_get_realtime_usec(s.Journal, &realtime)
	if r < 0 {
		panic(fmt.Sprintf("failed to get realtime timestamp: %s", C.strerror(-r)))
	}

	var cursor *C.char
	r = C.sd_journal_get_cursor(s.Journal, &cursor)
	if r < 0 {
		panic(fmt.Sprintf("failed to get cursor: %s", C.strerror(-r)))
	}

	row := make(map[string]interface{})

	timestamp := time.Unix(int64(realtime/1000000), int64(realtime%1000000)).UTC()

	row["ts"] = timestamp.Format("2006-01-02T15:04:05Z")
	row["host"] = hostname
	s.ProcessEntryFields(row)

	message, _ := json.Marshal(row)
	indexName := fmt.Sprintf("%v-%v", s.Config.IndexPrefix, timestamp.Format("2006-01-02"))
	cursorId := C.GoString(cursor)

	s.Indexer.Index(
		indexName,       // index
		"journal",       // type
		cursorId,        // id
		"",              // parent
		"",              // ttl
		nil,             // date
		string(message), // content
	)
}

func (s *Service) ProcessEntryFields(row map[string]interface{}) {
	var length C.size_t
	var cData *C.char

	for C.sd_journal_restart_data(s.Journal); C.sd_journal_enumerate_data(s.Journal, (*unsafe.Pointer)(unsafe.Pointer(&cData)), &length) > 0; {
		data := C.GoString(cData)

		parts := strings.SplitN(data, "=", 2)

		key := strings.ToLower(parts[0])
		value := parts[1]

		switch key {
		// don't index bloat
		case "_cap_effective":
		case "_cmdline":
		case "_exe":
		case "_hostname":
		case "_systemd_cgroup":
		case "_systemd_slice":
		case "_transport":
		case "syslog_facility":
		case "syslog_identifier":
			continue
		case "message":
			row[key] = value
			match := s.messageFieldsRe.FindAllStringSubmatch(value, -1)
			for _, m := range match {
				row[m[1]] = m[2]
			}
		default:
			row[strings.TrimPrefix(key, "_")] = value
		}
	}
}

func (s *Service) InitJournal() {
	r := C.sd_journal_open(&s.Journal, C.SD_JOURNAL_LOCAL_ONLY)
	if r < 0 {
		panic(fmt.Sprintf("failed to open journal: %s", C.strerror(-r)))
	}

	bytes, err := ioutil.ReadFile(*elasticCursorFile)
	if err == nil {
		s.Cursor = string(bytes)
	}

	if s.Cursor != "" {
		r = C.sd_journal_seek_cursor(s.Journal, C.CString(s.Cursor))
		if r < 0 {
			panic(fmt.Sprintf("failed to seek journal: %s", C.strerror(-r)))
		}
		r = C.sd_journal_next_skip(s.Journal, 1)
		if r < 0 {
			panic(fmt.Sprintf("failed to skip current journal entry: %s", C.strerror(-r)))
		}
	}
}

func GetFQDN() *string {
	cmd := exec.Command("hostname", "-f")
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		return nil
	}
	fqdn := string(bytes.TrimSpace(out.Bytes()))
	return &fqdn
}

type elasticHostsType []string

func (e *elasticHostsType) String() string {
	return strings.Join(*e, ",")
}

func (e *elasticHostsType) Set(value string) error {
	for _, host := range strings.Split(value, ",") {
		*e = append(*e, host)
	}
	return nil
}

var elasticCursorFile = flag.String("cursor", ".elastic_journal_cursor", "The file to keep cursor state between runs")
var elasticHosts elasticHostsType
var elasticPrefix = flag.String("prefix", "journald", "The index prefix to use")

func init() {
	flag.Var(&elasticHosts, "hosts", "comma-separated list of elastic (target) hosts")
}
<|endoftext|>


<commit_before>package elastic_journald

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os/exec"
	"strings"
	"time"
	"unsafe"

	"github.com/mattbaird/elastigo/lib"
)

// #include <stdio.h>
// #include <string.h>
// #include <systemd/sd-journal.h>
// #cgo LDFLAGS: -lsystemd
import "C"

type Config struct {
	Hosts       elasticHostsType
	IndexPrefix string
}

type Service struct {
	Config  *Config
	Journal *C.sd_journal
	Cursor  string
	Elastic *elastigo.Conn
	Indexer *elastigo.BulkIndexer
}

func NewService() *Service {
	config := &Config{
		Hosts:       elasticHosts,
		IndexPrefix: *elasticPrefix,
	}

	elastic := elastigo.NewConn()
	indexer := elastic.NewBulkIndexerErrors(2, 30)
	indexer.BufferDelayMax = time.Duration(30) * time.Second
	indexer.Sender = func(buf *bytes.Buffer) error {
		respJson, err := elastic.DoCommand("POST", "/_bulk", nil, buf)
		if err != nil {
			// TODO
			panic(fmt.Sprintf("Bulk error: \n%v", err))
		} else {
			response := struct {
				Took   int64 `json:"took"`
				Errors bool  `json:"errors"`
				Items  []struct {
					Index struct {
						Id string `json:"_id"`
					} `json:"index"`
				} `json:"items"`
			}{}

			jsonErr := json.Unmarshal(respJson, &response)
			if jsonErr != nil {
				// TODO
				panic(jsonErr)
			}
			if response.Errors {
				// TODO
				fmt.Println(respJson)
				panic("elasticsearch reported errors on intake")
			}

			messagesStored := len(response.Items)
			lastStoredCursor := response.Items[messagesStored-1].Index.Id
			ioutil.WriteFile(*elasticCursorFile, []byte(lastStoredCursor), 0644)

			// this will cause a busy loop, don't do it
			// fmt.Printf("Sent %v entries\n", messagesStored)
		}
		return err
	}

	service := &Service{
		Config:  config,
		Elastic: elastic,
		Indexer: indexer,
	}
	return service
}

func (s *Service) Run() {
	s.Elastic.SetHosts(s.Config.Hosts)

	s.InitJournal()
	s.ProcessStream(GetFQDN())
}

func (s *Service) ProcessStream(hostname *string) {
	s.Indexer.Start()
	defer s.Indexer.Stop()

	for {
		r := C.sd_journal_next(s.Journal)
		if r < 0 {
			panic(fmt.Sprintf("failed to iterate to next entry: %s", C.strerror(-r)))
		}
		if r == 0 {
			r = C.sd_journal_wait(s.Journal, 1000000)
			if r < 0 {
				panic(fmt.Sprintf("failed to wait for changes: %s", C.strerror(-r)))
			}
			continue
		}
		s.ProcessEntry(hostname)
	}
}

func (s *Service) ProcessEntry(hostname *string) {
	var realtime C.uint64_t
	r := C.sd_journal_get_realtime_usec(s.Journal, &realtime)
	if r < 0 {
		panic(fmt.Sprintf("failed to get realtime timestamp: %s", C.strerror(-r)))
	}

	var cursor *C.char
	r = C.sd_journal_get_cursor(s.Journal, &cursor)
	if r < 0 {
		panic(fmt.Sprintf("failed to get cursor: %s", C.strerror(-r)))
	}

	row := make(map[string]interface{})

	timestamp := time.Unix(int64(realtime/1000000), int64(realtime%1000000)).UTC()

	row["ts"] = timestamp.Format("2006-01-02T15:04:05Z")
	row["host"] = hostname
	s.ProcessEntryFields(row)

	message, _ := json.Marshal(row)
	indexName := fmt.Sprintf("%v-%v", s.Config.IndexPrefix, timestamp.Format("2006-01-02"))
	cursorId := C.GoString(cursor)

	s.Indexer.Index(
		indexName,       // index
		"journal",       // type
		cursorId,        // id
		"",              // ttl
		nil,             // date
		string(message), // content
		false)           // immediate index refresh
}

func (s *Service) ProcessEntryFields(row map[string]interface{}) {
	var length C.size_t
	var cData *C.char

	for C.sd_journal_restart_data(s.Journal); C.sd_journal_enumerate_data(s.Journal, (*unsafe.Pointer)(unsafe.Pointer(&cData)), &length) > 0; {
		data := C.GoString(cData)

		parts := strings.SplitN(data, "=", 2)

		key := strings.ToLower(parts[0])
		value := parts[1]

		switch key {
		// don't index bloat
		case "_cap_effective":
		case "_cmdline":
		case "_exe":
		case "_hostname":
		case "_systemd_cgroup":
		case "_systemd_slice":
		case "_transport":
		case "syslog_facility":
		case "syslog_identifier":
			continue
		default:
			row[strings.TrimPrefix(key, "_")] = value
		}
	}
}

func (s *Service) InitJournal() {
	r := C.sd_journal_open(&s.Journal, C.SD_JOURNAL_LOCAL_ONLY)
	if r < 0 {
		panic(fmt.Sprintf("failed to open journal: %s", C.strerror(-r)))
	}

	bytes, err := ioutil.ReadFile(*elasticCursorFile)
	if err == nil {
		s.Cursor = string(bytes)
	}

	if s.Cursor != "" {
		r = C.sd_journal_seek_cursor(s.Journal, C.CString(s.Cursor))
		if r < 0 {
			panic(fmt.Sprintf("failed to seek journal: %s", C.strerror(-r)))
		}
		r = C.sd_journal_next_skip(s.Journal, 1)
		if r < 0 {
			panic(fmt.Sprintf("failed to skip current journal entry: %s", C.strerror(-r)))
		}
	}
}

func GetFQDN() *string {
	cmd := exec.Command("hostname", "-f")
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		return nil
	}
	fqdn := string(bytes.TrimSpace(out.Bytes()))
	return &fqdn
}

type elasticHostsType []string

func (e *elasticHostsType) String() string {
	return strings.Join(*e, ",")
}

func (e *elasticHostsType) Set(value string) error {
	for _, host := range strings.Split(value, ",") {
		*e = append(*e, host)
	}
	return nil
}

var elasticCursorFile = flag.String("cursor", ".elastic_journal_cursor", "The file to keep cursor state between runs")
var elasticHosts elasticHostsType
var elasticPrefix = flag.String("prefix", "journald", "The index prefix to use")

func init() {
	flag.Var(&elasticHosts, "hosts", "comma-separated list of elastic (target) hosts")
}
<commit_msg>increase bulk buffering some more<commit_after>package elastic_journald

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os/exec"
	"strings"
	"time"
	"unsafe"

	"github.com/mattbaird/elastigo/lib"
)

// #include <stdio.h>
// #include <string.h>
// #include <systemd/sd-journal.h>
// #cgo LDFLAGS: -lsystemd
import "C"

type Config struct {
	Hosts       elasticHostsType
	IndexPrefix string
}

type Service struct {
	Config  *Config
	Journal *C.sd_journal
	Cursor  string
	Elastic *elastigo.Conn
	Indexer *elastigo.BulkIndexer
}

func NewService() *Service {
	config := &Config{
		Hosts:       elasticHosts,
		IndexPrefix: *elasticPrefix,
	}

	elastic := elastigo.NewConn()
	indexer := elastic.NewBulkIndexerErrors(2, 30)
	indexer.BufferDelayMax = time.Duration(30) * time.Second
	indexer.BulkMaxDocs = 1000
	indexer.BulkMaxBuffer = 65536
	indexer.Sender = func(buf *bytes.Buffer) error {
		respJson, err := elastic.DoCommand("POST", "/_bulk", nil, buf)
		if err != nil {
			// TODO
			panic(fmt.Sprintf("Bulk error: \n%v", err))
		} else {
			response := struct {
				Took   int64 `json:"took"`
				Errors bool  `json:"errors"`
				Items  []struct {
					Index struct {
						Id string `json:"_id"`
					} `json:"index"`
				} `json:"items"`
			}{}

			jsonErr := json.Unmarshal(respJson, &response)
			if jsonErr != nil {
				// TODO
				panic(jsonErr)
			}
			if response.Errors {
				// TODO
				fmt.Println(respJson)
				panic("elasticsearch reported errors on intake")
			}

			messagesStored := len(response.Items)
			lastStoredCursor := response.Items[messagesStored-1].Index.Id
			ioutil.WriteFile(*elasticCursorFile, []byte(lastStoredCursor), 0644)

			// this will cause a busy loop, don't do it
			// fmt.Printf("Sent %v entries\n", messagesStored)
		}
		return err
	}

	service := &Service{
		Config:  config,
		Elastic: elastic,
		Indexer: indexer,
	}
	return service
}

func (s *Service) Run() {
	s.Elastic.SetHosts(s.Config.Hosts)

	s.InitJournal()
	s.ProcessStream(GetFQDN())
}

func (s *Service) ProcessStream(hostname *string) {
	s.Indexer.Start()
	defer s.Indexer.Stop()

	for {
		r := C.sd_journal_next(s.Journal)
		if r < 0 {
			panic(fmt.Sprintf("failed to iterate to next entry: %s", C.strerror(-r)))
		}
		if r == 0 {
			r = C.sd_journal_wait(s.Journal, 1000000)
			if r < 0 {
				panic(fmt.Sprintf("failed to wait for changes: %s", C.strerror(-r)))
			}
			continue
		}
		s.ProcessEntry(hostname)
	}
}

func (s *Service) ProcessEntry(hostname *string) {
	var realtime C.uint64_t
	r := C.sd_journal_get_realtime_usec(s.Journal, &realtime)
	if r < 0 {
		panic(fmt.Sprintf("failed to get realtime timestamp: %s", C.strerror(-r)))
	}

	var cursor *C.char
	r = C.sd_journal_get_cursor(s.Journal, &cursor)
	if r < 0 {
		panic(fmt.Sprintf("failed to get cursor: %s", C.strerror(-r)))
	}

	row := make(map[string]interface{})

	timestamp := time.Unix(int64(realtime/1000000), int64(realtime%1000000)).UTC()

	row["ts"] = timestamp.Format("2006-01-02T15:04:05Z")
	row["host"] = hostname
	s.ProcessEntryFields(row)

	message, _ := json.Marshal(row)
	indexName := fmt.Sprintf("%v-%v", s.Config.IndexPrefix, timestamp.Format("2006-01-02"))
	cursorId := C.GoString(cursor)

	s.Indexer.Index(
		indexName,       // index
		"journal",       // type
		cursorId,        // id
		"",              // ttl
		nil,             // date
		string(message), // content
		false)           // immediate index refresh
}

func (s *Service) ProcessEntryFields(row map[string]interface{}) {
	var length C.size_t
	var cData *C.char

	for C.sd_journal_restart_data(s.Journal); C.sd_journal_enumerate_data(s.Journal, (*unsafe.Pointer)(unsafe.Pointer(&cData)), &length) > 0; {
		data := C.GoString(cData)

		parts := strings.SplitN(data, "=", 2)

		key := strings.ToLower(parts[0])
		value := parts[1]

		switch key {
		// don't index bloat
		case "_cap_effective":
		case "_cmdline":
		case "_exe":
		case "_hostname":
		case "_systemd_cgroup":
		case "_systemd_slice":
		case "_transport":
		case "syslog_facility":
		case "syslog_identifier":
			continue
		default:
			row[strings.TrimPrefix(key, "_")] = value
		}
	}
}

func (s *Service) InitJournal() {
	r := C.sd_journal_open(&s.Journal, C.SD_JOURNAL_LOCAL_ONLY)
	if r < 0 {
		panic(fmt.Sprintf("failed to open journal: %s", C.strerror(-r)))
	}

	bytes, err := ioutil.ReadFile(*elasticCursorFile)
	if err == nil {
		s.Cursor = string(bytes)
	}

	if s.Cursor != "" {
		r = C.sd_journal_seek_cursor(s.Journal, C.CString(s.Cursor))
		if r < 0 {
			panic(fmt.Sprintf("failed to seek journal: %s", C.strerror(-r)))
		}
		r = C.sd_journal_next_skip(s.Journal, 1)
		if r < 0 {
			panic(fmt.Sprintf("failed to skip current journal entry: %s", C.strerror(-r)))
		}
	}
}

func GetFQDN() *string {
	cmd := exec.Command("hostname", "-f")
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		return nil
	}
	fqdn := string(bytes.TrimSpace(out.Bytes()))
	return &fqdn
}

type elasticHostsType []string

func (e *elasticHostsType) String() string {
	return strings.Join(*e, ",")
}

func (e *elasticHostsType) Set(value string) error {
	for _, host := range strings.Split(value, ",") {
		*e = append(*e, host)
	}
	return nil
}

var elasticCursorFile = flag.String("cursor", ".elastic_journal_cursor", "The file to keep cursor state between runs")
var elasticHosts elasticHostsType
var elasticPrefix = flag.String("prefix", "journald", "The index prefix to use")

func init() {
	flag.Var(&elasticHosts, "hosts", "comma-separated list of elastic (target) hosts")
}
<|endoftext|>


<commit_before>package enroll

import "io/ioutil"

type Service interface {
	Enroll() (Profile, error)
}

func NewService(pushCertPath string, pushCertPass string, caCertPath string) (Service, error) {
	pushTopic, err := GetPushTopicFromPKCS12(pushCertPath, pushCertPass)
	if err != nil {
		return nil, err
	}

	var caCert []byte

	if caCertPath != "" {
		caCert, err = ioutil.ReadFile(caCertPath)

		if err != nil {
			return nil, err
		}
	}

	scepSubject := [][][]string{
		[][]string{
			[]string{"O", "MicroMDM"},
			[]string{"CN", "MDM Identity Certificate:UDID"},
		},
	}

	return &service{
		Url:         "https://micromdm.local:6443",
		SCEPUrl:     "http://micromdm.local:2019/scep",
		SCEPSubject: scepSubject,
		Topic:       pushTopic,
		CACert:      caCert,
	}, nil
}

type service struct {
	Url           string
	SCEPUrl       string
	SCEPChallenge string
	SCEPSubject   [][][]string
	Topic         string // APNS Topic for MDM notifications
	CACert        []byte
}

func (svc service) Enroll() (Profile, error) {
	profile := NewProfile()
	profile.PayloadIdentifier = "com.github.micromdm.micromdm.mdm"
	profile.PayloadOrganization = "MicroMDM"
	profile.PayloadDisplayName = "Enrollment Profile"
	profile.PayloadDescription = "The server may alter your settings"

	scepContent := SCEPPayloadContent{
		Challenge: svc.SCEPChallenge,
		URL:       svc.SCEPUrl,
		Keysize:   1024,
		KeyType:   "RSA",
		KeyUsage:  0,
		Name:      "Device Management Identity Certificate",
		Subject:   svc.SCEPSubject,
	}

	scepPayload := NewPayload("com.apple.security.scep")
	scepPayload.PayloadDescription = "Configures SCEP"
	scepPayload.PayloadDisplayName = "SCEP"
	scepPayload.PayloadIdentifier = "com.github.micromdm.scep"
	scepPayload.PayloadContent = scepContent

	mdmPayload := NewPayload("com.apple.mdm")
	mdmPayload.PayloadDescription = "Enrolls with the MDM server"
	mdmPayload.PayloadOrganization = "MicroMDM"
	mdmPayload.PayloadIdentifier = "com.github.micromdm.mdm"

	mdmPayloadContent := MDMPayloadContent{
		Payload:                 *mdmPayload,
		AccessRights:            8191,
		CheckInURL:              svc.Url + "/mdm/checkin",
		CheckOutWhenRemoved:     true,
		ServerURL:               svc.Url + "/mdm/connect",
		IdentityCertificateUUID: scepPayload.PayloadUUID,
		Topic: svc.Topic,
	}

	if len(svc.CACert) > 0 {
		caPayload := NewPayload("com.apple.ssl.certificate")
		caPayload.PayloadDisplayName = "Root certificate for MicroMDM"
		caPayload.PayloadDescription = "Installs the root CA certificate for MicroMDM"
		caPayload.PayloadContent = svc.CACert

		profile.PayloadContent = []interface{}{*scepPayload, mdmPayloadContent, *caPayload}
	} else {
		profile.PayloadContent = []interface{}{*scepPayload, mdmPayloadContent}
	}

	return *profile, nil
}
<commit_msg>Make public facing URL and SCEP server URL configurable. Degrade gracefully to disabling enrollment endpoint if either URL or SCEP url are not provided.<commit_after>package enroll

import "io/ioutil"

type Service interface {
	Enroll() (Profile, error)
}

func NewService(pushCertPath string, pushCertPass string, caCertPath string, url string, scepUrl string) (Service, error) {
	pushTopic, err := GetPushTopicFromPKCS12(pushCertPath, pushCertPass)
	if err != nil {
		return nil, err
	}

	var caCert []byte

	if caCertPath != "" {
		caCert, err = ioutil.ReadFile(caCertPath)

		if err != nil {
			return nil, err
		}
	}

	scepSubject := [][][]string{
		[][]string{
			[]string{"O", "MicroMDM"},
			[]string{"CN", "MDM Identity Certificate:UDID"},
		},
	}

	return &service{
		Url:         url,
		SCEPUrl:     scepUrl,
		SCEPSubject: scepSubject,
		Topic:       pushTopic,
		CACert:      caCert,
	}, nil
}

type service struct {
	Url           string
	SCEPUrl       string
	SCEPChallenge string
	SCEPSubject   [][][]string
	Topic         string // APNS Topic for MDM notifications
	CACert        []byte
}

func (svc service) Enroll() (Profile, error) {
	profile := NewProfile()
	profile.PayloadIdentifier = "com.github.micromdm.micromdm.mdm"
	profile.PayloadOrganization = "MicroMDM"
	profile.PayloadDisplayName = "Enrollment Profile"
	profile.PayloadDescription = "The server may alter your settings"

	scepContent := SCEPPayloadContent{
		Challenge: svc.SCEPChallenge,
		URL:       svc.SCEPUrl,
		Keysize:   1024,
		KeyType:   "RSA",
		KeyUsage:  0,
		Name:      "Device Management Identity Certificate",
		Subject:   svc.SCEPSubject,
	}

	scepPayload := NewPayload("com.apple.security.scep")
	scepPayload.PayloadDescription = "Configures SCEP"
	scepPayload.PayloadDisplayName = "SCEP"
	scepPayload.PayloadIdentifier = "com.github.micromdm.scep"
	scepPayload.PayloadContent = scepContent

	mdmPayload := NewPayload("com.apple.mdm")
	mdmPayload.PayloadDescription = "Enrolls with the MDM server"
	mdmPayload.PayloadOrganization = "MicroMDM"
	mdmPayload.PayloadIdentifier = "com.github.micromdm.mdm"

	mdmPayloadContent := MDMPayloadContent{
		Payload:                 *mdmPayload,
		AccessRights:            8191,
		CheckInURL:              svc.Url + "/mdm/checkin",
		CheckOutWhenRemoved:     true,
		ServerURL:               svc.Url + "/mdm/connect",
		IdentityCertificateUUID: scepPayload.PayloadUUID,
		Topic: svc.Topic,
	}

	if len(svc.CACert) > 0 {
		caPayload := NewPayload("com.apple.ssl.certificate")
		caPayload.PayloadDisplayName = "Root certificate for MicroMDM"
		caPayload.PayloadDescription = "Installs the root CA certificate for MicroMDM"
		caPayload.PayloadContent = svc.CACert

		profile.PayloadContent = []interface{}{*scepPayload, mdmPayloadContent, *caPayload}
	} else {
		profile.PayloadContent = []interface{}{*scepPayload, mdmPayloadContent}
	}

	return *profile, nil
}
<|endoftext|>