<commit_before>// Package gonn is a port from this: http://inkdrop.net/dave/docs/neural-net-tutorial.cpp
package gonn

import (
	"fmt"
	"log"
	"math"
	"math/rand"
)

var (
	// Eta [0.0..1.0] overall net training rate
	Eta = 0.15
	// Alpha [0.0..1.0] multiplier of last weight chagne (momentum)
	Alpha = 0.5
)

func randomWeight() float64 {
	return rand.Float64()
}

func transferFunction(x float64) float64 {
	// tanh - output range [-1.0..1.0]
	//return 1.0 / (1.0 + math.Exp(-x))
	return math.Tanh(x)
}

func transferFunctionDerivative(x float64) float64 {
	// tanh derivative
	// not the actual formula
	//return 1.0
	return 1.0 - x*x
}

type neuronConnection struct {
	Weight      float64
	DeltaWeight float64
}

// Neuron object
type Neuron struct {
	outputVal     float64
	outputWeights []neuronConnection
	myIndex       int
	gradient      float64

	//eta   float64 // [0.0..1.0] overall net training rate
	//Alpha float64 // [0.0..n] multiplier of last weight chagne (momentum)
}

// NewNeuron intializes new neuron object
func NewNeuron(numOutputs, myIndex int) *Neuron {
	n := new(Neuron)
	// c for connections
	for c := 0; c < numOutputs; c++ {
		n.outputWeights = append(n.outputWeights, *new(neuronConnection))
		n.outputWeights[len(n.outputWeights)-1].Weight = randomWeight()
	}
	n.myIndex = myIndex
	return n
}

// FeedForward does the math magic to its self
func (n *Neuron) FeedForward(prevLayer *Layer) {
	var sum float64

	// Sum the previous layer's outputs (which are our inputs)
	// Include the bias node from the previous layer

	for i := 0; i < len(*prevLayer); i++ {
		sum += (*prevLayer)[i].outputVal *
			(*prevLayer)[i].outputWeights[n.myIndex].Weight
	}

	n.outputVal = transferFunction(sum)
}

func (n Neuron) sumDOW(nextLayer *Layer) float64 {
	var sum float64

	for i := 0; i < len(*nextLayer)-1; i++ {
		sum += n.outputWeights[i].Weight * (*nextLayer)[i].gradient
	}

	return sum
}

func (n *Neuron) calculateOutputGradients(targetVal float64) {
	delta := targetVal - n.outputVal
	n.gradient = delta * transferFunctionDerivative(n.outputVal)
}

func (n *Neuron) calculateHiddenGradients(nextLayer *Layer) {
	dow := n.sumDOW(nextLayer)
	n.gradient = dow * transferFunctionDerivative(n.outputVal)
}

func (n *Neuron) updateInputWeights(prevLayer *Layer) {
	// The weights to be updated are in the Conneciton container
	// int the neurons in the preceding layer
	for i := 0; i < len(*prevLayer); i++ {
		neuron := &(*prevLayer)[i]
		oldDeltaWeight := neuron.outputWeights[n.myIndex].DeltaWeight

		newDeltaWeight :=
			// Individual input, magnified by the gradient and train rate:
			Eta*neuron.outputVal*n.gradient +
				// Also add momentun = a fraction of the previous delta wieght
				Alpha*oldDeltaWeight
		neuron.outputWeights[n.myIndex].DeltaWeight = newDeltaWeight
		neuron.outputWeights[n.myIndex].Weight += newDeltaWeight
	}
}

// Layer is just array of neurons
type Layer []Neuron

// NeuralNetwork holds all the data of the network
type NeuralNetwork struct {
	Layers                            []Layer // layers[layerNum][neuronNum]
	err                               float64
	recentAverageError                float64
	RecentAverageErrorSmoothingFactor float64
}

// NewNetwork initializes new network
func NewNetwork(topology []int) *NeuralNetwork {
	n := new(NeuralNetwork)
	// Number of training smaples to average over
	//n.RecentAverageErrorSmoothingFactor = 112.0

	for layerNum := 0; layerNum < len(topology); layerNum++ {
		n.Layers = append(n.Layers, *new(Layer))
		var numOutputs int
		if layerNum == len(topology)-1 {
			numOutputs = 0
		} else {
			numOutputs = topology[layerNum+1]
		}

		// We have made new layer, now fill in its neurons
		// and a bias neuron
		for neuronNum := 0; neuronNum <= topology[layerNum]; neuronNum++ {
			n.Layers[len(n.Layers)-1] =
				append(n.Layers[len(n.Layers)-1],
					*NewNeuron(numOutputs, neuronNum))
		}

		// Force the bias node's output value to 1.0. It's the last neuron
		// created above
		layer := &n.Layers[layerNum]
		(*layer)[len(*layer)-1].outputVal = 1.0
	}
	return n
}

// FeedForward takes inputs
func (n *NeuralNetwork) FeedForward(inputVals []float64) {
	// Ignore bias
	if len(inputVals) > len(n.Layers[0])-1 {
		log.Fatalf("Length if inputsVals must be the same as length of"+
			" the first layer (%d != %d)", len(inputVals), len(n.Layers[0]))
	}

	// assign the input values into the input neurons
	for i := 0; i < len(inputVals); i++ {
		n.Layers[0][i].outputVal = inputVals[i]
	}

	// Forward propagate
	for layerNum := 1; layerNum < len(n.Layers); layerNum++ {
		prevLayer := &n.Layers[layerNum-1]
		for i := 0; i < len(n.Layers[layerNum])-1; i++ {
			n.Layers[layerNum][i].FeedForward(prevLayer)
		}
	}
}

// BackProp does the backpropagation (this is where the net learns)
func (n *NeuralNetwork) BackProp(targetVals []float64) {
	// Calculate overall net error (RMS of output errors)
	// RMS = "Root Mean Square Error"
	outputLayer := &n.Layers[len(n.Layers)-1]
	n.err = 0.0

	for i := 0; i < len(*outputLayer)-1; i++ {
		delta := targetVals[i] - (*outputLayer)[i].outputVal
		n.err += delta * delta
	}
	n.err /= float64(len(*outputLayer)) - 1.0 // get average error squared
	n.err = math.Sqrt(n.err)                  // RMS

	// Implements a recent average measurement
	n.recentAverageError =
		(n.recentAverageError*n.RecentAverageErrorSmoothingFactor + n.err) /
			(n.RecentAverageErrorSmoothingFactor + 1.0)

	// Calculate output layer gradiants
	for i := 0; i < len(*outputLayer)-1; i++ {
		(*outputLayer)[i].calculateOutputGradients(targetVals[i])
	}

	// Calculate gradients on hidden layers
	for layerNum := len(n.Layers) - 2; layerNum > 0; layerNum-- {
		hiddenLayer := &n.Layers[layerNum]
		nextLayer := &n.Layers[layerNum+1]

		for i := 0; i < len(*hiddenLayer); i++ {
			(*hiddenLayer)[i].calculateHiddenGradients(nextLayer)
		}
	}

	// For all layers from outputs to first hidden layer,
	// update connection weights

	for layerNum := len(n.Layers) - 1; layerNum > 0; layerNum-- {
		layer := &n.Layers[layerNum]
		prevLayer := &n.Layers[layerNum-1]

		for i := 0; i < len(*layer)-1; i++ {
			(*layer)[i].updateInputWeights(prevLayer)
		}
	}
}

// GetAverageError return recentAvarageError value
func (n *NeuralNetwork) GetAverageError() float64 {
	return n.recentAverageError
}

// GetResults returns results from all output neurons as a string
func (n *NeuralNetwork) GetResults() string {
	var out string
	for i, outputNeuron := range n.Layers[len(n.Layers)-1] {
		if i == len(n.Layers[len(n.Layers)-1])-1 {
			// Ignore bias
			continue
		}
		out += fmt.Sprintf("%f ", outputNeuron.outputVal)
	}
	return out
}
<commit_msg>New derivative function<commit_after>// Package gonn is a port from this: http://inkdrop.net/dave/docs/neural-net-tutorial.cpp
package gonn

import (
	"fmt"
	"log"
	"math"
	"math/rand"
)

var (
	// Eta [0.0..1.0] overall net training rate
	Eta = 0.15
	// Alpha [0.0..1.0] multiplier of last weight chagne (momentum)
	Alpha = 0.5
)

func randomWeight() float64 {
	return rand.Float64()
}

func transferFunction(x float64) float64 {
	// tanh - output range [-1.0..1.0]
	//return 1.0 / (1.0 + math.Exp(-x))
	return math.Tanh(x)
}

func transferFunctionDerivative(x float64) float64 {
	// tanh derivative
	// not the actual formula
	//return 1.0
	return (1.0 / (1.0 + math.Exp(-x))) - (1.0+math.Exp(-x))*(1.0+math.Exp(-x))
	//return 1.0 - x*x
}

type neuronConnection struct {
	Weight      float64
	DeltaWeight float64
}

// Neuron object
type Neuron struct {
	outputVal     float64
	outputWeights []neuronConnection
	myIndex       int
	gradient      float64

	//eta   float64 // [0.0..1.0] overall net training rate
	//Alpha float64 // [0.0..n] multiplier of last weight chagne (momentum)
}

// NewNeuron intializes new neuron object
func NewNeuron(numOutputs, myIndex int) *Neuron {
	n := new(Neuron)
	// c for connections
	for c := 0; c < numOutputs; c++ {
		n.outputWeights = append(n.outputWeights, *new(neuronConnection))
		n.outputWeights[len(n.outputWeights)-1].Weight = randomWeight()
	}
	n.myIndex = myIndex
	return n
}

// FeedForward does the math magic to its self
func (n *Neuron) FeedForward(prevLayer *Layer) {
	var sum float64

	// Sum the previous layer's outputs (which are our inputs)
	// Include the bias node from the previous layer

	for i := 0; i < len(*prevLayer); i++ {
		sum += (*prevLayer)[i].outputVal *
			(*prevLayer)[i].outputWeights[n.myIndex].Weight
	}

	n.outputVal = transferFunction(sum)
}

func (n Neuron) sumDOW(nextLayer *Layer) float64 {
	var sum float64

	for i := 0; i < len(*nextLayer)-1; i++ {
		sum += n.outputWeights[i].Weight * (*nextLayer)[i].gradient
	}

	return sum
}

func (n *Neuron) calculateOutputGradients(targetVal float64) {
	delta := targetVal - n.outputVal
	n.gradient = delta * transferFunctionDerivative(n.outputVal)
}

func (n *Neuron) calculateHiddenGradients(nextLayer *Layer) {
	dow := n.sumDOW(nextLayer)
	n.gradient = dow * transferFunctionDerivative(n.outputVal)
}

func (n *Neuron) updateInputWeights(prevLayer *Layer) {
	// The weights to be updated are in the Conneciton container
	// int the neurons in the preceding layer
	for i := 0; i < len(*prevLayer); i++ {
		neuron := &(*prevLayer)[i]
		oldDeltaWeight := neuron.outputWeights[n.myIndex].DeltaWeight

		newDeltaWeight :=
			// Individual input, magnified by the gradient and train rate:
			Eta*neuron.outputVal*n.gradient +
				// Also add momentun = a fraction of the previous delta wieght
				Alpha*oldDeltaWeight
		neuron.outputWeights[n.myIndex].DeltaWeight = newDeltaWeight
		neuron.outputWeights[n.myIndex].Weight += newDeltaWeight
	}
}

// Layer is just array of neurons
type Layer []Neuron

// NeuralNetwork holds all the data of the network
type NeuralNetwork struct {
	Layers                            []Layer // layers[layerNum][neuronNum]
	err                               float64
	recentAverageError                float64
	RecentAverageErrorSmoothingFactor float64
}

// NewNetwork initializes new network
func NewNetwork(topology []int) *NeuralNetwork {
	n := new(NeuralNetwork)
	// Number of training smaples to average over
	//n.RecentAverageErrorSmoothingFactor = 112.0

	for layerNum := 0; layerNum < len(topology); layerNum++ {
		n.Layers = append(n.Layers, *new(Layer))
		var numOutputs int
		if layerNum == len(topology)-1 {
			numOutputs = 0
		} else {
			numOutputs = topology[layerNum+1]
		}

		// We have made new layer, now fill in its neurons
		// and a bias neuron
		for neuronNum := 0; neuronNum <= topology[layerNum]; neuronNum++ {
			n.Layers[len(n.Layers)-1] =
				append(n.Layers[len(n.Layers)-1],
					*NewNeuron(numOutputs, neuronNum))
		}

		// Force the bias node's output value to 1.0. It's the last neuron
		// created above
		layer := &n.Layers[layerNum]
		(*layer)[len(*layer)-1].outputVal = 1.0
	}
	return n
}

// FeedForward takes inputs
func (n *NeuralNetwork) FeedForward(inputVals []float64) {
	// Ignore bias
	if len(inputVals) > len(n.Layers[0])-1 {
		log.Fatalf("Length if inputsVals must be the same as length of"+
			" the first layer (%d != %d)", len(inputVals), len(n.Layers[0]))
	}

	// assign the input values into the input neurons
	for i := 0; i < len(inputVals); i++ {
		n.Layers[0][i].outputVal = inputVals[i]
	}

	// Forward propagate
	for layerNum := 1; layerNum < len(n.Layers); layerNum++ {
		prevLayer := &n.Layers[layerNum-1]
		for i := 0; i < len(n.Layers[layerNum])-1; i++ {
			n.Layers[layerNum][i].FeedForward(prevLayer)
		}
	}
}

// BackProp does the backpropagation (this is where the net learns)
func (n *NeuralNetwork) BackProp(targetVals []float64) {
	// Calculate overall net error (RMS of output errors)
	// RMS = "Root Mean Square Error"
	outputLayer := &n.Layers[len(n.Layers)-1]
	n.err = 0.0

	for i := 0; i < len(*outputLayer)-1; i++ {
		delta := targetVals[i] - (*outputLayer)[i].outputVal
		n.err += delta * delta
	}
	n.err /= float64(len(*outputLayer)) - 1.0 // get average error squared
	n.err = math.Sqrt(n.err)                  // RMS

	//if n.err > 2.0 {
	//n.err = 1.0
	//}

	// Implements a recent average measurement
	n.recentAverageError =
		(n.recentAverageError*n.RecentAverageErrorSmoothingFactor + n.err) /
			(n.RecentAverageErrorSmoothingFactor + 1.0)

	// Calculate output layer gradiants
	for i := 0; i < len(*outputLayer)-1; i++ {
		(*outputLayer)[i].calculateOutputGradients(targetVals[i])
	}

	// Calculate gradients on hidden layers
	for layerNum := len(n.Layers) - 2; layerNum > 0; layerNum-- {
		hiddenLayer := &n.Layers[layerNum]
		nextLayer := &n.Layers[layerNum+1]

		for i := 0; i < len(*hiddenLayer); i++ {
			(*hiddenLayer)[i].calculateHiddenGradients(nextLayer)
		}
	}

	// For all layers from outputs to first hidden layer,
	// update connection weights

	for layerNum := len(n.Layers) - 1; layerNum > 0; layerNum-- {
		layer := &n.Layers[layerNum]
		prevLayer := &n.Layers[layerNum-1]

		for i := 0; i < len(*layer)-1; i++ {
			(*layer)[i].updateInputWeights(prevLayer)
		}
	}
}

// GetAverageError return recentAvarageError value
func (n *NeuralNetwork) GetAverageError() float64 {
	return n.recentAverageError
}

// GetResults returns results from all output neurons as a string
func (n *NeuralNetwork) GetResults() string {
	var out string
	for i, outputNeuron := range n.Layers[len(n.Layers)-1] {
		if i == len(n.Layers[len(n.Layers)-1])-1 {
			// Ignore bias
			continue
		}
		out += fmt.Sprintf("%f ", outputNeuron.outputVal)
	}
	return out
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A fuse file system for Google Cloud Storage buckets.
//
// Usage:
//
//     gcsfuse [flags] bucket mount_point
//
package main

import (
	"crypto/tls"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"path"
	"path/filepath"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"

	"github.com/googlecloudplatform/gcsfuse/internal/auth"
	"github.com/googlecloudplatform/gcsfuse/internal/canned"
	"github.com/googlecloudplatform/gcsfuse/internal/gcsx"
	"github.com/googlecloudplatform/gcsfuse/internal/logger"
	"github.com/googlecloudplatform/gcsfuse/internal/perf"
	"github.com/jacobsa/daemonize"
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/syncutil"
	"github.com/kardianos/osext"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/urfave/cli"
)

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func registerSIGINTHandler(mountPoint string) {
	// Register for SIGINT.
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt)

	// Start a goroutine that will unmount when the signal is received.
	go func() {
		for {
			<-signalChan
			logger.Info("Received SIGINT, attempting to unmount...")

			err := fuse.Unmount(mountPoint)
			if err != nil {
				logger.Infof("Failed to unmount in response to SIGINT: %v", err)
			} else {
				logger.Infof("Successfully unmounted in response to SIGINT.")
				return
			}
		}
	}()
}

func startMonitoringHTTPHandler(monitoringPort int) {
	logger.Infof("Exporting metrics at localhost:%v/metrics\n", monitoringPort)
	go func() {
		http.Handle("/metrics", promhttp.Handler())
		http.ListenAndServe(fmt.Sprintf(":%v", monitoringPort), nil)
	}()
}

func getConn(flags *flagStorage) (c *gcsx.Connection, err error) {
	var tokenSrc oauth2.TokenSource
	if flags.Endpoint.Hostname() == "www.googleapis.com" {
		tokenSrc, err = auth.GetTokenSource(
			context.Background(),
			flags.KeyFile,
			flags.TokenUrl,
		)
		if err != nil {
			err = fmt.Errorf("GetTokenSource: %w", err)
			return
		}
	} else {
		// Do not use OAuth with non-Google hosts.
		tokenSrc = oauth2.StaticTokenSource(&oauth2.Token{})
	}

	// Create the connection.
	const userAgent = "gcsfuse/0.0"
	cfg := &gcs.ConnConfig{
		Url:             flags.Endpoint,
		TokenSource:     tokenSrc,
		UserAgent:       userAgent,
		MaxBackoffSleep: flags.MaxRetrySleep,
	}

	// The default HTTP transport uses HTTP/2 with TCP multiplexing, which
	// does not create new TCP connections even when the idle connections
	// run out. To specify multiple connections per host, HTTP/2 is disabled
	// on purpose.
	if flags.DisableHTTP2 {
		cfg.Transport = &http.Transport{
			MaxConnsPerHost: flags.MaxConnsPerHost,
			// This disables HTTP/2 in the transport.
			TLSNextProto: make(
				map[string]func(string, *tls.Conn) http.RoundTripper,
			),
		}
	}

	if flags.DebugHTTP {
		cfg.HTTPDebugLogger = logger.NewDebug("http: ")
	}

	if flags.DebugGCS {
		cfg.GCSDebugLogger = logger.NewDebug("gcs: ")
	}

	return gcsx.NewConnection(cfg)
}

////////////////////////////////////////////////////////////////////////
// main logic
////////////////////////////////////////////////////////////////////////

// Mount the file system according to arguments in the supplied context.
func mountWithArgs(
	bucketName string,
	mountPoint string,
	flags *flagStorage,
	mountStatus *log.Logger) (mfs *fuse.MountedFileSystem, err error) {
	// Enable invariant checking if requested.
	if flags.DebugInvariants {
		syncutil.EnableInvariantChecking()
	}

	// Grab the connection.
	//
	// Special case: if we're mounting the fake bucket, we don't need an actual
	// connection.
	var conn *gcsx.Connection
	if bucketName != canned.FakeBucketName {
		mountStatus.Println("Opening GCS connection...")

		conn, err = getConn(flags)
		if err != nil {
			err = fmt.Errorf("getConn: %w", err)
			return
		}
	}

	// Mount the file system.
	mfs, err = mountWithConn(
		context.Background(),
		bucketName,
		mountPoint,
		flags,
		conn,
		mountStatus)

	if err != nil {
		err = fmt.Errorf("mountWithConn: %w", err)
		return
	}

	return
}

func populateArgs(c *cli.Context) (
	bucketName string,
	mountPoint string,
	err error) {
	// Extract arguments.
	switch len(c.Args()) {
	case 1:
		bucketName = ""
		mountPoint = c.Args()[0]

	case 2:
		bucketName = c.Args()[0]
		mountPoint = c.Args()[1]

	default:
		err = fmt.Errorf(
			"%s takes one or two arguments. Run `%s --help` for more info.",
			path.Base(os.Args[0]),
			path.Base(os.Args[0]))

		return
	}

	// Canonicalize the mount point, making it absolute. This is important when
	// daemonizing below, since the daemon will change its working directory
	// before running this code again.
	mountPoint, err = filepath.Abs(mountPoint)
	if err != nil {
		err = fmt.Errorf("canonicalizing mount point: %w", err)
		return
	}
	return
}

func runCLIApp(c *cli.Context) (err error) {
	flags := populateFlags(c)

	if flags.Foreground && flags.LogFile != "" {
		err = logger.InitLogFile(flags.LogFile, flags.LogFormat)
		if err != nil {
			return fmt.Errorf("init log file: %w", err)
		}
	}

	var bucketName string
	var mountPoint string
	bucketName, mountPoint, err = populateArgs(c)
	if err != nil {
		return
	}

	logger.Infof("Using mount point: %s\n", mountPoint)

	// If we haven't been asked to run in foreground mode, we should run a daemon
	// with the foreground flag set and wait for it to mount.
	if !flags.Foreground {
		// Find the executable.
		var path string
		path, err = osext.Executable()
		if err != nil {
			err = fmt.Errorf("osext.Executable: %w", err)
			return
		}

		// Set up arguments. Be sure to use foreground mode, and to send along the
		// potentially-modified mount point.
		args := append([]string{"--foreground"}, os.Args[1:]...)
		args[len(args)-1] = mountPoint

		// Pass along PATH so that the daemon can find fusermount on Linux.
		env := []string{
			fmt.Sprintf("PATH=%s", os.Getenv("PATH")),
		}

		// Pass along GOOGLE_APPLICATION_CREDENTIALS, since we document in
		// mounting.md that it can be used for specifying a key file.
		if p, ok := os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS"); ok {
			env = append(env, fmt.Sprintf("GOOGLE_APPLICATION_CREDENTIALS=%s", p))
		}
		// Pass through the https_proxy/http_proxy environment variable,
		// in case the host requires a proxy server to reach the GCS endpoint.
		// https_proxy has precedence over http_proxy, in case both are set
		if p, ok := os.LookupEnv("https_proxy"); ok {
			env = append(env, fmt.Sprintf("https_proxy=%s", p))
			fmt.Fprintf(
				os.Stdout,
				"Added environment https_proxy: %s\n",
				p)
		} else if p, ok := os.LookupEnv("http_proxy"); ok {
			env = append(env, fmt.Sprintf("http_proxy=%s", p))
			fmt.Fprintf(
				os.Stdout,
				"Added environment http_proxy: %s\n",
				p)
		}

		// Run.
		err = daemonize.Run(path, args, env, os.Stdout)
		if err != nil {
			err = fmt.Errorf("daemonize.Run: %w", err)
			return
		}

		return
	}

	// Mount, writing information about our progress to the writer that package
	// daemonize gives us and telling it about the outcome.
	var mfs *fuse.MountedFileSystem
	{
		mountStatus := logger.NewNotice("")
		mfs, err = mountWithArgs(bucketName, mountPoint, flags, mountStatus)

		if err == nil {
			mountStatus.Println("File system has been successfully mounted.")
			daemonize.SignalOutcome(nil)
		} else {
			err = fmt.Errorf("mountWithArgs: %w", err)
			daemonize.SignalOutcome(err)
			return
		}
	}

	// Open a port for exporting monitoring metrics
	if flags.MonitoringPort > 0 {
		startMonitoringHTTPHandler(flags.MonitoringPort)
	}

	// Let the user unmount with Ctrl-C (SIGINT).
	registerSIGINTHandler(mfs.Dir())

	// Wait for the file system to be unmounted.
	err = mfs.Join(context.Background())
	if err != nil {
		err = fmt.Errorf("MountedFileSystem.Join: %w", err)
		return
	}

	return
}

func run() (err error) {
	// Set up the app.
	app := newApp()

	var appErr error
	app.Action = func(c *cli.Context) {
		appErr = runCLIApp(c)
	}

	// Run it.
	err = app.Run(os.Args)
	if err != nil {
		return
	}

	err = appErr
	return
}

func main() {
	// Make logging output better.
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)

	// Set up profiling handlers.
	go perf.HandleCPUProfileSignals()
	go perf.HandleMemoryProfileSignals()

	// Run.
	err := run()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<commit_msg>Use gcsfuse version and app name in the user-agent for HTTP headers<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A fuse file system for Google Cloud Storage buckets.
//
// Usage:
//
//     gcsfuse [flags] bucket mount_point
//
package main

import (
	"crypto/tls"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"path"
	"path/filepath"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"

	"github.com/googlecloudplatform/gcsfuse/internal/auth"
	"github.com/googlecloudplatform/gcsfuse/internal/canned"
	"github.com/googlecloudplatform/gcsfuse/internal/gcsx"
	"github.com/googlecloudplatform/gcsfuse/internal/logger"
	"github.com/googlecloudplatform/gcsfuse/internal/perf"
	"github.com/jacobsa/daemonize"
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/syncutil"
	"github.com/kardianos/osext"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/urfave/cli"
)

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func registerSIGINTHandler(mountPoint string) {
	// Register for SIGINT.
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt)

	// Start a goroutine that will unmount when the signal is received.
	go func() {
		for {
			<-signalChan
			logger.Info("Received SIGINT, attempting to unmount...")

			err := fuse.Unmount(mountPoint)
			if err != nil {
				logger.Infof("Failed to unmount in response to SIGINT: %v", err)
			} else {
				logger.Infof("Successfully unmounted in response to SIGINT.")
				return
			}
		}
	}()
}

func startMonitoringHTTPHandler(monitoringPort int) {
	logger.Infof("Exporting metrics at localhost:%v/metrics\n", monitoringPort)
	go func() {
		http.Handle("/metrics", promhttp.Handler())
		http.ListenAndServe(fmt.Sprintf(":%v", monitoringPort), nil)
	}()
}

func getConn(flags *flagStorage) (c *gcsx.Connection, err error) {
	var tokenSrc oauth2.TokenSource
	if flags.Endpoint.Hostname() == "www.googleapis.com" {
		tokenSrc, err = auth.GetTokenSource(
			context.Background(),
			flags.KeyFile,
			flags.TokenUrl,
		)
		if err != nil {
			err = fmt.Errorf("GetTokenSource: %w", err)
			return
		}
	} else {
		// Do not use OAuth with non-Google hosts.
		tokenSrc = oauth2.StaticTokenSource(&oauth2.Token{})
	}

	// Create the connection.
	cfg := &gcs.ConnConfig{
		Url:             flags.Endpoint,
		TokenSource:     tokenSrc,
		UserAgent:       fmt.Sprintf("gcsfuse/%s %s", getVersion(), flags.AppName),
		MaxBackoffSleep: flags.MaxRetrySleep,
	}

	// The default HTTP transport uses HTTP/2 with TCP multiplexing, which
	// does not create new TCP connections even when the idle connections
	// run out. To specify multiple connections per host, HTTP/2 is disabled
	// on purpose.
	if flags.DisableHTTP2 {
		cfg.Transport = &http.Transport{
			MaxConnsPerHost: flags.MaxConnsPerHost,
			// This disables HTTP/2 in the transport.
			TLSNextProto: make(
				map[string]func(string, *tls.Conn) http.RoundTripper,
			),
		}
	}

	if flags.DebugHTTP {
		cfg.HTTPDebugLogger = logger.NewDebug("http: ")
	}

	if flags.DebugGCS {
		cfg.GCSDebugLogger = logger.NewDebug("gcs: ")
	}

	return gcsx.NewConnection(cfg)
}

////////////////////////////////////////////////////////////////////////
// main logic
////////////////////////////////////////////////////////////////////////

// Mount the file system according to arguments in the supplied context.
func mountWithArgs(
	bucketName string,
	mountPoint string,
	flags *flagStorage,
	mountStatus *log.Logger) (mfs *fuse.MountedFileSystem, err error) {
	// Enable invariant checking if requested.
	if flags.DebugInvariants {
		syncutil.EnableInvariantChecking()
	}

	// Grab the connection.
	//
	// Special case: if we're mounting the fake bucket, we don't need an actual
	// connection.
	var conn *gcsx.Connection
	if bucketName != canned.FakeBucketName {
		mountStatus.Println("Opening GCS connection...")

		conn, err = getConn(flags)
		if err != nil {
			err = fmt.Errorf("getConn: %w", err)
			return
		}
	}

	// Mount the file system.
	mfs, err = mountWithConn(
		context.Background(),
		bucketName,
		mountPoint,
		flags,
		conn,
		mountStatus)

	if err != nil {
		err = fmt.Errorf("mountWithConn: %w", err)
		return
	}

	return
}

func populateArgs(c *cli.Context) (
	bucketName string,
	mountPoint string,
	err error) {
	// Extract arguments.
	switch len(c.Args()) {
	case 1:
		bucketName = ""
		mountPoint = c.Args()[0]

	case 2:
		bucketName = c.Args()[0]
		mountPoint = c.Args()[1]

	default:
		err = fmt.Errorf(
			"%s takes one or two arguments. Run `%s --help` for more info.",
			path.Base(os.Args[0]),
			path.Base(os.Args[0]))

		return
	}

	// Canonicalize the mount point, making it absolute. This is important when
	// daemonizing below, since the daemon will change its working directory
	// before running this code again.
	mountPoint, err = filepath.Abs(mountPoint)
	if err != nil {
		err = fmt.Errorf("canonicalizing mount point: %w", err)
		return
	}
	return
}

func runCLIApp(c *cli.Context) (err error) {
	flags := populateFlags(c)

	if flags.Foreground && flags.LogFile != "" {
		err = logger.InitLogFile(flags.LogFile, flags.LogFormat)
		if err != nil {
			return fmt.Errorf("init log file: %w", err)
		}
	}

	var bucketName string
	var mountPoint string
	bucketName, mountPoint, err = populateArgs(c)
	if err != nil {
		return
	}

	logger.Infof("Using mount point: %s\n", mountPoint)

	// If we haven't been asked to run in foreground mode, we should run a daemon
	// with the foreground flag set and wait for it to mount.
	if !flags.Foreground {
		// Find the executable.
		var path string
		path, err = osext.Executable()
		if err != nil {
			err = fmt.Errorf("osext.Executable: %w", err)
			return
		}

		// Set up arguments. Be sure to use foreground mode, and to send along the
		// potentially-modified mount point.
		args := append([]string{"--foreground"}, os.Args[1:]...)
		args[len(args)-1] = mountPoint

		// Pass along PATH so that the daemon can find fusermount on Linux.
		env := []string{
			fmt.Sprintf("PATH=%s", os.Getenv("PATH")),
		}

		// Pass along GOOGLE_APPLICATION_CREDENTIALS, since we document in
		// mounting.md that it can be used for specifying a key file.
		if p, ok := os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS"); ok {
			env = append(env, fmt.Sprintf("GOOGLE_APPLICATION_CREDENTIALS=%s", p))
		}
		// Pass through the https_proxy/http_proxy environment variable,
		// in case the host requires a proxy server to reach the GCS endpoint.
		// https_proxy has precedence over http_proxy, in case both are set
		if p, ok := os.LookupEnv("https_proxy"); ok {
			env = append(env, fmt.Sprintf("https_proxy=%s", p))
			fmt.Fprintf(
				os.Stdout,
				"Added environment https_proxy: %s\n",
				p)
		} else if p, ok := os.LookupEnv("http_proxy"); ok {
			env = append(env, fmt.Sprintf("http_proxy=%s", p))
			fmt.Fprintf(
				os.Stdout,
				"Added environment http_proxy: %s\n",
				p)
		}

		// Run.
		err = daemonize.Run(path, args, env, os.Stdout)
		if err != nil {
			err = fmt.Errorf("daemonize.Run: %w", err)
			return
		}

		return
	}

	// Mount, writing information about our progress to the writer that package
	// daemonize gives us and telling it about the outcome.
	var mfs *fuse.MountedFileSystem
	{
		mountStatus := logger.NewNotice("")
		mfs, err = mountWithArgs(bucketName, mountPoint, flags, mountStatus)

		if err == nil {
			mountStatus.Println("File system has been successfully mounted.")
			daemonize.SignalOutcome(nil)
		} else {
			err = fmt.Errorf("mountWithArgs: %w", err)
			daemonize.SignalOutcome(err)
			return
		}
	}

	// Open a port for exporting monitoring metrics
	if flags.MonitoringPort > 0 {
		startMonitoringHTTPHandler(flags.MonitoringPort)
	}

	// Let the user unmount with Ctrl-C (SIGINT).
	registerSIGINTHandler(mfs.Dir())

	// Wait for the file system to be unmounted.
	err = mfs.Join(context.Background())
	if err != nil {
		err = fmt.Errorf("MountedFileSystem.Join: %w", err)
		return
	}

	return
}

func run() (err error) {
	// Set up the app.
	app := newApp()

	var appErr error
	app.Action = func(c *cli.Context) {
		appErr = runCLIApp(c)
	}

	// Run it.
	err = app.Run(os.Args)
	if err != nil {
		return
	}

	err = appErr
	return
}

func main() {
	// Make logging output better.
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)

	// Set up profiling handlers.
	go perf.HandleCPUProfileSignals()
	go perf.HandleMemoryProfileSignals()

	// Run.
	err := run()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package fuseutil

import (
	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
	"golang.org/x/net/context"
)

// A struct representing the status of a mount operation, with methods for
// waiting on the mount to complete, waiting for unmounting, and causing
// unmounting.
type MountedFileSystem struct {
}

// Wait until the mount point is ready to be used. After a successful return
// from this function, the contents of the mounted file system should be
// visible in the directory supplied to NewMountPoint. May be called multiple
// times.
func (mfs *MountedFileSystem) WaitForReady(ctx context.Context) error

// Block until the file system has been unmounted. The return value will be
// non-nil if anything unexpected happened while mounting or serving. May be
// called multiple times.
func (mfs *MountedFileSystem) Join() error

// Attempt to unmount the file system. Use Join to wait for it to actually be
// unmounted.
func (mfs *MountedFileSystem) Unmount() error

// Attempt to mount the supplied file system on the given directory.
// mfs.WaitForReady() must be called to find out whether the mount was
// successful.
func MountFileSystem(
	dir string,
	fs fusefs.FS,
	options ...fuse.MountOption) (mfs *MountedFileSystem)
<commit_msg>Added more comments for Unmount.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package fuseutil

import (
	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
	"golang.org/x/net/context"
)

// A struct representing the status of a mount operation, with methods for
// waiting on the mount to complete, waiting for unmounting, and causing
// unmounting.
type MountedFileSystem struct {
}

// Wait until the mount point is ready to be used. After a successful return
// from this function, the contents of the mounted file system should be
// visible in the directory supplied to NewMountPoint. May be called multiple
// times.
func (mfs *MountedFileSystem) WaitForReady(ctx context.Context) error

// Block until the file system has been unmounted. The return value will be
// non-nil if anything unexpected happened while mounting or serving. May be
// called multiple times.
func (mfs *MountedFileSystem) Join() error

// Attempt to unmount the file system. Use Join to wait for it to actually be
// unmounted. You must first call WaitForReady to ensure there is no race with
// mounting.
func (mfs *MountedFileSystem) Unmount() error

// Attempt to mount the supplied file system on the given directory.
// mfs.WaitForReady() must be called to find out whether the mount was
// successful.
func MountFileSystem(
	dir string,
	fs fusefs.FS,
	options ...fuse.MountOption) (mfs *MountedFileSystem)
<|endoftext|>


<commit_before>package main

import (
    "github.com/trendmicro/gop"

    "flag"
    "fmt"
    "os"
    "time"
    "syscall"
)

func main() {
    var appName string
    flag.StringVar(&appName, "service", "", "Name of service to start")

    var projectName string
    flag.StringVar(&projectName, "project", "", "Name of project")

    var watchdogSecs float64
    flag.Float64Var(&watchdogSecs, "watchdog_secs", 1, "Number of seconds between checks")
    flag.Parse()

	if projectName == "" {
		println("You must specify the name of a projec with --project=project_name")
		os.Exit(1)
	}
	if appName == "" {
		println("You must specify the name of a gop exe to run with --service=exe_name")
		os.Exit(1)
	}

    // We won't run gop, but load it up for config and logging
    a := gop.Init(projectName, appName)

    a.Info("nelly initialised for [%s:%s]", projectName, appName)

    attr := new(os.ProcAttr)
    proc, err := os.StartProcess(appName, nil, attr)
    if err != nil {
        panic(fmt.Sprintf("Failed to start process [%s]: %s", appName, err.Error()))
    }
    a.Info("Started executable [%s] pid %d", appName, proc.Pid)

	// The child has to call setpgrp() to install itself as a process group leader. We can
	// then monitor whether the process group has become empty or not.

    // We can send a signal to all members of a process group with kill() with a -ve pid
    // We can send a 'do nothing' signal with a sig of 0

    // So we can send a sig of 0 to a process group and see if the process group is empty
    // empty process group => we need to restart

    ticker := time.Tick(time.Second * time.Duration(watchdogSecs))
    for {
        // Wait at least one tick, so the child has time to change it's
        // process group to be the same as it's pid
        if processGroupEmpty(a, proc.Pid) {
            a.Error("Process group empty")
            break
        }
        <- ticker
    }
    a.Error("Descendants are dead - exiting")
}

func processGroupEmpty(a *gop.App, pgid int) bool {
    err := syscall.Kill(-pgid, syscall.Signal(0x00))
    if err != nil {
        a.Error("Kill error: %s\n", err.Error())
    }
    return err != nil
}
<commit_msg>attempt to relay SIGTERM signal to child process group<commit_after>package main

import (
    "github.com/trendmicro/gop"

    "flag"
    "fmt"
    "os"
    "os/signal"
    "time"
    "syscall"
)

func main() {
    var appName string
    flag.StringVar(&appName, "service", "", "Name of service to start")

    var projectName string
    flag.StringVar(&projectName, "project", "", "Name of project")

	if projectName == "" {
		println("You must specify the name of a projec with --project=project_name")
		os.Exit(1)
	}
	if appName == "" {
		println("You must specify the name of a gop exe to run with --service=exe_name")
		os.Exit(1)
	}

    // We won't run gop, but load it up for config and logging
    a := gop.Init(projectName, appName)

    checkSecs, _ := a.Cfg.GetFloat32("gop", "nelly_check_secs", 1.0)

    a.Info("nelly initialised for [%s:%s]", projectName, appName)

    attr := new(os.ProcAttr)
    proc, err := os.StartProcess(appName, nil, attr)
    if err != nil {
        panic(fmt.Sprintf("Failed to start process [%s]: %s", appName, err.Error()))
    }
    a.Info("Started executable [%s] pid %d", appName, proc.Pid)

	// The child has to call setpgrp() to install itself as a process group leader. We can
	// then monitor whether the process group has become empty or not.

    // We can send a signal to all members of a process group with kill() with a -ve pid
    // We can send a 'do nothing' signal with a sig of 0

    // So we can send a sig of 0 to a process group and see if the process group is empty
    // empty process group => we need to restart

    sigChan := setupSignals(a, proc)

    pgid := proc.Pid

    ticker := time.Tick(time.Second * time.Duration(checkSecs))
LOOP:
    for {
        select {
            case <- ticker: {
                if processGroupEmpty(a, pgid) {
                    a.Error("Process group [%d] empty", pgid)
                    break LOOP
                }
            }
            case sig := <- sigChan: {
                a.Error("Caught signal: %s - killing process group", sig)
                syscall.Kill(-pgid, syscall.SIGTERM)
                a.Error("Exiting on SIGTERM")
                os.Exit(0)
            }
        }
    }
    a.Error("Descendants are dead - exiting")
}

func setupSignals(a *gop.App, proc *os.Process) chan os.Signal {
    sigChan := make(chan os.Signal, 10)     // 10 is arbitrary, we just need to keep up
    signal.Notify(sigChan, syscall.SIGTERM)
    return sigChan
}

func processGroupEmpty(a *gop.App, pgid int) bool {
    err := syscall.Kill(-pgid, syscall.Signal(0x00))
    if err != nil {
        a.Error("Kill error: %s\n", err.Error())
    }
    return err != nil
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os/exec"
	"strings"
)

func calculateHash(url, pathType string) (hash string) {
	prefetchCmd := exec.Command("nix-prefetch-"+pathType, url)
	prefetchOut, err := prefetchCmd.Output()
	if err != nil {
		log.Fatal(err)
	}
	return hashFromNixPrefetch(prefetchOut)
}

func hashFromNixPrefetch(prefetchOut []byte) string {
	prefetchStr := strings.TrimSpace(string(prefetchOut))
	prefetchLines := strings.Split(prefetchStr, "\n")
	return prefetchLines[len(prefetchLines)-1]
}
<commit_msg>Fetch submodules when computing hash<commit_after>package main

import (
	"log"
	"os/exec"
	"strings"
)

func calculateHash(url, pathType string) (hash string) {
	args := []string{}

	if pathType == "git" {
		// `fetchgit` passes this argument by default
		args = append(args, "--fetch-submodules")
	}

	args = append(args, url)
	prefetchCmd := exec.Command("nix-prefetch-"+pathType, args...)
	prefetchOut, err := prefetchCmd.Output()
	if err != nil {
		log.Fatal(err)
	}
	return hashFromNixPrefetch(prefetchOut)
}

func hashFromNixPrefetch(prefetchOut []byte) string {
	prefetchStr := strings.TrimSpace(string(prefetchOut))
	prefetchLines := strings.Split(prefetchStr, "\n")
	return prefetchLines[len(prefetchLines)-1]
}
<|endoftext|>


<commit_before>package service

import (
	"os"
	"time"

	"github.com/keybase/client/go/engine"
	keybase1 "github.com/keybase/client/protocol/go"
	"github.com/maxtaco/go-framed-msgpack-rpc/rpc2"
)

type CtlHandler struct {
	*BaseHandler
}

func NewCtlHandler(xp *rpc2.Transport) *CtlHandler {
	return &CtlHandler{BaseHandler: NewBaseHandler(xp)}
}

// Stop is called on the rpc keybase.1.ctl.stop, which shuts down the service.
func (c *CtlHandler) Stop(SessionID int) error {
	G.Log.Info("Received stop() RPC; shutting down")
	go func() {
		time.Sleep(1 * time.Second)
		G.Shutdown()
		os.Exit(0)
	}()
	return nil
}

func (c *CtlHandler) LogRotate(SessionID int) error {
	return G.Log.RotateLogFile()
}

func (c *CtlHandler) SetLogLevel(arg keybase1.SetLogLevelArg) error {
	G.Log.SetExternalLogLevel(arg.Level)
	return nil
}

func (c *CtlHandler) Reload(SessionID int) error {
	G.Log.Info("Reloading config file")
	return G.ConfigReload()
}

func (c *CtlHandler) DbNuke(SessionID int) error {
	ctx := engine.Context{
		LogUI: c.getLogUI(SessionID),
	}

	fn, err := G.LocalDb.Nuke()
	if err != nil {
		ctx.LogUI.Warning("Failed to nuke DB: %s", err)
		return err
	}
	ctx.LogUI.Warning("Nuking database %s", fn)

	// Now drop caches, since we had the DB's state in-memory too.
	return G.ConfigureCaches()
}
<commit_msg>Lowercase sessionIDs in Go code<commit_after>package service

import (
	"os"
	"time"

	"github.com/keybase/client/go/engine"
	keybase1 "github.com/keybase/client/protocol/go"
	"github.com/maxtaco/go-framed-msgpack-rpc/rpc2"
)

type CtlHandler struct {
	*BaseHandler
}

func NewCtlHandler(xp *rpc2.Transport) *CtlHandler {
	return &CtlHandler{BaseHandler: NewBaseHandler(xp)}
}

// Stop is called on the rpc keybase.1.ctl.stop, which shuts down the service.
func (c *CtlHandler) Stop(sessionID int) error {
	G.Log.Info("Received stop() RPC; shutting down")
	go func() {
		time.Sleep(1 * time.Second)
		G.Shutdown()
		os.Exit(0)
	}()
	return nil
}

func (c *CtlHandler) LogRotate(sessionID int) error {
	return G.Log.RotateLogFile()
}

func (c *CtlHandler) SetLogLevel(arg keybase1.SetLogLevelArg) error {
	G.Log.SetExternalLogLevel(arg.Level)
	return nil
}

func (c *CtlHandler) Reload(sessionID int) error {
	G.Log.Info("Reloading config file")
	return G.ConfigReload()
}

func (c *CtlHandler) DbNuke(sessionID int) error {
	ctx := engine.Context{
		LogUI: c.getLogUI(sessionID),
	}

	fn, err := G.LocalDb.Nuke()
	if err != nil {
		ctx.LogUI.Warning("Failed to nuke DB: %s", err)
		return err
	}
	ctx.LogUI.Warning("Nuking database %s", fn)

	// Now drop caches, since we had the DB's state in-memory too.
	return G.ConfigureCaches()
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"io"
	"os"
	"strings"
	"text/tabwriter"
	"text/template"
)

// AppHelpTemplate is the text template for the Default help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var AppHelpTemplate = `NAME:
   {{.Name}}{{if .Usage}} - {{.Usage}}{{end}}

USAGE:
   {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}} {{if .VisibleFlags}}[global options]{{end}}{{if .Commands}} command [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}}{{if .Version}}{{if not .HideVersion}}

VERSION:
   {{.Version}}{{end}}{{end}}{{if .Description}}

DESCRIPTION:
   {{.Description}}{{end}}{{if len .Authors}}

AUTHOR{{with $length := len .Authors}}{{if ne 1 $length}}S{{end}}{{end}}:
   {{range $index, $author := .Authors}}{{if $index}}
   {{end}}{{$author}}{{end}}{{end}}{{if .VisibleCommands}}

COMMANDS:{{range .VisibleCategories}}{{if .Name}}

   {{.Name}}:{{end}}{{range .VisibleCommands}}
     {{join .Names ", "}}{{"\t"}}{{.Usage}}{{end}}{{end}}{{end}}{{if .VisibleFlags}}

GLOBAL OPTIONS:
   {{range $index, $option := .VisibleFlags}}{{if $index}}
   {{end}}{{$option}}{{end}}{{end}}{{if .Copyright}}

COPYRIGHT:
   {{.Copyright}}{{end}}
`

// CommandHelpTemplate is the text template for the command help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var CommandHelpTemplate = `NAME:
   {{.HelpName}} - {{.Usage}}

USAGE:
   {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}}{{if .VisibleFlags}} [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}}{{if .Category}}

CATEGORY:
   {{.Category}}{{end}}{{if .Description}}

DESCRIPTION:
   {{.Description}}{{end}}{{if .VisibleFlags}}

OPTIONS:
   {{range .VisibleFlags}}{{.}}
   {{end}}{{end}}
`

// SubcommandHelpTemplate is the text template for the subcommand help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var SubcommandHelpTemplate = `NAME:
   {{.HelpName}} - {{if .Description}}{{.Description}}{{else}}{{.Usage}}{{end}}

USAGE:
   {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}} command{{if .VisibleFlags}} [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}}

COMMANDS:{{range .VisibleCategories}}{{if .Name}}
   {{.Name}}:{{end}}{{range .VisibleCommands}}
     {{join .Names ", "}}{{"\t"}}{{.Usage}}{{end}}
{{end}}{{if .VisibleFlags}}
OPTIONS:
   {{range .VisibleFlags}}{{.}}
   {{end}}{{end}}
`

var helpCommand = Command{
	Name:      "help",
	Aliases:   []string{"h"},
	Usage:     "Shows a list of commands or help for one command",
	ArgsUsage: "[command]",
	Action: func(c *Context) error {
		args := c.Args()
		if args.Present() {
			return ShowCommandHelp(c, args.First())
		}

		ShowAppHelp(c)
		return nil
	},
}

var helpSubcommand = Command{
	Name:      "help",
	Aliases:   []string{"h"},
	Usage:     "Shows a list of commands or help for one command",
	ArgsUsage: "[command]",
	Action: func(c *Context) error {
		args := c.Args()
		if args.Present() {
			return ShowCommandHelp(c, args.First())
		}

		return ShowSubcommandHelp(c)
	},
}

// Prints help for the App or Command
type helpPrinter func(w io.Writer, templ string, data interface{})

// Prints help for the App or Command with custom template function.
type helpPrinterCustom func(w io.Writer, templ string, data interface{}, customFunc map[string]interface{})

// HelpPrinter is a function that writes the help output. If not set a default
// is used. The function signature is:
// func(w io.Writer, templ string, data interface{})
var HelpPrinter helpPrinter = printHelp

// HelpPrinterCustom is same as HelpPrinter but
// takes a custom function for template function map.
var HelpPrinterCustom helpPrinterCustom = printHelpCustom

// VersionPrinter prints the version for the App
var VersionPrinter = printVersion

// ShowAppHelpAndExit - Prints the list of subcommands for the app and exits with exit code.
func ShowAppHelpAndExit(c *Context, exitCode int) {
	ShowAppHelp(c)
	os.Exit(exitCode)
}

// ShowAppHelp is an action that displays the help.
func ShowAppHelp(c *Context) (err error) {
	if c.App.CustomAppHelpTemplate == "" {
		HelpPrinter(c.App.Writer, AppHelpTemplate, c.App)
		return
	}
	customAppData := func() map[string]interface{} {
		if c.App.ExtraInfo == nil {
			return nil
		}
		return map[string]interface{}{
			"ExtraInfo": c.App.ExtraInfo,
		}
	}
	HelpPrinterCustom(c.App.Writer, c.App.CustomAppHelpTemplate, c.App, customAppData())
	return nil
}

// DefaultAppComplete prints the list of subcommands as the default app completion method
func DefaultAppComplete(c *Context) {
	for _, command := range c.App.Commands {
		if command.Hidden {
			continue
		}
		if os.Getenv("_CLI_ZSH_AUTOCOMPLETE_HACK") == "1" {
			for _, name := range command.Names() {
				fmt.Fprintf(c.App.Writer, "%s:%s\n", name, command.Usage)
			}
		} else {
			for _, name := range command.Names() {
				fmt.Fprintf(c.App.Writer, "%s\n", name)
			}
		}
	}
}

// ShowCommandHelpAndExit - exits with code after showing help
func ShowCommandHelpAndExit(c *Context, command string, code int) {
	ShowCommandHelp(c, command)
	os.Exit(code)
}

// ShowCommandHelp prints help for the given command
func ShowCommandHelp(ctx *Context, command string) error {
	// show the subcommand help for a command with subcommands
	if command == "" {
		HelpPrinter(ctx.App.Writer, SubcommandHelpTemplate, ctx.App)
		return nil
	}

	for _, c := range ctx.App.Commands {
		if c.HasName(command) {
			if c.CustomHelpTemplate != "" {
				HelpPrinterCustom(ctx.App.Writer, c.CustomHelpTemplate, c, nil)
			} else {
				HelpPrinter(ctx.App.Writer, CommandHelpTemplate, c)
			}
			return nil
		}
	}

	if ctx.App.CommandNotFound == nil {
		return NewExitError(fmt.Sprintf("No help topic for '%v'", command), 3)
	}

	ctx.App.CommandNotFound(ctx, command)
	return nil
}

// ShowSubcommandHelp prints help for the given subcommand
func ShowSubcommandHelp(c *Context) error {
	return ShowCommandHelp(c, c.Command.Name)
}

// ShowVersion prints the version number of the App
func ShowVersion(c *Context) {
	VersionPrinter(c)
}

func printVersion(c *Context) {
	fmt.Fprintf(c.App.Writer, "%v version %v\n", c.App.Name, c.App.Version)
}

// ShowCompletions prints the lists of commands within a given context
func ShowCompletions(c *Context) {
	a := c.App
	if a != nil && a.BashComplete != nil {
		a.BashComplete(c)
	}
}

// ShowCommandCompletions prints the custom completions for a given command
func ShowCommandCompletions(ctx *Context, command string) {
	c := ctx.App.Command(command)
	if c != nil && c.BashComplete != nil {
		c.BashComplete(ctx)
	}
}

func printHelpCustom(out io.Writer, templ string, data interface{}, customFunc map[string]interface{}) {
	funcMap := template.FuncMap{
		"join": strings.Join,
	}
	if customFunc != nil {
		for key, value := range customFunc {
			funcMap[key] = value
		}
	}

	w := tabwriter.NewWriter(out, 1, 8, 2, ' ', 0)
	t := template.Must(template.New("help").Funcs(funcMap).Parse(templ))
	err := t.Execute(w, data)
	if err != nil {
		// If the writer is closed, t.Execute will fail, and there's nothing
		// we can do to recover.
		if os.Getenv("CLI_TEMPLATE_ERROR_DEBUG") != "" {
			fmt.Fprintf(ErrWriter, "CLI TEMPLATE ERROR: %#v\n", err)
		}
		return
	}
	w.Flush()
}

func printHelp(out io.Writer, templ string, data interface{}) {
	printHelpCustom(out, templ, data, nil)
}

func checkVersion(c *Context) bool {
	found := false
	if VersionFlag.GetName() != "" {
		eachName(VersionFlag.GetName(), func(name string) {
			if c.GlobalBool(name) || c.Bool(name) {
				found = true
			}
		})
	}
	return found
}

func checkHelp(c *Context) bool {
	found := false
	if HelpFlag.GetName() != "" {
		eachName(HelpFlag.GetName(), func(name string) {
			if c.GlobalBool(name) || c.Bool(name) {
				found = true
			}
		})
	}
	return found
}

func checkCommandHelp(c *Context, name string) bool {
	if c.Bool("h") || c.Bool("help") {
		ShowCommandHelp(c, name)
		return true
	}

	return false
}

func checkSubcommandHelp(c *Context) bool {
	if c.Bool("h") || c.Bool("help") {
		ShowSubcommandHelp(c)
		return true
	}

	return false
}

func checkShellCompleteFlag(a *App, arguments []string) (bool, []string) {
	if !a.EnableBashCompletion {
		return false, arguments
	}

	pos := len(arguments) - 1
	lastArg := arguments[pos]

	if lastArg != "--"+BashCompletionFlag.GetName() {
		return false, arguments
	}

	return true, arguments[:pos]
}

func checkCompletions(c *Context) bool {
	if !c.shellComplete {
		return false
	}

	if args := c.Args(); args.Present() {
		name := args.First()
		if cmd := c.App.Command(name); cmd != nil {
			// let the command handle the completion
			return false
		}
	}

	ShowCompletions(c)
	return true
}

func checkCommandCompletions(c *Context, name string) bool {
	if !c.shellComplete {
		return false
	}

	ShowCommandCompletions(c, name)
	return true
}
<commit_msg>Remove redundant nil checks<commit_after>package cli

import (
	"fmt"
	"io"
	"os"
	"strings"
	"text/tabwriter"
	"text/template"
)

// AppHelpTemplate is the text template for the Default help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var AppHelpTemplate = `NAME:
   {{.Name}}{{if .Usage}} - {{.Usage}}{{end}}

USAGE:
   {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}} {{if .VisibleFlags}}[global options]{{end}}{{if .Commands}} command [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}}{{if .Version}}{{if not .HideVersion}}

VERSION:
   {{.Version}}{{end}}{{end}}{{if .Description}}

DESCRIPTION:
   {{.Description}}{{end}}{{if len .Authors}}

AUTHOR{{with $length := len .Authors}}{{if ne 1 $length}}S{{end}}{{end}}:
   {{range $index, $author := .Authors}}{{if $index}}
   {{end}}{{$author}}{{end}}{{end}}{{if .VisibleCommands}}

COMMANDS:{{range .VisibleCategories}}{{if .Name}}

   {{.Name}}:{{end}}{{range .VisibleCommands}}
     {{join .Names ", "}}{{"\t"}}{{.Usage}}{{end}}{{end}}{{end}}{{if .VisibleFlags}}

GLOBAL OPTIONS:
   {{range $index, $option := .VisibleFlags}}{{if $index}}
   {{end}}{{$option}}{{end}}{{end}}{{if .Copyright}}

COPYRIGHT:
   {{.Copyright}}{{end}}
`

// CommandHelpTemplate is the text template for the command help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var CommandHelpTemplate = `NAME:
   {{.HelpName}} - {{.Usage}}

USAGE:
   {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}}{{if .VisibleFlags}} [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}}{{if .Category}}

CATEGORY:
   {{.Category}}{{end}}{{if .Description}}

DESCRIPTION:
   {{.Description}}{{end}}{{if .VisibleFlags}}

OPTIONS:
   {{range .VisibleFlags}}{{.}}
   {{end}}{{end}}
`

// SubcommandHelpTemplate is the text template for the subcommand help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var SubcommandHelpTemplate = `NAME:
   {{.HelpName}} - {{if .Description}}{{.Description}}{{else}}{{.Usage}}{{end}}

USAGE:
   {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}} command{{if .VisibleFlags}} [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}}

COMMANDS:{{range .VisibleCategories}}{{if .Name}}
   {{.Name}}:{{end}}{{range .VisibleCommands}}
     {{join .Names ", "}}{{"\t"}}{{.Usage}}{{end}}
{{end}}{{if .VisibleFlags}}
OPTIONS:
   {{range .VisibleFlags}}{{.}}
   {{end}}{{end}}
`

var helpCommand = Command{
	Name:      "help",
	Aliases:   []string{"h"},
	Usage:     "Shows a list of commands or help for one command",
	ArgsUsage: "[command]",
	Action: func(c *Context) error {
		args := c.Args()
		if args.Present() {
			return ShowCommandHelp(c, args.First())
		}

		ShowAppHelp(c)
		return nil
	},
}

var helpSubcommand = Command{
	Name:      "help",
	Aliases:   []string{"h"},
	Usage:     "Shows a list of commands or help for one command",
	ArgsUsage: "[command]",
	Action: func(c *Context) error {
		args := c.Args()
		if args.Present() {
			return ShowCommandHelp(c, args.First())
		}

		return ShowSubcommandHelp(c)
	},
}

// Prints help for the App or Command
type helpPrinter func(w io.Writer, templ string, data interface{})

// Prints help for the App or Command with custom template function.
type helpPrinterCustom func(w io.Writer, templ string, data interface{}, customFunc map[string]interface{})

// HelpPrinter is a function that writes the help output. If not set a default
// is used. The function signature is:
// func(w io.Writer, templ string, data interface{})
var HelpPrinter helpPrinter = printHelp

// HelpPrinterCustom is same as HelpPrinter but
// takes a custom function for template function map.
var HelpPrinterCustom helpPrinterCustom = printHelpCustom

// VersionPrinter prints the version for the App
var VersionPrinter = printVersion

// ShowAppHelpAndExit - Prints the list of subcommands for the app and exits with exit code.
func ShowAppHelpAndExit(c *Context, exitCode int) {
	ShowAppHelp(c)
	os.Exit(exitCode)
}

// ShowAppHelp is an action that displays the help.
func ShowAppHelp(c *Context) (err error) {
	if c.App.CustomAppHelpTemplate == "" {
		HelpPrinter(c.App.Writer, AppHelpTemplate, c.App)
		return
	}
	customAppData := func() map[string]interface{} {
		if c.App.ExtraInfo == nil {
			return nil
		}
		return map[string]interface{}{
			"ExtraInfo": c.App.ExtraInfo,
		}
	}
	HelpPrinterCustom(c.App.Writer, c.App.CustomAppHelpTemplate, c.App, customAppData())
	return nil
}

// DefaultAppComplete prints the list of subcommands as the default app completion method
func DefaultAppComplete(c *Context) {
	for _, command := range c.App.Commands {
		if command.Hidden {
			continue
		}
		if os.Getenv("_CLI_ZSH_AUTOCOMPLETE_HACK") == "1" {
			for _, name := range command.Names() {
				fmt.Fprintf(c.App.Writer, "%s:%s\n", name, command.Usage)
			}
		} else {
			for _, name := range command.Names() {
				fmt.Fprintf(c.App.Writer, "%s\n", name)
			}
		}
	}
}

// ShowCommandHelpAndExit - exits with code after showing help
func ShowCommandHelpAndExit(c *Context, command string, code int) {
	ShowCommandHelp(c, command)
	os.Exit(code)
}

// ShowCommandHelp prints help for the given command
func ShowCommandHelp(ctx *Context, command string) error {
	// show the subcommand help for a command with subcommands
	if command == "" {
		HelpPrinter(ctx.App.Writer, SubcommandHelpTemplate, ctx.App)
		return nil
	}

	for _, c := range ctx.App.Commands {
		if c.HasName(command) {
			if c.CustomHelpTemplate != "" {
				HelpPrinterCustom(ctx.App.Writer, c.CustomHelpTemplate, c, nil)
			} else {
				HelpPrinter(ctx.App.Writer, CommandHelpTemplate, c)
			}
			return nil
		}
	}

	if ctx.App.CommandNotFound == nil {
		return NewExitError(fmt.Sprintf("No help topic for '%v'", command), 3)
	}

	ctx.App.CommandNotFound(ctx, command)
	return nil
}

// ShowSubcommandHelp prints help for the given subcommand
func ShowSubcommandHelp(c *Context) error {
	return ShowCommandHelp(c, c.Command.Name)
}

// ShowVersion prints the version number of the App
func ShowVersion(c *Context) {
	VersionPrinter(c)
}

func printVersion(c *Context) {
	fmt.Fprintf(c.App.Writer, "%v version %v\n", c.App.Name, c.App.Version)
}

// ShowCompletions prints the lists of commands within a given context
func ShowCompletions(c *Context) {
	a := c.App
	if a != nil && a.BashComplete != nil {
		a.BashComplete(c)
	}
}

// ShowCommandCompletions prints the custom completions for a given command
func ShowCommandCompletions(ctx *Context, command string) {
	c := ctx.App.Command(command)
	if c != nil && c.BashComplete != nil {
		c.BashComplete(ctx)
	}
}

func printHelpCustom(out io.Writer, templ string, data interface{}, customFunc map[string]interface{}) {
	funcMap := template.FuncMap{
		"join": strings.Join,
	}
	for key, value := range customFunc {
		funcMap[key] = value
	}

	w := tabwriter.NewWriter(out, 1, 8, 2, ' ', 0)
	t := template.Must(template.New("help").Funcs(funcMap).Parse(templ))
	err := t.Execute(w, data)
	if err != nil {
		// If the writer is closed, t.Execute will fail, and there's nothing
		// we can do to recover.
		if os.Getenv("CLI_TEMPLATE_ERROR_DEBUG") != "" {
			fmt.Fprintf(ErrWriter, "CLI TEMPLATE ERROR: %#v\n", err)
		}
		return
	}
	w.Flush()
}

func printHelp(out io.Writer, templ string, data interface{}) {
	printHelpCustom(out, templ, data, nil)
}

func checkVersion(c *Context) bool {
	found := false
	if VersionFlag.GetName() != "" {
		eachName(VersionFlag.GetName(), func(name string) {
			if c.GlobalBool(name) || c.Bool(name) {
				found = true
			}
		})
	}
	return found
}

func checkHelp(c *Context) bool {
	found := false
	if HelpFlag.GetName() != "" {
		eachName(HelpFlag.GetName(), func(name string) {
			if c.GlobalBool(name) || c.Bool(name) {
				found = true
			}
		})
	}
	return found
}

func checkCommandHelp(c *Context, name string) bool {
	if c.Bool("h") || c.Bool("help") {
		ShowCommandHelp(c, name)
		return true
	}

	return false
}

func checkSubcommandHelp(c *Context) bool {
	if c.Bool("h") || c.Bool("help") {
		ShowSubcommandHelp(c)
		return true
	}

	return false
}

func checkShellCompleteFlag(a *App, arguments []string) (bool, []string) {
	if !a.EnableBashCompletion {
		return false, arguments
	}

	pos := len(arguments) - 1
	lastArg := arguments[pos]

	if lastArg != "--"+BashCompletionFlag.GetName() {
		return false, arguments
	}

	return true, arguments[:pos]
}

func checkCompletions(c *Context) bool {
	if !c.shellComplete {
		return false
	}

	if args := c.Args(); args.Present() {
		name := args.First()
		if cmd := c.App.Command(name); cmd != nil {
			// let the command handle the completion
			return false
		}
	}

	ShowCompletions(c)
	return true
}

func checkCommandCompletions(c *Context, name string) bool {
	if !c.shellComplete {
		return false
	}

	ShowCommandCompletions(c, name)
	return true
}
<|endoftext|>


<commit_before>// Copyright 2017 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package dns

import (
	"encoding/hex"
	"net"
	"regexp"
	"strings"
)

// SUBRE is a regular expression that will match on all subdomains once the domain is appended.
const SUBRE = "(([a-zA-Z0-9]{1}|[_a-zA-Z0-9]{1}[_a-zA-Z0-9-]{0,61}[a-zA-Z0-9]{1})[.]{1})+"

// SubdomainRegex returns a Regexp object initialized to match
// subdomain names that end with the domain provided by the parameter.
func SubdomainRegex(domain string) *regexp.Regexp {
	return regexp.MustCompile(SubdomainRegexString(domain))
}

// SubdomainRegexString returns a regular expression string that matchs
// subdomain names ending with the domain provided by the parameter.
func SubdomainRegexString(domain string) string {
	// Change all the periods into literal periods for the regex
	return SUBRE + strings.Replace(domain, ".", "[.]", -1)
}

// AnySubdomainRegex returns a Regexp object initialized to match any DNS subdomain name.
func AnySubdomainRegex() *regexp.Regexp {
	return regexp.MustCompile(AnySubdomainRegexString())
}

// AnySubdomainRegexString returns a regular expression string to match any DNS subdomain name.
func AnySubdomainRegexString() string {
	return SUBRE + "[a-zA-Z]{2,61}"
}

// CopyString return a new string variable with the same value as the parameter.
func CopyString(src string) string {
	str := make([]byte, len(src))

	copy(str, src)
	return string(str)
}

// RemoveAsteriskLabel returns the provided DNS name with all asterisk labels removed.
func RemoveAsteriskLabel(s string) string {
	startIndex := strings.LastIndex(s, "*.")

	if startIndex == -1 {
		return s
	}

	return s[startIndex+2:]
}

// ReverseString returns the characters of the argument string in reverse order.
func ReverseString(s string) string {
	chrs := []rune(s)
	end := len(chrs) / 2

	for i, j := 0, len(chrs)-1; i < end; i, j = i+1, j-1 {
		chrs[i], chrs[j] = chrs[j], chrs[i]
	}

	return string(chrs)
}

// ReverseIP returns an IP address that is the ip parameter with the numbers reversed.
func ReverseIP(ip string) string {
	var reversed []string

	parts := strings.Split(ip, ".")
	li := len(parts) - 1

	for i := li; i >= 0; i-- {
		reversed = append(reversed, parts[i])
	}

	return strings.Join(reversed, ".")
}

// IPv6NibbleFormat expects an IPv6 address in the ip parameter and
// returns the address in nibble format.
func IPv6NibbleFormat(ip string) string {
	var reversed []string

	ip = strings.ReplaceAll(expandIPv6Addr(ip), ":", "")
	parts := strings.Split(ip, "")
	li := len(parts) - 1

	for i := li; i >= 0; i-- {
		reversed = append(reversed, parts[i])
	}

	return strings.Join(reversed, ".")
}

func expandIPv6Addr(addr string) string {
	ip := net.ParseIP(addr)

	dst := make([]byte, hex.EncodedLen(len(ip)))
	hex.Encode(dst, ip)

	return string(dst[0:4]) + ":" +
		string(dst[4:8]) + ":" +
		string(dst[8:12]) + ":" +
		string(dst[12:16]) + ":" +
		string(dst[16:20]) + ":" +
		string(dst[20:24]) + ":" +
		string(dst[24:28]) + ":" +
		string(dst[28:])
}
<commit_msg>Updated a source code comment<commit_after>// Copyright 2017-2021 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package dns

import (
	"encoding/hex"
	"net"
	"regexp"
	"strings"
)

// SUBRE is a regular expression that will match on all subdomains once the domain is appended.
const SUBRE = "(([a-zA-Z0-9]{1}|[_a-zA-Z0-9]{1}[_a-zA-Z0-9-]{0,61}[a-zA-Z0-9]{1})[.]{1})+"

// SubdomainRegex returns a Regexp object initialized to match
// subdomain names that end with the domain provided by the parameter.
func SubdomainRegex(domain string) *regexp.Regexp {
	return regexp.MustCompile(SubdomainRegexString(domain))
}

// SubdomainRegexString returns a regular expression string that matchs
// subdomain names ending with the domain provided by the parameter.
func SubdomainRegexString(domain string) string {
	// Change all the periods into literal periods for the regex
	return SUBRE + strings.Replace(domain, ".", "[.]", -1)
}

// AnySubdomainRegex returns a Regexp object initialized to match any DNS subdomain name.
func AnySubdomainRegex() *regexp.Regexp {
	return regexp.MustCompile(AnySubdomainRegexString())
}

// AnySubdomainRegexString returns a regular expression string to match any DNS subdomain name.
func AnySubdomainRegexString() string {
	return SUBRE + "[a-zA-Z]{2,61}"
}

// CopyString return a new string variable with the same value as the parameter.
func CopyString(src string) string {
	str := make([]byte, len(src))

	copy(str, src)
	return string(str)
}

// RemoveAsteriskLabel returns the provided DNS name with all asterisk labels removed.
func RemoveAsteriskLabel(s string) string {
	startIndex := strings.LastIndex(s, "*.")

	if startIndex == -1 {
		return s
	}

	return s[startIndex+2:]
}

// ReverseString returns the characters of the argument string in reverse order.
func ReverseString(s string) string {
	chrs := []rune(s)
	end := len(chrs) / 2

	for i, j := 0, len(chrs)-1; i < end; i, j = i+1, j-1 {
		chrs[i], chrs[j] = chrs[j], chrs[i]
	}

	return string(chrs)
}

// ReverseIP returns an IP address that is the ip parameter with the numbers reversed.
func ReverseIP(ip string) string {
	var reversed []string

	parts := strings.Split(ip, ".")
	li := len(parts) - 1

	for i := li; i >= 0; i-- {
		reversed = append(reversed, parts[i])
	}

	return strings.Join(reversed, ".")
}

// IPv6NibbleFormat expects an IPv6 address in the ip parameter and
// returns the address in nibble format.
func IPv6NibbleFormat(ip string) string {
	var reversed []string

	ip = strings.ReplaceAll(expandIPv6Addr(ip), ":", "")
	parts := strings.Split(ip, "")
	li := len(parts) - 1

	for i := li; i >= 0; i-- {
		reversed = append(reversed, parts[i])
	}

	return strings.Join(reversed, ".")
}

func expandIPv6Addr(addr string) string {
	ip := net.ParseIP(addr)

	dst := make([]byte, hex.EncodedLen(len(ip)))
	hex.Encode(dst, ip)

	return string(dst[0:4]) + ":" +
		string(dst[4:8]) + ":" +
		string(dst[8:12]) + ":" +
		string(dst[12:16]) + ":" +
		string(dst[16:20]) + ":" +
		string(dst[20:24]) + ":" +
		string(dst[24:28]) + ":" +
		string(dst[28:])
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "os"
    "io"
    "bytes"
    "errors"
    "bufio"
    "sync"
    "strings"
    "unsafe"
    "reflect"
    goopt "github.com/droundy/goopt"
    //"regexp"
    //sre2  "github.com/samthor/sre2"
    pcre "github.com/gijsbers/go-pcre"
    //rubex "github.com/moovweb/rubex"
)
var Usage = "gmlgrep [OPTIONS...] PATTERN[...] [--] [FILES...]"
var Summary = `
  grep(1) like tool, but "record-oriented", instead of line-oriented.
  Useful to search/print multi-line log entries separated by e.g., empty
  lines, '----' or timestamps, etc. If an argument in argument list is a
  name of existing file or '-' (means stdin), such argument and all arguments
  after that will be treated as filenames to read from. Otherwise arguments
  are considered to be regex to search. (could be confusing if you specify
  nonexistent filename!)`

// The Flag function creates a boolean flag, possibly with a negating
// alternative.  Note that you can specify either long or short flags
// naturally in the same list.
var optCount = goopt.Flag([]string{"-c", "--count"}, nil,
    "Print number of matches. (same as grep -c)", "")
var optIgnoreCase = goopt.Flag([]string{"-i", "--ignore-case"}, nil,
    "Case insensitive matching. Default is case sensitive.", "")
var optInvert = goopt.Flag([]string{"-v", "--invert"}, nil,
    "Select non-matching records (same as grep -v).", "")
var optAnd   = goopt.Flag([]string{"-a", "--and"}, nil,
    "Extract records with all of patterns. (default: any)", "")
var optTimestamp  = goopt.Flag([]string{"-t", "--timestamp"}, nil,
    "Same as --rs=TIMESTAMP_REGEX, where the regex matches timestamps often used in log files, e.g., '2014-12-31 12:34:56' or 'Dec 31 12:34:56'.", "")
var optColor = goopt.Flag([]string{"--color", "--hl"}, nil,
    "Highlight matches. Default is enabled iff stdout is a TTY.", "")

const RS_REGEX = "^$|^(=====*|-----*)$"
var rs = goopt.StringWithLabel([]string{"-r", "--rs"}, RS_REGEX, "RS_REGEX",
    fmt.Sprintf("Input record separator. default: /%s/", RS_REGEX))

//const DATETIME_REGEX = ""

//////////////////////////////////////////////////////////////////////////////
//maxBufferSize = 1 * 1024 * 1024

//////////////////////////////////////////////////////////////////////////////
// regex wrapper

type FindIndex func (d []byte) []int
type Regexp struct {
    //r *rubex.Regexp
    r pcre.Regexp
    FindIndex
}

func reComp(restr string) Regexp {
    //nlchars := regexp.MustCompile("\\^|\\$")
    //restr = nlchars.ReplaceAllString(restr, "\n")
    //return regexp.MustCompile( restr )
    //return regexp.MustCompile( "(?m)" + restr )
    //r := rubex.MustCompile(restr)
    //return Regexp{r, r.FindIndex}
    r := pcre.MustCompile( restr, pcre.MULTILINE )
    f := func (d []byte) []int { return r.FindIndex(d, 0) }
    return Regexp{r, f}
}

//////////////////////////////////////////////////////////////////////////////

func checkError(e error) {
    if e != nil {
        fmt.Fprintf(os.Stdout, "ERROR: %d\n", e)
        os.Exit(1)
    }
}

func debug(format string, args ...interface{}) {
    //fmt.Fprintf(os.Stderr, ">> DEBUG: " + format, args...)
}

//escape newline for debug output.
//func esc(s string) {
//    strings.Replace(s, "\n", "\\n", -1)
//}

func esc(b []byte) string {
    return strings.Replace(string(b), "\n", "\\n", -1)
}

//////////////////////////////////////////////////////////////////////////////
// Find-pattern-first algorithm

type PatternFirstFinder struct {
    found bool;
    patFinder   func(data []byte) (int, int)
    rsFinder    func(data []byte) (int, int)
    rsRevFinder func(data []byte) (int, int)
}

func NewPatternFirstFinder(pat, rs string) *PatternFirstFinder{
    //compile regex and set MLRFinder fields
    s := new(PatternFirstFinder)
    s.found = false
    s.patFinder   = func(d []byte) (int, int) { return bytes.Index(d, []byte(pat)), len(pat) }
    s.rsFinder    = func(d []byte) (int, int) { return bytes.Index(d, []byte(rs)), len(rs) }
    s.rsRevFinder = func(d []byte) (int, int) {
        if len(d) < len (rs) {
            return -1, 0
        }
        for pos := 0; pos < len(d); pos++ {
            if bytes.HasSuffix(d[0:len(d)-pos], []byte(rs)) {
                return pos, len(rs)
            }
        }
        return -1, 0
    }
    return s
}

func (s *PatternFirstFinder) Split(data []byte, atEOF bool, tooLong bool) (advance int, token []byte, err error) {
    s.found = false
    //debug("split(\"%s\", %v, %v)\n", esc(data[:60]), atEOF, tooLong)

    if atEOF && len(data) == 0 {
        return 0, nil, nil
    }

    if (tooLong) {
        // so this is retry with tooLong flag enabled; we cannot request more data
        // and there's no match of the pattern in data. So we return
        rsPos, rsSize := s.rsRevFinder(data)
        if rsPos < 0 {
            return 0, nil, errors.New("record is too long and didn't fit into a buffer")
        } else {
            //return non-match records with s.found set to false
            return len(data) - rsPos + rsSize, data[len(data):len(data)], nil
        }
    }

    loc, size := s.patFinder(data)
    if loc < 0 {
        return 0, nil, nil //request more data.
    }
    s.found = true
    debug("patFinder() loc=%d, size=%d, '%s'\n", loc, size, esc(data[loc:loc+size]))
    preLoc := 0
    preSize := 0
    if loc != 0 {
        var lastRsOffset int
        lastRsOffset, preSize = s.rsRevFinder(data[:loc])
        if lastRsOffset > 0 {
            preLoc = loc - lastRsOffset - preSize
        }
    }
    debug("rs='%s'\n", data[preLoc:preLoc+preSize])

    postLoc, postSize := s.rsFinder(data[loc+size:])
    if (postLoc < 0) {
        if (atEOF) {
            return len(data), data[preLoc:], nil
        } else {
            return 0, nil, nil //not enough data
        }
    }
    debug("postLoc, postSize = %d, %d\n", postLoc, postSize)
    debug("post string: %s\n", data[loc+size+postLoc:loc+size+postLoc+postSize])

    recBegin := preLoc+preSize
    recEnd   := loc+size+postLoc+postSize
    rec      := data[recBegin:recEnd]
    debug("RETURN: %d, %s\n", recEnd, esc(rec))
    return recEnd, rec, nil
}

//////////////////////////////////////////////////////////////////////////////
// Find-pattern-first algorithm

type SplitRecordFirstFinder struct {
    found bool
    rsSize int
    rsPos int
    rsFinder func(data []byte) (int, int)
}

func regexFinder(restr string) (func (d []byte) (int, int)) {
    re := reComp(restr)
    return func(d []byte) (int, int) {
        m := re.FindIndex(d)
        if m != nil {
            return m[0], (m[1] - m[0])
        } else {
            return -1, 0
        }
    }
}

func NewSplitRecordFirstFinder(pat, rs string) *SplitRecordFirstFinder{
    s := new(SplitRecordFirstFinder)
    s.rsFinder = regexFinder(rs)
    //s.rsFinder = func(d []byte) (int, int) { return bytes.Index(d, []byte(rs)), len(rs) }
    return s
}

func (s *SplitRecordFirstFinder) Split(data []byte, atEOF bool) (advance int, token []byte, err error) {
    s.rsPos = 0
    if atEOF && len(data) == 0 {
        return 0, nil, nil
    }
    pos, sz := s.rsFinder(data)
    if (pos < 0) {
        if (atEOF) {
            s.rsPos = len(data)
            return len(data), data, nil
        } else {
            return 0, nil, nil //not enough data
        }
    }
    if (pos+sz == 0) {
        //FIXME: is this the best way to handle empty match?
        // The only known case so far is when using /^$/ with (?m) flag
        s.rsPos = 1
        return 1, data[0:1], nil
    } else {
        s.rsPos = pos
        return pos+sz, data[0:pos+sz], nil
    }
}

//////////////////////////////////////////////////////////////////////////////
//Split Record First

// records returned from the splitter is terminted with RS
// for speed reason, but we want to have RS at the begining of
// records (it makes sense if RS is a time stamp or other time
// header info.
type Record struct {
    chunk string
    rsPos int
}

func unsafeStrToByte(s string) []byte {
    strHeader := (*reflect.StringHeader)(unsafe.Pointer(&s))

    var b []byte
    byteHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b))
    byteHeader.Data = strHeader.Data

    // need to take the length of s here to ensure s is live until after we update b's Data
    // field since the garbage collector can collect a variable once it is no longer used
    // not when it goes out of scope, for more details see https://github.com/golang/go/issues/9046
    //l := len(s)
    //byteHeader.Len = l
    //byteHeader.Cap = l
    return b
}

func grep_record(pat string, pipe chan Record, wg* sync.WaitGroup) {
    defer wg.Done()
    var prevRS string
    /*
    // plain text
    for rec := range pipe {
        if strings.Index(rec.chunk, pat) > 0 {
            fmt.Print(prevRS)
            fmt.Print(rec.chunk[:rec.rsPos])
        }
        prevRS = rec.chunk[rec.rsPos:]
    }
    */

    //regex
    re := reComp(pat)
    for rec := range pipe {
        //if ( re.FindIndex([]byte(rec.chunk)) != nil ) {
        if ( re.FindIndex( unsafeStrToByte(rec.chunk) ) != nil ) {
            fmt.Print(prevRS)
            fmt.Print(rec.chunk[:rec.rsPos])
        }
        prevRS = rec.chunk[rec.rsPos:]
        //fmt.Println(">>'" + prevRS + "'")
    }
}

func mlrgrep_srf(pat string, rs string, r io.Reader) {
    var wg sync.WaitGroup
    w := bufio.NewWriter(os.Stdout)
    pipe := make(chan Record, 128)
    scanner := bufio.NewScanner(r)
    splitter := NewSplitRecordFirstFinder(pat, rs)

    scanner.Split(splitter.Split)
    wg.Add(1)
    go grep_record(pat, pipe, &wg)

    for scanner.Scan() {
        rec := scanner.Text()
        pipe <- Record{chunk: rec, rsPos: splitter.rsPos}
    }
    close(pipe)
    wg.Wait()
    w.Flush()
}

//Find Pattern First
func mlrgrep_fpf(pat string, rs string, r io.Reader) {
    w := bufio.NewWriter(os.Stdout)
    scanner := NewScanner(r)
    splitter := NewPatternFirstFinder(pat, rs)

    scanner.Split(splitter.Split)

    for scanner.Scan() {
        line := scanner.Bytes()
        if splitter.found {
            w.Write(line)
        }
    }
    w.Flush()
}

func main() {
    goopt.Description = func() string {
        return "Example program for using the goopt flag library."
    }
    goopt.Version = "0.1"
    goopt.Usage = func() string {
        usage := "Usage: " + Usage
        usage += fmt.Sprintf("%s", Summary)
        usage += fmt.Sprintf("\n%s", goopt.Help())
        return usage
    }
    goopt.Parse(nil)

    var regex []string
    var files []string
    //defer fmt.Print("\033[0m") // defer resetting the terminal to default colors

    debug("os.Args: %s\n", os.Args)
    debug("rs=%s\n", *rs)

    i := 0;
    for _, a := range goopt.Args[i:] {
        if (a == "--") {
            i++
            break;
        }
        // if an argument is a filename for existing one,
        // assume that (and everything follows) as filename.
        f, err := os.Stat(a)
        if (err == nil && !f.IsDir() ) {
            break;
        }
        regex = append(regex, a)
        i++
    }

    for _, a := range goopt.Args[i:] {
        if (a == "--") {
            regex = append(regex, files...)
            files = nil
        }
        files = append(files, a)
    }
    debug("regex: %s\n", regex)
    debug("files: %s\n", files)

    for _, f := range files {
        file, e := os.Open(f)
        checkError(e)
        defer file.Close()
        mlrgrep_srf(regex[0], *rs, file)
        //mlrgrep_fpf(regex[0], *rs, file)
    }
}
<commit_msg>use go-oniguruma, fix unsafe byte conversion, still broken for --rs='^$'<commit_after>package main

import (
	"bufio"
	"fmt"
	goopt "github.com/droundy/goopt"
	"io"
	"os"
	"reflect"
	//"regexp"
	"strings"
	"sync"
	"unsafe"
	//pcre "github.com/gijsbers/go-pcre"
	rubex "github.com/go-enry/go-oniguruma"
)

var Usage = "gmlgrep [OPTIONS...] PATTERN[...] [--] [FILES...]"
var Summary = `
  grep(1) like tool, but "record-oriented", instead of line-oriented.
  Useful to search/print multi-line log entries separated by e.g., empty
  lines, '----' or timestamps, etc. If an argument in argument list is a
  name of existing file or '-' (means stdin), such argument and all arguments
  after that will be treated as filenames to read from. Otherwise arguments
  are considered to be regex to search. (could be confusing if you specify
  nonexistent filename!)`

// The Flag function creates a boolean flag, possibly with a negating
// alternative.  Note that you can specify either long or short flags
// naturally in the same list.
var optCount = goopt.Flag([]string{"-c", "--count"}, nil,
	"Print number of matches. (same as grep -c)", "")
var optIgnoreCase = goopt.Flag([]string{"-i", "--ignore-case"}, nil,
	"Case insensitive matching. Default is case sensitive.", "")
var optInvert = goopt.Flag([]string{"-v", "--invert"}, nil,
	"Select non-matching records (same as grep -v).", "")
var optAnd = goopt.Flag([]string{"-a", "--and"}, nil,
	"Extract records with all of patterns. (default: any)", "")

//var optTimestamp = goopt.Flag([]string{"-t", "--timestamp"}, nil,
//	"Same as --rs=TIMESTAMP_REGEX, where the regex matches timestamps often used in log files, e.g., '2014-12-31 12:34:56' or 'Dec 31 12:34:56'.", "")
//var optColor = goopt.Flag([]string{"--color", "--hl"}, nil,
//	"Highlight matches. Default is enabled iff stdout is a TTY.", "")

const RS_REGEX = "^$|^(=====*|-----*)$"

var rs = goopt.StringWithLabel([]string{"-r", "--rs"}, RS_REGEX, "RS_REGEX",
	fmt.Sprintf("Input record separator. default: /%s/", RS_REGEX))

//////////////////////////////////////////////////////////////////////////////
//
// regex wrapper (fix import as well)
//
type FindIndex func(d []byte) []int

type Regexp struct {
	//r *regexp.Regexp
	//r pcre.Regexp
	r *rubex.Regexp
	FindIndex
}

func reComp(restr string) Regexp {
	// golang's standard regexp - broken for last newline
	/*
		nlchars := regexp.MustCompile("\\^|\\$")
		restr = nlchars.ReplaceAllString(restr, "\n")
		r := regexp.MustCompile(restr)
		return Regexp{r, r.FindIndex}
	*/

	/* go-oniguruma(rubex) */
	r := rubex.MustCompile(restr)
	f := func(d []byte) []int { return r.FindIndex(d) }

	/* PCRE */
	//r := pcre.MustCompile(restr, pcre.MULTILINE)
	//f := func(d []byte) []int { return r.FindIndex(d, 0) }

	return Regexp{r, f} // common for go-oniguruma & PCRE
}

//////////////////////////////////////////////////////////////////////////////

func checkError(e error) {
	if e != nil {
		fmt.Fprintf(os.Stdout, "ERROR: %d\n", e)
		os.Exit(1)
	}
}

func debug(format string, args ...interface{}) {
	//fmt.Fprintf(os.Stderr, ">> DEBUG: "+format, args...)
}

//escape newline for debug output.
//func esc(s string) {
//    strings.Replace(s, "\n", "\\n", -1)
//}

func esc(b []byte) string {
	return strings.Replace(string(b), "\n", "\\n", -1)
}

//////////////////////////////////////////////////////////////////////////////
// Find-pattern-first algorithm

type PatternFirstFinder struct {
	found       bool
	patFinder   func(data []byte) (int, int)
	rsFinder    func(data []byte) (int, int)
	rsRevFinder func(data []byte) (int, int)
}

//////////////////////////////////////////////////////////////////////////////
// Find-pattern-first algorithm

type SplitRecordFirstFinder struct {
	found    bool
	rsSize   int
	rsPos    int
	rsFinder func(data []byte) (int, int)
}

func regexFinder(restr string) func(d []byte) (pos int, size int) {
	re := reComp(restr)
	return func(d []byte) (int, int) {
		m := re.FindIndex(d)
		if m != nil {
			return m[0], (m[1] - m[0])
		} else {
			return -1, 0
		}
	}
}

func NewSplitRecordFirstFinder(pat, rs string) *SplitRecordFirstFinder {
	s := new(SplitRecordFirstFinder)
	s.rsFinder = regexFinder(rs)
	return s
}

func (s *SplitRecordFirstFinder) Split(data []byte, atEOF bool) (advance int, token []byte, err error) {
	s.rsPos = 0
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	pos, sz := s.rsFinder(data)
	if pos < 0 {
		if atEOF {
			s.rsPos = len(data)
			return len(data), data, nil
		} else {
			return 0, nil, nil //not enough data
		}
	}
	if pos+sz == 0 {
		//FIXME: is this the best way to handle empty match?
		// The only known case so far is when using /^$/ with (?m) flag
		s.rsPos = 1
		return 1, data[0:1], nil
	} else {
		debug("Not empty\n")
		s.rsPos = pos
		return pos + sz, data[0 : pos+sz], nil
	}
}

//////////////////////////////////////////////////////////////////////////////
//Split Record First

// records returned from the splitter is terminted with RS
// for speed reason, but we want to have RS at the begining of
// records (it makes sense if RS is a time stamp or other time
// header info.
type Record struct {
	chunk string
	rsPos int
}

func unsafeStrToByte(s string) []byte {
	strHeader := (*reflect.StringHeader)(unsafe.Pointer(&s))

	var b []byte
	byteHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b))
	byteHeader.Data = strHeader.Data

	// need to take the length of s here to ensure s is live until after we update b's Data
	// field since the garbage collector can collect a variable once it is no longer used
	// not when it goes out of scope, for more details see https://github.com/golang/go/issues/9046
	l := len(s)
	byteHeader.Len = l
	byteHeader.Cap = l
	return b
}

func grep_record(pat string, pipe chan Record, wg *sync.WaitGroup) {
	defer wg.Done()
	var prevRS string

	re := reComp(pat)
	for rec := range pipe {
		debug("------------------------------------\n")
		if re.FindIndex(unsafeStrToByte(rec.chunk)) != nil {
			fmt.Print(prevRS)
			fmt.Print(rec.chunk[:rec.rsPos])
		}
		prevRS = rec.chunk[rec.rsPos:]
		debug("prevRS='%s'\n", prevRS)
	}
}

func mlrgrep_srf(pat string, rs string, r io.Reader) {
	var wg sync.WaitGroup
	w := bufio.NewWriter(os.Stdout)
	pipe := make(chan Record, 128)
	scanner := bufio.NewScanner(r)
	splitter := NewSplitRecordFirstFinder(pat, rs)

	scanner.Split(splitter.Split)
	wg.Add(1)
	go grep_record(pat, pipe, &wg)

	for scanner.Scan() {
		rec := scanner.Text()
		pipe <- Record{chunk: rec, rsPos: splitter.rsPos}
	}
	close(pipe)
	wg.Wait()
	w.Flush()
}

func main() {
	goopt.Version = "0.1"
	goopt.Usage = func() string {
		usage := "Usage: " + Usage
		usage += fmt.Sprintf("%s", Summary)
		usage += fmt.Sprintf("\n%s", goopt.Help())
		return usage
	}
	goopt.Parse(nil)

	var regex []string
	var files []string

	debug("os.Args: %s\n", os.Args)
	debug("rs=%s\n", *rs)

	i := 0
	for _, a := range goopt.Args[i:] {
		if a == "--" {
			i++
			break
		}
		// if an argument is a filename for existing one,
		// assume that (and everything follows) as filename.
		f, err := os.Stat(a)
		if err == nil && !f.IsDir() {
			break
		}
		regex = append(regex, a)
		i++
	}

	for _, a := range goopt.Args[i:] {
		if a == "--" {
			regex = append(regex, files...)
			files = nil
		}
		files = append(files, a)
	}
	debug("regex: %s\n", regex)
	debug("files: %s\n", files)

	for _, f := range files {
		file, e := os.Open(f)
		checkError(e)
		defer file.Close()
		mlrgrep_srf(regex[0], *rs, file)
	}
}
<|endoftext|>


<commit_before>package corehttp

import (
	"bufio"
	"fmt"
	"net"
	"io"
	"net/http"
	"net/http/httputil"
	"strings"

	core "github.com/ipfs/go-ipfs/core"

	protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol"
	peer "gx/ipfs/QmbNepETomvmXfz1X5pHNFD2QuPqnqi47dTd94QJWSorQ3/go-libp2p-peer"
	inet "gx/ipfs/QmfDPh144WGBqRxZb1TGDHerbMnZATrHZggAPw7putNnBq/go-libp2p-net"
)

// ProxyOption is an endpoint for proxying a HTTP request to another ipfs peer
func ProxyOption() ServeOption {
	return func(ipfsNode *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
		mux.HandleFunc("/proxy/http/", func(w http.ResponseWriter, request *http.Request) {
			// parse request
			parsedRequest, err := parseRequest(request)
			if err != nil {
				handleError(w, "Failed to parse request", err, 400)
				return
			}

			// open connect to peer
			stream, err := ipfsNode.P2P.PeerHost.NewStream(ipfsNode.Context(), parsedRequest.target, protocol.ID("/x/"+parsedRequest.name))
			if err != nil {
				msg := fmt.Sprintf("Failed to open stream '%v' to target peer '%v'", parsedRequest.name, parsedRequest.target)
				handleError(w, msg, err, 500)
				return
			}
			//send proxy request and response to client
			newReverseHTTPProxy(parsedRequest, stream).ServeHTTP(w, request)
		})
		return mux, nil
	}
}

type proxyRequest struct {
	target   peer.ID
	name     string
	httpPath string // path to send to the proxy-host
}

// from the url path parse the peer-ID, name and http path
// /proxy/http/$peer_id/$name/$http_path
func parseRequest(request *http.Request) (*proxyRequest, error) {
	path := request.URL.Path

	split := strings.SplitN(path, "/", 6)
	if len(split) < 6 {
		return nil, fmt.Errorf("Invalid request path '%s'", path)
	}

	peerID, err := peer.IDB58Decode(split[3])

	if err != nil {
		return nil, err
	}

	return &proxyRequest{peerID, split[4], "/" + split[5]}, nil
}

func handleError(w http.ResponseWriter, msg string, err error, code int) {
	w.WriteHeader(code)
	fmt.Fprintf(w, "%s: %s\n", msg, err)
	log.Warningf("server error: %s: %s", err)
}

func newReverseHTTPProxy(req *proxyRequest, streamToPeer inet.Stream) *httputil.ReverseProxy {
	director := func(r *http.Request) {
		r.URL.Path = req.httpPath //the scheme etc. doesn't matter
	}

	return &httputil.ReverseProxy{
		Director:  director,
		Transport: &roundTripper{streamToPeer}}
}

type roundTripper struct {
	stream inet.Stream
}

// we wrap the response body and close the stream
// only when it's closed.
type respBody struct {
	io.ReadCloser
	stream inet.Stream
}

// Closes the response's body and the connection.
func (rb *respBody) Close() error {
	rb.stream.Close()
	return rb.ReadCloser.Close()
}

func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {

	sendRequest := func() {
		err := req.Write(rt.stream)
		if err != nil {
			rt.stream.Close()
		}
		if req.Body != nil {
			req.Body.Close()
		}
	}
	//send request while reading response
	go sendRequest()
	s := bufio.NewReader(rt.stream)

	resp, err := http.ReadResponse(s, req)
	if err != nil {
		return resp, err
	}

	resp.Body = &respBody{
		ReadCloser: resp.Body,
		stream:       rt.stream,
	}

	return resp, nil
}
<commit_msg>Use request context in p2p stream http proxy<commit_after>package corehttp

import (
	"bufio"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"strings"

	core "github.com/ipfs/go-ipfs/core"

	protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol"
	peer "gx/ipfs/QmbNepETomvmXfz1X5pHNFD2QuPqnqi47dTd94QJWSorQ3/go-libp2p-peer"
	inet "gx/ipfs/QmfDPh144WGBqRxZb1TGDHerbMnZATrHZggAPw7putNnBq/go-libp2p-net"
)

// ProxyOption is an endpoint for proxying a HTTP request to another ipfs peer
func ProxyOption() ServeOption {
	return func(ipfsNode *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
		mux.HandleFunc("/proxy/http/", func(w http.ResponseWriter, request *http.Request) {
			// parse request
			parsedRequest, err := parseRequest(request)
			if err != nil {
				handleError(w, "Failed to parse request", err, 400)
				return
			}

			// open connect to peer
			stream, err := ipfsNode.P2P.PeerHost.NewStream(request.Context(), parsedRequest.target, protocol.ID("/x/"+parsedRequest.name))
			if err != nil {
				msg := fmt.Sprintf("Failed to open stream '%v' to target peer '%v'", parsedRequest.name, parsedRequest.target)
				handleError(w, msg, err, 500)
				return
			}
			//send proxy request and response to client
			newReverseHTTPProxy(parsedRequest, stream).ServeHTTP(w, request)
		})
		return mux, nil
	}
}

type proxyRequest struct {
	target   peer.ID
	name     string
	httpPath string // path to send to the proxy-host
}

// from the url path parse the peer-ID, name and http path
// /proxy/http/$peer_id/$name/$http_path
func parseRequest(request *http.Request) (*proxyRequest, error) {
	path := request.URL.Path

	split := strings.SplitN(path, "/", 6)
	if len(split) < 6 {
		return nil, fmt.Errorf("Invalid request path '%s'", path)
	}

	peerID, err := peer.IDB58Decode(split[3])

	if err != nil {
		return nil, err
	}

	return &proxyRequest{peerID, split[4], "/" + split[5]}, nil
}

func handleError(w http.ResponseWriter, msg string, err error, code int) {
	w.WriteHeader(code)
	fmt.Fprintf(w, "%s: %s\n", msg, err)
	log.Warningf("server error: %s: %s", err)
}

func newReverseHTTPProxy(req *proxyRequest, streamToPeer inet.Stream) *httputil.ReverseProxy {
	director := func(r *http.Request) {
		r.URL.Path = req.httpPath //the scheme etc. doesn't matter
	}

	return &httputil.ReverseProxy{
		Director:  director,
		Transport: &roundTripper{streamToPeer}}
}

type roundTripper struct {
	stream inet.Stream
}

// we wrap the response body and close the stream
// only when it's closed.
type respBody struct {
	io.ReadCloser
	stream inet.Stream
}

// Closes the response's body and the connection.
func (rb *respBody) Close() error {
	rb.stream.Close()
	return rb.ReadCloser.Close()
}

func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {

	sendRequest := func() {
		err := req.Write(rt.stream)
		if err != nil {
			rt.stream.Close()
		}
		if req.Body != nil {
			req.Body.Close()
		}
	}
	//send request while reading response
	go sendRequest()
	s := bufio.NewReader(rt.stream)

	resp, err := http.ReadResponse(s, req)
	if err != nil {
		return resp, err
	}

	resp.Body = &respBody{
		ReadCloser: resp.Body,
		stream:     rt.stream,
	}

	return resp, nil
}
<|endoftext|>