<commit_before>// The metrics package defines prometheus metric types and provides
// convenience methods to add accounting to various parts of the pipeline.
//
// When defining new operations or metrics, these are helpful values to track:
//  - things coming into or go out of the system: requests, files, tests, api calls.
//  - the success or error status of any of the above.
//  - the distribution of processing latency.
package metrics

import (
	"math"
	"net/http"
	"time"

	"github.com/prometheus/client_golang/prometheus"
)

func init() {
	// Register the metrics defined with Prometheus's default registry.
	prometheus.MustRegister(WorkerCount)
	prometheus.MustRegister(TaskCount)
	prometheus.MustRegister(BigQueryInsert)
	prometheus.MustRegister(DurationHistogram)
	prometheus.MustRegister(InsertionHistogram)
	prometheus.MustRegister(FileSizeHistogram)
}

var (
	// Counts the number of tasks processed by the pipeline.
	//
	// Provides metrics:
	//   etl_worker_count
	// Example usage:
	//   metrics.TaskCount.Inc() / .Dec()
	WorkerCount = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "etl_worker_count",
		Help: "Number of active workers.",
	})

	// Counts the number of tasks processed by the pipeline.
	//
	// Provides metrics:
	//   etl_task_count{worker, status}
	// Example usage:
	//   metrics.TaskCount.WithLabelValues("ndt", "ok").Inc()
	TaskCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_task_count",
			Help: "Number of tasks/archive files processed.",
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"worker", "status"},
	)

	// Counts the number of into BigQuery insert operations.
	//
	// Provides metrics:
	//   etl_worker_bigquery_insert_total{worker, status}
	// Usage example:
	//   metrics.BigQueryInsert.WithLabelValues("ndt", "200").Inc()
	BigQueryInsert = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_worker_bigquery_insert_total",
			Help: "Number of BigQuery insert operations.",
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"worker", "status"},
	)

	// A histogram of bigquery insertion times. The buckets should use
	// periods that are intuitive for people.
	//
	// Provides metrics:
	//   etl_insertion_time_seconds_bucket{type="...", le="..."}
	//   ...
	//   etl_insertion_time_seconds_sum{type="..."}
	//   etl_insertion_time_seconds_count{type="..."}
	// Usage example:
	//   t := time.Now()
	//   // do some stuff.
	//   metrics.InsertionHistogram.WithLabelValues(
	//           "ndt_test", "ok").Observe(time.Since(t).Seconds())
	InsertionHistogram = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "etl_insertion_time_seconds",
			Help: "Insertion time distributions.",
			Buckets: []float64{
				0.001, 0.003, 0.01, 0.03, 0.1, 0.2, 0.5, 1.0, 2.0,
				5.0, 10.0, 20.0, 50.0, 100.0, math.Inf(+1),
			},
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"table", "status"},
	)

	// A histogram of worker processing times. The buckets should use
	// periods that are intuitive for people.
	//
	// Provides metrics:
	//   etl_worker_duration_seconds_bucket{worker="...", le="..."}
	//   ...
	//   etl_worker_duration_seconds_sum{worker="..."}
	//   etl_worker_duration_seconds_count{worker="..."}
	// Usage example:
	//   t := time.Now()
	//   // do some stuff.
	//   metrics.DurationHistogram.WithLabelValues(
	//           "ndt").Observe(time.Since(t).Seconds())
	DurationHistogram = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "etl_worker_duration_seconds",
			Help: "Worker execution time distributions.",
			Buckets: []float64{
				0.001, 0.01, 0.1, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0,
				600.0, 1800.0, 3600.0, 7200.0, math.Inf(+1),
			},
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		// TODO(soltesz): support a status field based on HTTP status.
		[]string{"worker"},
	)

	FileSizeHistogram = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Name: "etl_web100_snaplog_file_size_bytes",
			Help: "Size of individual snaplog files.",
			Buckets: []float64{
				100000,
				1000000,    // mb
				2000000,    // mb
				4000000,    // mb
				8000000,    // mb
				10000000,   // 10 mb
				20000000,   // 20
				40000000,   // 40
				80000000,   // 80
				100000000,  // 100 mb
				200000000,  // 200
				400000000,  // 400
				800000000,  // 800
				1000000000, // 1 gb
			},
		},
	)
)

// DurationHandler wraps the call of an inner http.HandlerFunc and records the runtime.
func DurationHandler(name string, inner http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		t := time.Now()
		inner.ServeHTTP(w, r)
		// TODO(soltesz): collect success or failure status.
		DurationHistogram.WithLabelValues(name).Observe(time.Since(t).Seconds())
	}
}
<commit_msg>Use fine-grained file size hist buckets.<commit_after>// The metrics package defines prometheus metric types and provides
// convenience methods to add accounting to various parts of the pipeline.
//
// When defining new operations or metrics, these are helpful values to track:
//  - things coming into or go out of the system: requests, files, tests, api calls.
//  - the success or error status of any of the above.
//  - the distribution of processing latency.
package metrics

import (
	"math"
	"net/http"
	"time"

	"github.com/prometheus/client_golang/prometheus"
)

func init() {
	// Register the metrics defined with Prometheus's default registry.
	prometheus.MustRegister(WorkerCount)
	prometheus.MustRegister(TaskCount)
	prometheus.MustRegister(BigQueryInsert)
	prometheus.MustRegister(DurationHistogram)
	prometheus.MustRegister(InsertionHistogram)
	prometheus.MustRegister(FileSizeHistogram)
}

var (
	// Counts the number of tasks processed by the pipeline.
	//
	// Provides metrics:
	//   etl_worker_count
	// Example usage:
	//   metrics.TaskCount.Inc() / .Dec()
	WorkerCount = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "etl_worker_count",
		Help: "Number of active workers.",
	})

	// Counts the number of tasks processed by the pipeline.
	//
	// Provides metrics:
	//   etl_task_count{worker, status}
	// Example usage:
	//   metrics.TaskCount.WithLabelValues("ndt", "ok").Inc()
	TaskCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_task_count",
			Help: "Number of tasks/archive files processed.",
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"worker", "status"},
	)

	// Counts the number of into BigQuery insert operations.
	//
	// Provides metrics:
	//   etl_worker_bigquery_insert_total{worker, status}
	// Usage example:
	//   metrics.BigQueryInsert.WithLabelValues("ndt", "200").Inc()
	BigQueryInsert = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_worker_bigquery_insert_total",
			Help: "Number of BigQuery insert operations.",
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"worker", "status"},
	)

	// A histogram of bigquery insertion times. The buckets should use
	// periods that are intuitive for people.
	//
	// Provides metrics:
	//   etl_insertion_time_seconds_bucket{type="...", le="..."}
	//   ...
	//   etl_insertion_time_seconds_sum{type="..."}
	//   etl_insertion_time_seconds_count{type="..."}
	// Usage example:
	//   t := time.Now()
	//   // do some stuff.
	//   metrics.InsertionHistogram.WithLabelValues(
	//           "ndt_test", "ok").Observe(time.Since(t).Seconds())
	InsertionHistogram = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "etl_insertion_time_seconds",
			Help: "Insertion time distributions.",
			Buckets: []float64{
				0.001, 0.003, 0.01, 0.03, 0.1, 0.2, 0.5, 1.0, 2.0,
				5.0, 10.0, 20.0, 50.0, 100.0, math.Inf(+1),
			},
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"table", "status"},
	)

	// A histogram of worker processing times. The buckets should use
	// periods that are intuitive for people.
	//
	// Provides metrics:
	//   etl_worker_duration_seconds_bucket{worker="...", le="..."}
	//   ...
	//   etl_worker_duration_seconds_sum{worker="..."}
	//   etl_worker_duration_seconds_count{worker="..."}
	// Usage example:
	//   t := time.Now()
	//   // do some stuff.
	//   metrics.DurationHistogram.WithLabelValues(
	//           "ndt").Observe(time.Since(t).Seconds())
	DurationHistogram = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "etl_worker_duration_seconds",
			Help: "Worker execution time distributions.",
			Buckets: []float64{
				0.001, 0.01, 0.1, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0,
				600.0, 1800.0, 3600.0, 7200.0, math.Inf(+1),
			},
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		// TODO(soltesz): support a status field based on HTTP status.
		[]string{"worker"},
	)

	FileSizeHistogram = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Name: "etl_web100_snaplog_file_size_bytes",
			Help: "Size of individual snaplog files.",
			Buckets: []float64{
				0,
				400000,     // 400k
				500000,     // 500k
				600000,     // 600k
				700000,     // 700k
				800000,     // 800k
				900000,     // 900k
				1000000,    // 1 mb
				1100000,    // 1.1 mb
				1200000,    // 1.2 mb
				1400000,    // 1.4 mb
				1600000,    // 1.6 mb
				1800000,    // 1.8 mb
				2000000,    // 2.0 mb
				2400000,    // 2.4 mb
				2800000,    // 2.8 mb
				3200000,    // 3.2 mb
				3600000,    // 3.6 mb
				4000000,    // 4 mb
				6000000,    // 6 mb
				8000000,    // 8 mb
				10000000,   // 10 mb
				20000000,   // 20
				40000000,   // 40
				80000000,   // 80
				100000000,  // 100 mb
				200000000,  // 200
				400000000,  // 400
				800000000,  // 800
				1000000000, // 1 gb
				math.Inf(+1),
			},
		},
	)
)

// DurationHandler wraps the call of an inner http.HandlerFunc and records the runtime.
func DurationHandler(name string, inner http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		t := time.Now()
		inner.ServeHTTP(w, r)
		// TODO(soltesz): collect success or failure status.
		DurationHistogram.WithLabelValues(name).Observe(time.Since(t).Seconds())
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package metrics

import "github.com/prometheus/client_golang/prometheus"

// Session metrics.
var (
	SessionExecuteParseDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "parse_duration_seconds",
			Help:      "Bucketed histogram of processing time (s) in parse SQL.",
			Buckets:   prometheus.ExponentialBuckets(0.00004, 2, 22), // 40us ~ 168s
		}, []string{LblSQLType})
	SessionExecuteCompileDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "compile_duration_seconds",
			Help:      "Bucketed histogram of processing time (s) in query optimize.",
			// Build plan may execute the statement, or allocate table ID, so it might take a long time.
			Buckets: prometheus.ExponentialBuckets(0.00004, 2, 22), // 40us ~ 168s
		}, []string{LblSQLType})
	SessionExecuteRunDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "execute_duration_seconds",
			Help:      "Bucketed histogram of processing time (s) in running executor.",
			Buckets:   prometheus.ExponentialBuckets(0.0001, 2, 22), // 100us ~ 419s
		}, []string{LblSQLType})
	SchemaLeaseErrorCounter = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "schema_lease_error_total",
			Help:      "Counter of schema lease error",
		}, []string{LblType})
	SessionRetry = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "retry_num",
			Help:      "Bucketed histogram of session retry count.",
			Buckets:   prometheus.LinearBuckets(0, 1, 20), // 0 ~ 20
		})
	SessionRetryErrorCounter = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "retry_error_total",
			Help:      "Counter of session retry error.",
		}, []string{LblSQLType, LblType})

	SessionRestrictedSQLCounter = prometheus.NewCounter(
		prometheus.CounterOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "restricted_sql_total",
			Help:      "Counter of internal restricted sql.",
		})

	StatementPerTransaction = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "transaction_statement_num",
			Help:      "Bucketed histogram of statements count in each transaction.",
			Buckets:   prometheus.ExponentialBuckets(1, 2, 16), // 1 ~ 65536
		}, []string{LblSQLType, LblType})

	TransactionDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "transaction_duration_seconds",
			Help:      "Bucketed histogram of a transaction execution duration, including retry.",
			Buckets:   prometheus.ExponentialBuckets(0.001, 2, 20), // 1ms ~ 1049s
		}, []string{LblSQLType, LblType})

	StatementDeadlockDetectDuration = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "statement_deadlock_detect_duration_seconds",
			Help:      "Bucketed histogram of a statement deadlock detect duration.",
			Buckets:   prometheus.ExponentialBuckets(0.001, 2, 20), // 1ms ~ 1049s
		},
	)

	StatementPessimisticRetryCount = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "statement_pessimistic_retry_count",
			Help:      "Bucketed histogram of statement pessimistic retry count",
			Buckets:   prometheus.ExponentialBuckets(1, 1.5, 14), // 1 ~ 291
		})

	StatementLockKeysCount = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "statement_lock_keys_count",
			Help:      "Keys locking for a single statement",
			Buckets:   prometheus.ExponentialBuckets(1, 2, 21), // 1 ~ 2097152
		})
)

// Label constants.
const (
	LblUnretryable = "unretryable"
	LblReachMax    = "reach_max"
	LblOK          = "ok"
	LblError       = "error"
	LblCommit      = "commit"
	LblAbort       = "abort"
	LblRollback    = "rollback"
	LblType        = "type"
	LblDb          = "db"
	LblResult      = "result"
	LblSQLType     = "sql_type"
	LblGeneral     = "general"
	LblInternal    = "internal"
	LblStore       = "store"
	LblAddress     = "address"
)
<commit_msg>metrics: enlarge statement retry count upper limit (#16196)<commit_after>// Copyright 2018 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package metrics

import "github.com/prometheus/client_golang/prometheus"

// Session metrics.
var (
	SessionExecuteParseDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "parse_duration_seconds",
			Help:      "Bucketed histogram of processing time (s) in parse SQL.",
			Buckets:   prometheus.ExponentialBuckets(0.00004, 2, 22), // 40us ~ 168s
		}, []string{LblSQLType})
	SessionExecuteCompileDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "compile_duration_seconds",
			Help:      "Bucketed histogram of processing time (s) in query optimize.",
			// Build plan may execute the statement, or allocate table ID, so it might take a long time.
			Buckets: prometheus.ExponentialBuckets(0.00004, 2, 22), // 40us ~ 168s
		}, []string{LblSQLType})
	SessionExecuteRunDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "execute_duration_seconds",
			Help:      "Bucketed histogram of processing time (s) in running executor.",
			Buckets:   prometheus.ExponentialBuckets(0.0001, 2, 22), // 100us ~ 419s
		}, []string{LblSQLType})
	SchemaLeaseErrorCounter = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "schema_lease_error_total",
			Help:      "Counter of schema lease error",
		}, []string{LblType})
	SessionRetry = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "retry_num",
			Help:      "Bucketed histogram of session retry count.",
			Buckets:   prometheus.LinearBuckets(0, 1, 20), // 0 ~ 20
		})
	SessionRetryErrorCounter = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "retry_error_total",
			Help:      "Counter of session retry error.",
		}, []string{LblSQLType, LblType})

	SessionRestrictedSQLCounter = prometheus.NewCounter(
		prometheus.CounterOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "restricted_sql_total",
			Help:      "Counter of internal restricted sql.",
		})

	StatementPerTransaction = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "transaction_statement_num",
			Help:      "Bucketed histogram of statements count in each transaction.",
			Buckets:   prometheus.ExponentialBuckets(1, 2, 16), // 1 ~ 65536
		}, []string{LblSQLType, LblType})

	TransactionDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "transaction_duration_seconds",
			Help:      "Bucketed histogram of a transaction execution duration, including retry.",
			Buckets:   prometheus.ExponentialBuckets(0.001, 2, 20), // 1ms ~ 1049s
		}, []string{LblSQLType, LblType})

	StatementDeadlockDetectDuration = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "statement_deadlock_detect_duration_seconds",
			Help:      "Bucketed histogram of a statement deadlock detect duration.",
			Buckets:   prometheus.ExponentialBuckets(0.001, 2, 20), // 1ms ~ 1049s
		},
	)

	StatementPessimisticRetryCount = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "statement_pessimistic_retry_count",
			Help:      "Bucketed histogram of statement pessimistic retry count",
			Buckets:   prometheus.ExponentialBuckets(1, 2, 16), // 1 ~ 65536
		})

	StatementLockKeysCount = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Namespace: "tidb",
			Subsystem: "session",
			Name:      "statement_lock_keys_count",
			Help:      "Keys locking for a single statement",
			Buckets:   prometheus.ExponentialBuckets(1, 2, 21), // 1 ~ 2097152
		})
)

// Label constants.
const (
	LblUnretryable = "unretryable"
	LblReachMax    = "reach_max"
	LblOK          = "ok"
	LblError       = "error"
	LblCommit      = "commit"
	LblAbort       = "abort"
	LblRollback    = "rollback"
	LblType        = "type"
	LblDb          = "db"
	LblResult      = "result"
	LblSQLType     = "sql_type"
	LblGeneral     = "general"
	LblInternal    = "internal"
	LblStore       = "store"
	LblAddress     = "address"
)
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"net"
	"os"

	"github.com/karasz/gtclock/tai64"
)

const port = ":4014"

func checkError(err error) {
	if err != nil {
		fmt.Println("Error: ", err)
		os.Exit(0)
	}
}

func sendResponse(conn *net.UDPConn, addr *net.UDPAddr, b []byte) {
	s := []byte("s")
	copy(b[0:], s)
	copy(b[4:], tai64.TainPack(tai64.TainNow()))
	fmt.Println(tai64.TainUnpack(tai64.TainPack(tai64.TainNow())))
	_, err := conn.WriteToUDP(b, addr)
	if err != nil {
		fmt.Printf("Couldn't send response %v", err)
	}
}

func main() {
	ServAddr, err := net.ResolveUDPAddr("udp", port)
	checkError(err)
	ServConn, err := net.ListenUDP("udp", ServAddr)
	checkError(err)
	defer ServConn.Close()

	buf := make([]byte, 256)

	for {
		n, remoteaddr, err := ServConn.ReadFromUDP(buf)
		if err != nil {
			fmt.Printf("Error  %v", err)
			continue
		} else {
			if (n >= 20) && (bytes.Equal(buf[:4], []byte("ctai"))) {
				go sendResponse(ServConn, remoteaddr, buf)
			}
		}
	}
}
<commit_msg>gtclock: removed spurious Println.<commit_after>package main

import (
	"bytes"
	"fmt"
	"net"
	"os"

	"github.com/karasz/gtclock/tai64"
)

const port = ":4014"

func checkError(err error) {
	if err != nil {
		fmt.Println("Error: ", err)
		os.Exit(0)
	}
}

func sendResponse(conn *net.UDPConn, addr *net.UDPAddr, b []byte) {
	s := []byte("s")
	copy(b[0:], s)
	copy(b[4:], tai64.TainPack(tai64.TainNow()))
	_, err := conn.WriteToUDP(b, addr)
	if err != nil {
		fmt.Printf("Couldn't send response %v", err)
	}
}

func main() {
	ServAddr, err := net.ResolveUDPAddr("udp", port)
	checkError(err)
	ServConn, err := net.ListenUDP("udp", ServAddr)
	checkError(err)
	defer ServConn.Close()

	buf := make([]byte, 256)

	for {
		n, remoteaddr, err := ServConn.ReadFromUDP(buf)
		if err != nil {
			fmt.Printf("Error  %v", err)
			continue
		} else {
			if (n >= 20) && (bytes.Equal(buf[:4], []byte("ctai"))) {
				go sendResponse(ServConn, remoteaddr, buf)
			}
		}
	}
}
<|endoftext|>


<commit_before>package backuper

type AgentConfig struct {
	Destination Destination  `json:"destination"`
	TmpDir      string       `json:"tmp_dir"`
	Tasks       []TaskConfig `json:"tasks"`
	Period      Period       `json:"period"`
}

type Period struct {
	Type       string   `json:"type"`
	Time       string   `json:"time"`
	DaysOfWeek []string `json:"days_of_week"`
}

type Destination struct {
	Type   string            `json:"type"`
	Params map[string]string `json:"params"`
}

type TaskConfig struct {
	Id     string            `json:"id"`
	Type   string            `json:"type"`
	Name   string            `json:"name"`
	Params map[string]string `json:"params"`
}

type BackupResult struct {
	Prepare PathResult
	Lock    PathResult
	Backup  []BackupTaskResult
	Encrypt PathResult
	Upload  PathResult
	Unlock  PathResult
	Cleanup PathResult
}

type PathResult struct {
	Err    string
	Path   string
	Output string
}

type BackupTaskResult struct {
	PathResult
	TaskId string
}

func NewPathResult(err error, path, output string) (result PathResult) {
	result.Path = path
	result.Output = output
	if err != nil {
		result.Err = err.Error()
	}
	return
}
<commit_msg>Remove TaskConfig.Id<commit_after>package backuper

type AgentConfig struct {
	Destination Destination  `json:"destination"`
	TmpDir      string       `json:"tmp_dir"`
	Tasks       []TaskConfig `json:"tasks"`
	Period      Period       `json:"period"`
}

type Period struct {
	Type       string   `json:"type"`
	Time       string   `json:"time"`
	DaysOfWeek []string `json:"days_of_week"`
}

type Destination struct {
	Type   string            `json:"type"`
	Params map[string]string `json:"params"`
}

type TaskConfig struct {
	Type   string            `json:"type"`
	Name   string            `json:"name"`
	Params map[string]string `json:"params"`
}

type BackupResult struct {
	Prepare PathResult
	Lock    PathResult
	Backup  []BackupTaskResult
	Encrypt PathResult
	Upload  PathResult
	Unlock  PathResult
	Cleanup PathResult
}

type PathResult struct {
	Err    string
	Path   string
	Output string
}

type BackupTaskResult struct {
	PathResult
	TaskId string
}

func NewPathResult(err error, path, output string) (result PathResult) {
	result.Path = path
	result.Output = output
	if err != nil {
		result.Err = err.Error()
	}
	return
}
<|endoftext|>


<commit_before>package hawser

import (
	"fmt"
	"github.com/hawser/git-hawser/git"
	"github.com/rubyist/tracerx"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
)

const Version = "0.4.0"

var (
	LargeSizeThreshold = 5 * 1024 * 1024
	TempDir            = filepath.Join(os.TempDir(), "hawser")
	UserAgent          string
	LocalWorkingDir    string
	LocalGitDir        string
	LocalMediaDir      string
	LocalLogDir        string
	checkedTempDir     string
)

func TempFile(prefix string) (*os.File, error) {
	if checkedTempDir != TempDir {
		if err := os.MkdirAll(TempDir, 0774); err != nil {
			return nil, err
		}
		checkedTempDir = TempDir
	}

	return ioutil.TempFile(TempDir, prefix)
}

func ResetTempDir() error {
	checkedTempDir = ""
	return os.RemoveAll(TempDir)
}

func LocalMediaPath(sha string) (string, error) {
	path := filepath.Join(LocalMediaDir, sha[0:2], sha[2:4])
	if err := os.MkdirAll(path, 0744); err != nil {
		return "", fmt.Errorf("Error trying to create local media directory in '%s': %s", path, err)
	}

	return filepath.Join(path, sha), nil
}

func Environ() []string {
	osEnviron := os.Environ()
	env := make([]string, 4, len(osEnviron)+4)
	env[0] = fmt.Sprintf("LocalWorkingDir=%s", LocalWorkingDir)
	env[1] = fmt.Sprintf("LocalGitDir=%s", LocalGitDir)
	env[2] = fmt.Sprintf("LocalMediaDir=%s", LocalMediaDir)
	env[3] = fmt.Sprintf("TempDir=%s", TempDir)

	for _, e := range osEnviron {
		if !strings.Contains(e, "GIT_") {
			continue
		}
		env = append(env, e)
	}

	return env
}

func InRepo() bool {
	return LocalWorkingDir != ""
}

func init() {
	var err error

	tracerx.DefaultKey = "GIT"
	tracerx.Prefix = "trace hawser: "

	LocalWorkingDir, LocalGitDir, err = resolveGitDir()
	if err == nil {
		LocalMediaDir = filepath.Join(LocalGitDir, "hawser", "objects")
		LocalLogDir = filepath.Join(LocalMediaDir, "logs")
		TempDir = filepath.Join(LocalGitDir, "hawser", "tmp")

		if err := os.MkdirAll(LocalMediaDir, 0744); err != nil {
			panic(fmt.Errorf("Error trying to create objects directory in '%s': %s", LocalMediaDir, err))
		}

		if err := os.MkdirAll(LocalLogDir, 0744); err != nil {
			panic(fmt.Errorf("Error trying to create log directory in '%s': %s", LocalLogDir, err))
		}

		if err := os.MkdirAll(TempDir, 0744); err != nil {
			panic(fmt.Errorf("Error trying to create temp directory in '%s': %s", TempDir, err))
		}

	}

	gitVersion, err := git.Config.Version()
	if err != nil {
		gitVersion = "unknown"
	}

	UserAgent = fmt.Sprintf("git-hawser/%s (GitHub; %s %s; git %s; go %s)", Version,
		runtime.GOOS,
		runtime.GOARCH,
		strings.Replace(gitVersion, "git version ", "", 1),
		strings.Replace(runtime.Version(), "go", "", 1))
}

func resolveGitDir() (string, string, error) {
	wd, err := os.Getwd()
	if err != nil {
		return "", "", err
	}

	return recursiveResolveGitDir(wd)
}

func recursiveResolveGitDir(dir string) (string, string, error) {
	var cleanDir = filepath.Clean(dir)
	if cleanDir[len(cleanDir)-1] == os.PathSeparator {
		return "", "", fmt.Errorf("Git repository not found")
	}

	if filepath.Base(dir) == gitExt {
		return filepath.Dir(dir), dir, nil
	}

	gitDir := filepath.Join(dir, gitExt)
	if info, err := os.Stat(gitDir); err == nil {
		if info.IsDir() {
			return dir, gitDir, nil
		} else {
			return processDotGitFile(gitDir)
		}
	}

	return recursiveResolveGitDir(filepath.Dir(dir))
}

func processDotGitFile(file string) (string, string, error) {
	f, err := os.Open(file)
	defer f.Close()

	if err != nil {
		return "", "", err
	}

	data := make([]byte, 512)
	n, err := f.Read(data)
	if err != nil {
		return "", "", err
	}

	contents := string(data[0:n])
	wd, _ := os.Getwd()
	if strings.HasPrefix(contents, gitPtrPrefix) {
		dir := strings.TrimSpace(strings.Split(contents, gitPtrPrefix)[1])
		absDir, _ := filepath.Abs(dir)
		return wd, absDir, nil
	}

	return wd, "", nil
}

const (
	gitExt       = ".git"
	gitPtrPrefix = "gitdir: "
)
<commit_msg>ンンン ンンン ンン<commit_after>package hawser

import (
	"fmt"
	"github.com/hawser/git-hawser/git"
	"github.com/rubyist/tracerx"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
)

const Version = "0.4.1"

var (
	LargeSizeThreshold = 5 * 1024 * 1024
	TempDir            = filepath.Join(os.TempDir(), "hawser")
	UserAgent          string
	LocalWorkingDir    string
	LocalGitDir        string
	LocalMediaDir      string
	LocalLogDir        string
	checkedTempDir     string
)

func TempFile(prefix string) (*os.File, error) {
	if checkedTempDir != TempDir {
		if err := os.MkdirAll(TempDir, 0774); err != nil {
			return nil, err
		}
		checkedTempDir = TempDir
	}

	return ioutil.TempFile(TempDir, prefix)
}

func ResetTempDir() error {
	checkedTempDir = ""
	return os.RemoveAll(TempDir)
}

func LocalMediaPath(sha string) (string, error) {
	path := filepath.Join(LocalMediaDir, sha[0:2], sha[2:4])
	if err := os.MkdirAll(path, 0744); err != nil {
		return "", fmt.Errorf("Error trying to create local media directory in '%s': %s", path, err)
	}

	return filepath.Join(path, sha), nil
}

func Environ() []string {
	osEnviron := os.Environ()
	env := make([]string, 4, len(osEnviron)+4)
	env[0] = fmt.Sprintf("LocalWorkingDir=%s", LocalWorkingDir)
	env[1] = fmt.Sprintf("LocalGitDir=%s", LocalGitDir)
	env[2] = fmt.Sprintf("LocalMediaDir=%s", LocalMediaDir)
	env[3] = fmt.Sprintf("TempDir=%s", TempDir)

	for _, e := range osEnviron {
		if !strings.Contains(e, "GIT_") {
			continue
		}
		env = append(env, e)
	}

	return env
}

func InRepo() bool {
	return LocalWorkingDir != ""
}

func init() {
	var err error

	tracerx.DefaultKey = "GIT"
	tracerx.Prefix = "trace hawser: "

	LocalWorkingDir, LocalGitDir, err = resolveGitDir()
	if err == nil {
		LocalMediaDir = filepath.Join(LocalGitDir, "hawser", "objects")
		LocalLogDir = filepath.Join(LocalMediaDir, "logs")
		TempDir = filepath.Join(LocalGitDir, "hawser", "tmp")

		if err := os.MkdirAll(LocalMediaDir, 0744); err != nil {
			panic(fmt.Errorf("Error trying to create objects directory in '%s': %s", LocalMediaDir, err))
		}

		if err := os.MkdirAll(LocalLogDir, 0744); err != nil {
			panic(fmt.Errorf("Error trying to create log directory in '%s': %s", LocalLogDir, err))
		}

		if err := os.MkdirAll(TempDir, 0744); err != nil {
			panic(fmt.Errorf("Error trying to create temp directory in '%s': %s", TempDir, err))
		}

	}

	gitVersion, err := git.Config.Version()
	if err != nil {
		gitVersion = "unknown"
	}

	UserAgent = fmt.Sprintf("git-hawser/%s (GitHub; %s %s; git %s; go %s)", Version,
		runtime.GOOS,
		runtime.GOARCH,
		strings.Replace(gitVersion, "git version ", "", 1),
		strings.Replace(runtime.Version(), "go", "", 1))
}

func resolveGitDir() (string, string, error) {
	wd, err := os.Getwd()
	if err != nil {
		return "", "", err
	}

	return recursiveResolveGitDir(wd)
}

func recursiveResolveGitDir(dir string) (string, string, error) {
	var cleanDir = filepath.Clean(dir)
	if cleanDir[len(cleanDir)-1] == os.PathSeparator {
		return "", "", fmt.Errorf("Git repository not found")
	}

	if filepath.Base(dir) == gitExt {
		return filepath.Dir(dir), dir, nil
	}

	gitDir := filepath.Join(dir, gitExt)
	if info, err := os.Stat(gitDir); err == nil {
		if info.IsDir() {
			return dir, gitDir, nil
		} else {
			return processDotGitFile(gitDir)
		}
	}

	return recursiveResolveGitDir(filepath.Dir(dir))
}

func processDotGitFile(file string) (string, string, error) {
	f, err := os.Open(file)
	defer f.Close()

	if err != nil {
		return "", "", err
	}

	data := make([]byte, 512)
	n, err := f.Read(data)
	if err != nil {
		return "", "", err
	}

	contents := string(data[0:n])
	wd, _ := os.Getwd()
	if strings.HasPrefix(contents, gitPtrPrefix) {
		dir := strings.TrimSpace(strings.Split(contents, gitPtrPrefix)[1])
		absDir, _ := filepath.Abs(dir)
		return wd, absDir, nil
	}

	return wd, "", nil
}

const (
	gitExt       = ".git"
	gitPtrPrefix = "gitdir: "
)
<|endoftext|>


<commit_before>package helpers

import (
    "encoding/base64"
    "fmt"
    "strconv"
    "strings"
    "golang.org/x/text/unicode/norm"
)

// BtoA is a polyfill for javascript's window#btoa()
func BtoA(s string) string {
    b64 := base64.URLEncoding.WithPadding(base64.NoPadding)
    src := []byte(s)
    buf := make([]byte, b64.EncodedLen(len(src)))
    b64.Encode(buf, src)

    return string(buf)
}

// DrawTable draws a fancy ASCII table
// Inspired by MySQL
func DrawTable(headers []string, rows [][]string) string {
    // Whether we hit discord's limit or not
    contentsOmitted := false
    rowsPrinted := 0

    // Result container
    sb := ""

    // Determine biggest padding for each col
    // First headers, then rows
    paddings := make([]int, len(headers))

    for idx, header := range headers {
        if paddings[idx] < len(header) {
            paddings[idx] = len(header)
        }
    }

    for _, row := range rows {
        for cidx, col := range row {
            if paddings[cidx] < len(col) {
                paddings[cidx] = len(col)
            }
        }
    }

    // Make this a code tag
    sb += "```\n"

    // Draw header
    sb += drawLine("+", "-", "+", paddings, headers)
    sb += drawContent("|", "|", "|", paddings, headers)
    sb += drawLine("+", "-", "+", paddings, headers)

    // Draw content
    for _, row := range rows {
        // If we're about to hit discord's limit print ... to indicate there's more we can't show
        if len(sb) >= 1600 {
            contentsOmitted = true

            dummyRow := make([]string, len(headers))
            for idx := range dummyRow {
                dummyRow[idx] = "..."
            }

            sb += drawContent("|", "|", "|", paddings, dummyRow)
            break
        }

        // Else print row
        rowsPrinted++
        sb += drawContent("|", "|", "|", paddings, row)
    }

    // Draw bottom border
    sb += drawLine("+", "-", "+", paddings, headers)

    // If we hit discord's limit let the user know
    if contentsOmitted {
        rowCount := len(rows) - rowsPrinted
        sb += strconv.Itoa(rowCount)

        if rowCount == 1 {
            sb += " row"
        } else {
            sb += " rows"
        }

        sb += " omitted because of discord's message size limit.\n"
    }

    // End code tag
    sb += "```"

    return sb
}

// drawLine draws a line with given paddings and chars (eg "+-----+-----+-----+")
func drawLine(start string, mid string, end string, paddings []int, data []string) string {
    sb := ""
    for idx := range data {
        sb += start + strings.Repeat(mid, paddings[idx])
    }
    sb += end + "\n"

    return sb
}

// drawContent draws content with padding and custom separators (eg "|A    |B    |C    |")
func drawContent(start string, separator string, end string, paddings []int, data []string) string {
    sb := ""
    for idx, content := range data {
        if idx == 0 {
            sb += start
        } else {
            sb += separator
        }

        sb += fmt.Sprintf("%-" + strconv.Itoa(paddings[idx]) + "s", content)
    }
    sb += end + "\n"

    return sb
}
<commit_msg>Add some unicode magic<commit_after>package helpers

import (
    "encoding/base64"
    "fmt"
    "strconv"
    "strings"
    "golang.org/x/text/unicode/norm"
    "regexp"
)

// BtoA is a polyfill for javascript's window#btoa()
func BtoA(s string) string {
    b64 := base64.URLEncoding.WithPadding(base64.NoPadding)
    src := []byte(s)
    buf := make([]byte, b64.EncodedLen(len(src)))
    b64.Encode(buf, src)

    return string(buf)
}

// DrawTable draws a fancy ASCII table
// Inspired by MySQL
func DrawTable(headers []string, rows [][]string) string {
    // Whether we hit discord's limit or not
    contentsOmitted := false
    rowsPrinted := 0

    // Result container
    sb := ""

    // Determine biggest padding for each col
    // First headers, then rows
    paddings := make([]int, len(headers))

    for idx, header := range headers {
        if paddings[idx] < len(header) {
            paddings[idx] = len(header)
        }
    }

    for _, row := range rows {
        for cidx, col := range row {
            tmp := norm.NFC.String(col)
            length := len(tmp)

            if paddings[cidx] < length {
                paddings[cidx] = length
            }
        }
    }

    // Make this a code tag
    sb += "```\n"

    // Draw header
    sb += drawLine("+", "-", "+", paddings, headers)
    sb += drawContent("|", "|", "|", paddings, headers)
    sb += drawLine("+", "-", "+", paddings, headers)

    // Draw content
    for _, row := range rows {
        // If we're about to hit discord's limit print ... to indicate there's more we can't show
        if len(sb) >= 1600 {
            contentsOmitted = true

            dummyRow := make([]string, len(headers))
            for idx := range dummyRow {
                dummyRow[idx] = "..."
            }

            sb += drawContent("|", "|", "|", paddings, dummyRow)
            break
        }

        // Else print row
        rowsPrinted++
        sb += drawContent("|", "|", "|", paddings, row)
    }

    // Draw bottom border
    sb += drawLine("+", "-", "+", paddings, headers)

    // If we hit discord's limit let the user know
    if contentsOmitted {
        rowCount := len(rows) - rowsPrinted
        sb += strconv.Itoa(rowCount)

        if rowCount == 1 {
            sb += " row"
        } else {
            sb += " rows"
        }

        sb += " omitted because of discord's message size limit.\n"
    }

    // End code tag
    sb += "```"

    return sb
}

// drawLine draws a line with given paddings and chars (eg "+-----+-----+-----+")
func drawLine(start string, mid string, end string, paddings []int, data []string) string {
    sb := ""
    for idx := range data {
        sb += start + strings.Repeat(mid, paddings[idx])
    }
    sb += end + "\n"

    return sb
}

// drawContent draws content with padding and custom separators (eg "|A    |B    |C    |")
func drawContent(start string, separator string, end string, paddings []int, data []string) string {
    sanitizer := regexp.MustCompile(
        `[\r\n\t\f\v\x{2028}\x{2029}]+`,
    )
    unifier := regexp.MustCompile(
        `[` +
            `\x{0020}\x{00A0}\x{1680}\x{180E}` +
            `\x{2000}\x{2001}\x{2002}\x{2003}` +
            `\x{2004}\x{2005}\x{2006}\x{2007}` +
            `\x{2008}\x{2009}\x{200A}\x{200B}` +
            `\x{202F}\x{205F}\x{3000}\x{FEFF}` +
            `\x{2423}\x{2422}\x{2420}` + // "visible" spaces
            `]+`,
    )

    sb := ""
    for idx, content := range data {
        if idx == 0 {
            sb += start
        } else {
            sb += separator
        }

        content = norm.NFC.String(content)
        content = sanitizer.ReplaceAllString(content, "")
        content = unifier.ReplaceAllString(content, " ")
        sb += fmt.Sprintf(
            "%-" + strconv.Itoa(paddings[idx]) + "s",
            content,
        )
    }

    sb += end + "\n"

    return sb
}
<|endoftext|>


<commit_before>package bittorrent

import (
	"errors"
	"net/url"
	"strconv"
	"strings"

	"github.com/chihaya/chihaya/pkg/log"
)

// Params is used to fetch (optional) request parameters from an Announce.
// For HTTP Announces this includes the request path and parsed query, for UDP
// Announces this is the extracted path and parsed query from optional URLData
// as specified in BEP41.
//
// See ParseURLData for specifics on parsing and limitations.
type Params interface {
	// String returns a string parsed from a query. Every key can be
	// returned as a string because they are encoded in the URL as strings.
	String(key string) (string, bool)

	// RawPath returns the raw path from the request URL.
	// The path returned can contain URL encoded data.
	// For a request of the form "/announce?port=1234" this would return
	// "/announce".
	RawPath() string

	// RawQuery returns the raw query from the request URL, excluding the
	// delimiter '?'.
	// For a request of the form "/announce?port=1234" this would return
	// "port=1234"
	RawQuery() string
}

// ErrKeyNotFound is returned when a provided key has no value associated with
// it.
var ErrKeyNotFound = errors.New("query: value for the provided key does not exist")

// ErrInvalidInfohash is returned when parsing a query encounters an infohash
// with invalid length.
var ErrInvalidInfohash = ClientError("provided invalid infohash")

// ErrInvalidQueryEscape is returned when a query string contains invalid
// escapes.
var ErrInvalidQueryEscape = ClientError("invalid query escape")

// QueryParams parses a URL Query and implements the Params interface with some
// additional helpers.
type QueryParams struct {
	path       string
	query      string
	params     map[string]string
	infoHashes []InfoHash
}

type routeParamsKey struct{}

// RouteParamsKey is a key for the context of a request that
// contains the named parameters from the http router
var RouteParamsKey = routeParamsKey{}

// RouteParam is a type that contains the values from the named parameters
// on the route
type RouteParam struct {
	Key   string
	Value string
}

// RouteParams is a collection of RouteParam instances
type RouteParams []RouteParam

// ParseURLData parses a request URL or UDP URLData as defined in BEP41.
// It expects a concatenated string of the request's path and query parts as
// defined in RFC 3986. As both the udp: and http: scheme used by BitTorrent
// include an authority part the path part must always begin with a slash.
// An example of the expected URLData would be "/announce?port=1234&uploaded=0"
// or "/?auth=0x1337".
// HTTP servers should pass (*http.Request).RequestURI, UDP servers should
// pass the concatenated, unchanged URLData as defined in BEP41.
//
// Note that, in the case of a key occurring multiple times in the query, only
// the last value for that key is kept.
// The only exception to this rule is the key "info_hash" which will attempt to
// parse each value as an InfoHash and return an error if parsing fails. All
// InfoHashes are collected and can later be retrieved by calling the InfoHashes
// method.
//
// Also note that any error that is encountered during parsing is returned as a
// ClientError, as this method is expected to be used to parse client-provided
// data.
func ParseURLData(urlData string) (*QueryParams, error) {
	var path, query string

	queryDelim := strings.IndexAny(urlData, "?")
	if queryDelim == -1 {
		path = urlData
	} else {
		path = urlData[:queryDelim]
		query = urlData[queryDelim+1:]
	}

	q, err := parseQuery(query)
	if err != nil {
		return nil, ClientError(err.Error())
	}
	q.path = path
	return q, nil
}

// parseQuery parses a URL query into QueryParams.
// The query is expected to exclude the delimiting '?'.
func parseQuery(query string) (q *QueryParams, err error) {
	// This is basically url.parseQuery, but with a map[string]string
	// instead of map[string][]string for the values.
	q = &QueryParams{
		query:      query,
		infoHashes: nil,
		params:     make(map[string]string),
	}

	for query != "" {
		key := query
		if i := strings.IndexAny(key, "&;"); i >= 0 {
			key, query = key[:i], key[i+1:]
		} else {
			query = ""
		}
		if key == "" {
			continue
		}
		value := ""
		if i := strings.Index(key, "="); i >= 0 {
			key, value = key[:i], key[i+1:]
		}
		key, err = url.QueryUnescape(key)
		if err != nil {
			// QueryUnescape returns an error like "invalid escape: '%x'".
			// But frontends record these errors to prometheus, which generates
			// a lot of time series.
			// We log it here for debugging instead.
			log.Debug("failed to unescape query param key", log.Err(err))
			return nil, ErrInvalidQueryEscape
		}
		value, err = url.QueryUnescape(value)
		if err != nil {
			// QueryUnescape returns an error like "invalid escape: '%x'".
			// But frontends record these errors to prometheus, which generates
			// a lot of time series.
			// We log it here for debugging instead.
			log.Debug("failed to unescape query param value", log.Err(err))
			return nil, ErrInvalidQueryEscape
		}

		if key == "info_hash" {
			if len(value) != 20 {
				return nil, ErrInvalidInfohash
			}
			q.infoHashes = append(q.infoHashes, InfoHashFromString(value))
		} else {
			q.params[strings.ToLower(key)] = value
		}
	}

	return q, nil
}

// String returns a string parsed from a query. Every key can be returned as a
// string because they are encoded in the URL as strings.
func (qp *QueryParams) String(key string) (string, bool) {
	value, ok := qp.params[key]
	return value, ok
}

// Uint64 returns a uint parsed from a query. After being called, it is safe to
// cast the uint64 to your desired length.
func (qp *QueryParams) Uint64(key string) (uint64, error) {
	str, exists := qp.params[key]
	if !exists {
		return 0, ErrKeyNotFound
	}

	val, err := strconv.ParseUint(str, 10, 64)
	if err != nil {
		return 0, err
	}

	return val, nil
}

// InfoHashes returns a list of requested infohashes.
func (qp *QueryParams) InfoHashes() []InfoHash {
	return qp.infoHashes
}

// RawPath returns the raw path from the parsed URL.
func (qp *QueryParams) RawPath() string {
	return qp.path
}

// RawQuery returns the raw query from the parsed URL.
func (qp *QueryParams) RawQuery() string {
	return qp.query
}
<commit_msg>correct godoc comments to include period<commit_after>package bittorrent

import (
	"errors"
	"net/url"
	"strconv"
	"strings"

	"github.com/chihaya/chihaya/pkg/log"
)

// Params is used to fetch (optional) request parameters from an Announce.
// For HTTP Announces this includes the request path and parsed query, for UDP
// Announces this is the extracted path and parsed query from optional URLData
// as specified in BEP41.
//
// See ParseURLData for specifics on parsing and limitations.
type Params interface {
	// String returns a string parsed from a query. Every key can be
	// returned as a string because they are encoded in the URL as strings.
	String(key string) (string, bool)

	// RawPath returns the raw path from the request URL.
	// The path returned can contain URL encoded data.
	// For a request of the form "/announce?port=1234" this would return
	// "/announce".
	RawPath() string

	// RawQuery returns the raw query from the request URL, excluding the
	// delimiter '?'.
	// For a request of the form "/announce?port=1234" this would return
	// "port=1234"
	RawQuery() string
}

// ErrKeyNotFound is returned when a provided key has no value associated with
// it.
var ErrKeyNotFound = errors.New("query: value for the provided key does not exist")

// ErrInvalidInfohash is returned when parsing a query encounters an infohash
// with invalid length.
var ErrInvalidInfohash = ClientError("provided invalid infohash")

// ErrInvalidQueryEscape is returned when a query string contains invalid
// escapes.
var ErrInvalidQueryEscape = ClientError("invalid query escape")

// QueryParams parses a URL Query and implements the Params interface with some
// additional helpers.
type QueryParams struct {
	path       string
	query      string
	params     map[string]string
	infoHashes []InfoHash
}

type routeParamsKey struct{}

// RouteParamsKey is a key for the context of a request that
// contains the named parameters from the http router.
var RouteParamsKey = routeParamsKey{}

// RouteParam is a type that contains the values from the named parameters
// on the route.
type RouteParam struct {
	Key   string
	Value string
}

// RouteParams is a collection of RouteParam instances.
type RouteParams []RouteParam

// ParseURLData parses a request URL or UDP URLData as defined in BEP41.
// It expects a concatenated string of the request's path and query parts as
// defined in RFC 3986. As both the udp: and http: scheme used by BitTorrent
// include an authority part the path part must always begin with a slash.
// An example of the expected URLData would be "/announce?port=1234&uploaded=0"
// or "/?auth=0x1337".
// HTTP servers should pass (*http.Request).RequestURI, UDP servers should
// pass the concatenated, unchanged URLData as defined in BEP41.
//
// Note that, in the case of a key occurring multiple times in the query, only
// the last value for that key is kept.
// The only exception to this rule is the key "info_hash" which will attempt to
// parse each value as an InfoHash and return an error if parsing fails. All
// InfoHashes are collected and can later be retrieved by calling the InfoHashes
// method.
//
// Also note that any error that is encountered during parsing is returned as a
// ClientError, as this method is expected to be used to parse client-provided
// data.
func ParseURLData(urlData string) (*QueryParams, error) {
	var path, query string

	queryDelim := strings.IndexAny(urlData, "?")
	if queryDelim == -1 {
		path = urlData
	} else {
		path = urlData[:queryDelim]
		query = urlData[queryDelim+1:]
	}

	q, err := parseQuery(query)
	if err != nil {
		return nil, ClientError(err.Error())
	}
	q.path = path
	return q, nil
}

// parseQuery parses a URL query into QueryParams.
// The query is expected to exclude the delimiting '?'.
func parseQuery(query string) (q *QueryParams, err error) {
	// This is basically url.parseQuery, but with a map[string]string
	// instead of map[string][]string for the values.
	q = &QueryParams{
		query:      query,
		infoHashes: nil,
		params:     make(map[string]string),
	}

	for query != "" {
		key := query
		if i := strings.IndexAny(key, "&;"); i >= 0 {
			key, query = key[:i], key[i+1:]
		} else {
			query = ""
		}
		if key == "" {
			continue
		}
		value := ""
		if i := strings.Index(key, "="); i >= 0 {
			key, value = key[:i], key[i+1:]
		}
		key, err = url.QueryUnescape(key)
		if err != nil {
			// QueryUnescape returns an error like "invalid escape: '%x'".
			// But frontends record these errors to prometheus, which generates
			// a lot of time series.
			// We log it here for debugging instead.
			log.Debug("failed to unescape query param key", log.Err(err))
			return nil, ErrInvalidQueryEscape
		}
		value, err = url.QueryUnescape(value)
		if err != nil {
			// QueryUnescape returns an error like "invalid escape: '%x'".
			// But frontends record these errors to prometheus, which generates
			// a lot of time series.
			// We log it here for debugging instead.
			log.Debug("failed to unescape query param value", log.Err(err))
			return nil, ErrInvalidQueryEscape
		}

		if key == "info_hash" {
			if len(value) != 20 {
				return nil, ErrInvalidInfohash
			}
			q.infoHashes = append(q.infoHashes, InfoHashFromString(value))
		} else {
			q.params[strings.ToLower(key)] = value
		}
	}

	return q, nil
}

// String returns a string parsed from a query. Every key can be returned as a
// string because they are encoded in the URL as strings.
func (qp *QueryParams) String(key string) (string, bool) {
	value, ok := qp.params[key]
	return value, ok
}

// Uint64 returns a uint parsed from a query. After being called, it is safe to
// cast the uint64 to your desired length.
func (qp *QueryParams) Uint64(key string) (uint64, error) {
	str, exists := qp.params[key]
	if !exists {
		return 0, ErrKeyNotFound
	}

	val, err := strconv.ParseUint(str, 10, 64)
	if err != nil {
		return 0, err
	}

	return val, nil
}

// InfoHashes returns a list of requested infohashes.
func (qp *QueryParams) InfoHashes() []InfoHash {
	return qp.infoHashes
}

// RawPath returns the raw path from the parsed URL.
func (qp *QueryParams) RawPath() string {
	return qp.path
}

// RawQuery returns the raw query from the parsed URL.
func (qp *QueryParams) RawQuery() string {
	return qp.query
}
<|endoftext|>


<commit_before>package hosts

import (
	"github.com/mackerelio/mkr/mackerelclient"
	cli "gopkg.in/urfave/cli.v1"
)

// Command is definition of mkr hosts subcommand
var Command = cli.Command{
	Name:      "hosts",
	Usage:     "List hosts",
	ArgsUsage: "[--verbose | -v] [--name | -n <name>] [--service | -s <service>] [[--role | -r <role>]...] [[--status | --st <status>]...]",
	Description: `
    List the information of the hosts refined by host name, service name, role name and/or status.
    Requests "GET /api/v0/hosts.json". See https://mackerel.io/api-docs/entry/hosts#list .
`,
	Action: doHosts,
	Flags: []cli.Flag{
		cli.StringFlag{Name: "name, n", Value: "", Usage: "List hosts only matched with <name>"},
		cli.StringFlag{Name: "service, s", Value: "", Usage: "List hosts only belonging to <service>"},
		cli.StringSliceFlag{
			Name:  "role, r",
			Value: &cli.StringSlice{},
			Usage: "List hosts only belonging to <role>. Multiple choices are allowed. Required --service",
		},
		cli.StringSliceFlag{
			Name:  "status, st",
			Value: &cli.StringSlice{},
			Usage: "List hosts only matched <status>. Multiple choices are allowed.",
		},
		cli.StringFlag{Name: "format, f", Value: "", Usage: "Output format template"},
		cli.BoolFlag{Name: "verbose, v", Usage: "Verbose output mode"},
	},
}

func doHosts(c *cli.Context) error {
	cli, err := mackerelclient.New(c.GlobalString("conf"), c.GlobalString("apibase"))
	if err != nil {
		return err
	}

	return (&hostApp{
		cli: cli,

		verbose: c.Bool("verbose"),

		name:     c.String("name"),
		service:  c.String("service"),
		roles:    c.StringSlice("role"),
		statuses: c.StringSlice("statuses"),

		format: c.String("format"),
	}).run()
}
<commit_msg>[hosts] fix status argument<commit_after>package hosts

import (
	"github.com/mackerelio/mkr/mackerelclient"
	cli "gopkg.in/urfave/cli.v1"
)

// Command is definition of mkr hosts subcommand
var Command = cli.Command{
	Name:      "hosts",
	Usage:     "List hosts",
	ArgsUsage: "[--verbose | -v] [--name | -n <name>] [--service | -s <service>] [[--role | -r <role>]...] [[--status | --st <status>]...]",
	Description: `
    List the information of the hosts refined by host name, service name, role name and/or status.
    Requests "GET /api/v0/hosts.json". See https://mackerel.io/api-docs/entry/hosts#list .
`,
	Action: doHosts,
	Flags: []cli.Flag{
		cli.StringFlag{Name: "name, n", Value: "", Usage: "List hosts only matched with <name>"},
		cli.StringFlag{Name: "service, s", Value: "", Usage: "List hosts only belonging to <service>"},
		cli.StringSliceFlag{
			Name:  "role, r",
			Value: &cli.StringSlice{},
			Usage: "List hosts only belonging to <role>. Multiple choices are allowed. Required --service",
		},
		cli.StringSliceFlag{
			Name:  "status, st",
			Value: &cli.StringSlice{},
			Usage: "List hosts only matched <status>. Multiple choices are allowed.",
		},
		cli.StringFlag{Name: "format, f", Value: "", Usage: "Output format template"},
		cli.BoolFlag{Name: "verbose, v", Usage: "Verbose output mode"},
	},
}

func doHosts(c *cli.Context) error {
	cli, err := mackerelclient.New(c.GlobalString("conf"), c.GlobalString("apibase"))
	if err != nil {
		return err
	}

	return (&hostApp{
		cli: cli,

		verbose: c.Bool("verbose"),

		name:     c.String("name"),
		service:  c.String("service"),
		roles:    c.StringSlice("role"),
		statuses: c.StringSlice("status"),

		format: c.String("format"),
	}).run()
}
<|endoftext|>


<commit_before>package html

import (
	"code.google.com/p/go.net/html"
	"errors"
	"io"
	"regexp"
	"unicode"
)

const (
	// We remember a few special node types when descending into their
	// children.
	AncestorArticle = 1 << iota
	AncestorAside
	AncestorBlockquote
	AncestorList
)

var (
	ignorePattern = regexp.MustCompile("(?i)comment|caption|credit|header|foot|blq-dotcom|story-feature")
)

type Document struct {
	Title  *Chunk   // the <title>...</title> text
	Chunks []*Chunk // list of all chunks found in this document

	// Unexported fields.
	root *html.Node // the ... part
	head *html.Node // the ... part
	body *html.Node // the ... part

	// State variables used when collectiong chunks.
	ancestors int // bitmask which stores ancestor of the current node

	// Number of non-space characters inside link tags / normal tags
	// per html.ElementNode.
	linkText map[*html.Node]int // length of text inside  tags
	normText map[*html.Node]int // length of text outside  tags
}

func NewDocument(r io.Reader) (*Document, error) {
	doc := new(Document)
	doc.Chunks = make([]*Chunk, 0, 512)
	if err := doc.Parse(r); err != nil {
		return nil, err
	}
	return doc, nil
}

func (doc *Document) Parse(r io.Reader) error {
	root, err := html.Parse(r)
	if err != nil {
		return err
	}
	// Assign the fields root, head and body from the HTML page.
	doc.setNodes(root)

	// Check if ,  and  nodes were found.
	if doc.root == nil || doc.head == nil || doc.body == nil {
		return errors.New("Document missing ,  or .")
	}

	doc.parseHead(doc.head)

	// No title found? The title plays a crucial role in detecting
	// the article content. We need it and every page should contain it.
	// So skip parsing in case we could not find a title.
	if doc.Title == nil {
		return errors.New("Document missing <title>.")
	}
	doc.linkText = make(map[*html.Node]int)
	doc.normText = make(map[*html.Node]int)

	doc.cleanBody(doc.body, 0)
	doc.countText(doc.body, false)
	doc.parseBody(doc.body)

	// Now link the chunks.
	for i := range doc.Chunks {
		if i > 0 {
			doc.Chunks[i].Prev = doc.Chunks[i-1]
		}
		if i < len(doc.Chunks)-1 {
			doc.Chunks[i].Next = doc.Chunks[i+1]
		}
	}
	return nil
}

// Assign the struct fields root, head and body from the HTML tree of node n.
//  doc.root ->
//  doc.head ->
//  doc.body ->
func (doc *Document) setNodes(n *html.Node) {
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		switch c.Data {
		case "html":
			doc.root = c
			doc.setNodes(c)
		case "body":
			doc.body = c
		case "head":
			doc.head = c
		}
	}
}

// parseHead parses the ... part of the HTML page. Right now it
// only detects the <title>...</title>.
func (doc *Document) parseHead(n *html.Node) {
	if n.Type == html.ElementNode && n.Data == "title" {
		if chunk, err := NewChunk(doc, n); err == nil {
			doc.Title = chunk
		}
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		doc.parseHead(c)
	}
}

// countText counts the link text and the normal text per html.Node.
// "Link text" is text inside  tags and "normal text" is text inside
// anything but  tags. Of course, counting is done cumulative, so the
// numbers of a parent node include the numbers of it's child nodes.
func (doc *Document) countText(n *html.Node, insideLink bool) (linkText int, normText int) {
	linkText = 0
	normText = 0
	if n.Type == html.ElementNode && n.Data == "a" {
		insideLink = true
	}
	for s := n.FirstChild; s != nil; s = s.NextSibling {
		linkTextChild, normTextChild := doc.countText(s, insideLink)
		linkText += linkTextChild
		normText += normTextChild
	}
	if n.Type == html.TextNode {
		count := 0
		for _, rune := range n.Data {
			if unicode.IsLetter(rune) {
				count += 1
			}
		}
		if insideLink {
			linkText += count
		} else {
			normText += count
		}
	}
	doc.linkText[n] = linkText
	doc.normText[n] = normText
	return
}

// cleanBody removes unwanted HTML elements from the HTML body.
func (doc *Document) cleanBody(n *html.Node, level int) {
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if c.Type != html.ElementNode {
			continue
		}
		switch c.Data {
		// Elements save to ignore.
		case "address", "audio", "button", "canvas", "caption", "fieldset",
			"figcaption", "figure", "footer", "form", "frame", "header", "iframe",
			"map", "menu", "nav", "noscript", "object", "option", "output",
			"script", "select", "style", "svg", "textarea", "video":
			n.RemoveChild(c)
		// High-level tables might be used to layout the document, so we better
		// not ignore them.
		case "table":
			if level > 5 {
				n.RemoveChild(c)
			}
		default:
			doc.cleanBody(c, level+1)
		}
	}
}

// parseBody parses the ... part of the HTML page. It creates
// Chunks for every html.TextNode found in the body.
func (doc *Document) parseBody(n *html.Node) {
	switch n.Type {
	case html.ElementNode:
		// We ignore the node if it has some nasty classes/ids/itemprobs.
		for _, attr := range n.Attr {
			switch attr.Key {
			case "id", "class", "itemprop":
				if ignorePattern.FindStringIndex(attr.Val) != nil {
					return
				}
			}
		}
		ancestorMask := 0
		switch n.Data {
		// We convert headings and links to text immediately. This is easier
		// and feasible because headings and links don't contain many children.
		// Descending into these children and handling every TextNode separately
		// would make things unnecessary complicated and our results noisy.
		case "h1", "h2", "h3", "h4", "h5", "h6", "a":
			if chunk, err := NewChunk(doc, n); err == nil {
				doc.Chunks = append(doc.Chunks, chunk)
			}
			return
		// Now mask the element type, but only if it isn't already set.
		// If we mask a bit which was already set by one of our callers, we'd also
		// clear it at the of this function, though it actually should be cleared
		// by the caller.
		case "article":
			ancestorMask = AncestorArticle &^ doc.ancestors
		case "aside":
			ancestorMask = AncestorAside &^ doc.ancestors
		case "blockquote":
			ancestorMask = AncestorBlockquote &^ doc.ancestors
		case "ul", "ol":
			ancestorMask = AncestorList &^ doc.ancestors
		}
		// Add our mask to the ancestor bitmask.
		doc.ancestors |= ancestorMask
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			doc.parseBody(c)
		}
		// Remove our mask from the ancestor bitmask.
		doc.ancestors &^= ancestorMask
	case html.TextNode:
		if chunk, err := NewChunk(doc, n); err == nil {
			doc.Chunks = append(doc.Chunks, chunk)
		}
	}
}

type TextStat struct {
	Words     int
	Sentences int
	Count     int
}

// GetClassStats groups the document chunks by their classes (defined by the
// class attribute of HTML nodes) and calculates TextStats for each class.
func (doc *Document) GetClassStats() map[string]*TextStat {
	result := make(map[string]*TextStat)
	for _, chunk := range doc.Chunks {
		for _, class := range chunk.Classes {
			if stat, ok := result[class]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				result[class] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
		}
	}
	return result
}

// GetClusterStats groups the document chunks by common ancestors and
// calculates TextStats for each group of chunks.
func (doc *Document) GetClusterStats() map[*Chunk]*TextStat {
	// Don't ascend further than this constant.
	const maxAncestors = 3

	// Count TextStats for Chunk ancestors.
	ancestorStat := make(map[*html.Node]*TextStat)
	for _, chunk := range doc.Chunks {
		node, count := chunk.Block, 0
		for node != nil && count < maxAncestors {
			if stat, ok := ancestorStat[node]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				ancestorStat[node] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
			node, count = node.Parent, count+1
		}
	}

	// Generate result.
	result := make(map[*Chunk]*TextStat)
	for _, chunk := range doc.Chunks {
		node := chunk.Block
		if node == nil {
			continue
		}
		// Start with the parent's TextStat. Then ascend and check if the
		// current chunk has an ancestor with better stats. Use the best stat
		// as result.
		stat := ancestorStat[node]
		for {
			if node = node.Parent; node == nil {
				break
			}
			if statPrev, ok := ancestorStat[node]; ok {
				if stat.Count < statPrev.Count {
					stat = statPrev
				}
			} else {
				break
			}
		}
		result[chunk] = stat
	}
	return result
}
<commit_msg>fixed bug exiting loop too early<commit_after>package html

import (
	"code.google.com/p/go.net/html"
	"errors"
	"io"
	"regexp"
	"unicode"
)

const (
	// We remember a few special node types when descending into their
	// children.
	AncestorArticle = 1 << iota
	AncestorAside
	AncestorBlockquote
	AncestorList
)

var (
	ignorePattern = regexp.MustCompile("(?i)comment|caption|credit|header|foot|blq-dotcom|story-feature")
)

type Document struct {
	Title  *Chunk   // the <title>...</title> text
	Chunks []*Chunk // list of all chunks found in this document

	// Unexported fields.
	root *html.Node // the ... part
	head *html.Node // the ... part
	body *html.Node // the ... part

	// State variables used when collectiong chunks.
	ancestors int // bitmask which stores ancestor of the current node

	// Number of non-space characters inside link tags / normal tags
	// per html.ElementNode.
	linkText map[*html.Node]int // length of text inside  tags
	normText map[*html.Node]int // length of text outside  tags
}

func NewDocument(r io.Reader) (*Document, error) {
	doc := new(Document)
	doc.Chunks = make([]*Chunk, 0, 512)
	if err := doc.Parse(r); err != nil {
		return nil, err
	}
	return doc, nil
}

func (doc *Document) Parse(r io.Reader) error {
	root, err := html.Parse(r)
	if err != nil {
		return err
	}
	// Assign the fields root, head and body from the HTML page.
	doc.setNodes(root)

	// Check if ,  and  nodes were found.
	if doc.root == nil || doc.head == nil || doc.body == nil {
		return errors.New("Document missing ,  or .")
	}

	doc.parseHead(doc.head)

	// No title found? The title plays a crucial role in detecting
	// the article content. We need it and every page should contain it.
	// So skip parsing in case we could not find a title.
	if doc.Title == nil {
		return errors.New("Document missing <title>.")
	}
	doc.linkText = make(map[*html.Node]int)
	doc.normText = make(map[*html.Node]int)

	doc.cleanBody(doc.body, 0)
	doc.countText(doc.body, false)
	doc.parseBody(doc.body)

	// Now link the chunks.
	for i := range doc.Chunks {
		if i > 0 {
			doc.Chunks[i].Prev = doc.Chunks[i-1]
		}
		if i < len(doc.Chunks)-1 {
			doc.Chunks[i].Next = doc.Chunks[i+1]
		}
	}
	return nil
}

// Assign the struct fields root, head and body from the HTML tree of node n.
//  doc.root ->
//  doc.head ->
//  doc.body ->
func (doc *Document) setNodes(n *html.Node) {
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		switch c.Data {
		case "html":
			doc.root = c
			doc.setNodes(c)
		case "body":
			doc.body = c
		case "head":
			doc.head = c
		}
	}
}

// parseHead parses the ... part of the HTML page. Right now it
// only detects the <title>...</title>.
func (doc *Document) parseHead(n *html.Node) {
	if n.Type == html.ElementNode && n.Data == "title" {
		if chunk, err := NewChunk(doc, n); err == nil {
			doc.Title = chunk
		}
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		doc.parseHead(c)
	}
}

// countText counts the link text and the normal text per html.Node.
// "Link text" is text inside  tags and "normal text" is text inside
// anything but  tags. Of course, counting is done cumulative, so the
// numbers of a parent node include the numbers of it's child nodes.
func (doc *Document) countText(n *html.Node, insideLink bool) (linkText int, normText int) {
	linkText = 0
	normText = 0
	if n.Type == html.ElementNode && n.Data == "a" {
		insideLink = true
	}
	for s := n.FirstChild; s != nil; s = s.NextSibling {
		linkTextChild, normTextChild := doc.countText(s, insideLink)
		linkText += linkTextChild
		normText += normTextChild
	}
	if n.Type == html.TextNode {
		count := 0
		for _, rune := range n.Data {
			if unicode.IsLetter(rune) {
				count += 1
			}
		}
		if insideLink {
			linkText += count
		} else {
			normText += count
		}
	}
	doc.linkText[n] = linkText
	doc.normText[n] = normText
	return
}

// cleanBody removes unwanted HTML elements from the HTML body.
func (doc *Document) cleanBody(n *html.Node, level int) {
	var curr *html.Node = n.FirstChild
	var next *html.Node = nil
	for ; curr != nil; curr = next {
		// We have to remember the next sibling here becase calling RemoveChild
		// sets curr's NextSibling pointer to nil and we would quit the loop
		// prematurely.
		next = curr.NextSibling
		if curr.Type != html.ElementNode {
			continue
		}
		switch curr.Data {
		// Elements save to ignore.
		case "address", "audio", "button", "canvas", "caption", "fieldset",
			"figcaption", "figure", "footer", "form", "frame", "header", "iframe",
			"map", "menu", "nav", "noscript", "object", "option", "output",
			"script", "select", "style", "svg", "textarea", "video":
			n.RemoveChild(curr)
		// High-level tables might be used to layout the document, so we better
		// not ignore them.
		case "table":
			if level > 5 {
				n.RemoveChild(curr)
			}
		default:
			doc.cleanBody(curr, level+1)
		}
	}
}

// parseBody parses the ... part of the HTML page. It creates
// Chunks for every html.TextNode found in the body.
func (doc *Document) parseBody(n *html.Node) {
	switch n.Type {
	case html.ElementNode:
		// We ignore the node if it has some nasty classes/ids/itemprobs.
		for _, attr := range n.Attr {
			switch attr.Key {
			case "id", "class", "itemprop":
				if ignorePattern.FindStringIndex(attr.Val) != nil {
					return
				}
			}
		}
		ancestorMask := 0
		switch n.Data {
		// We convert headings and links to text immediately. This is easier
		// and feasible because headings and links don't contain many children.
		// Descending into these children and handling every TextNode separately
		// would make things unnecessary complicated and our results noisy.
		case "h1", "h2", "h3", "h4", "h5", "h6", "a":
			if chunk, err := NewChunk(doc, n); err == nil {
				doc.Chunks = append(doc.Chunks, chunk)
			}
			return
		// Now mask the element type, but only if it isn't already set.
		// If we mask a bit which was already set by one of our callers, we'd also
		// clear it at the end of this function, though it actually should be cleared
		// by the caller.
		case "article":
			ancestorMask = AncestorArticle &^ doc.ancestors
		case "aside":
			ancestorMask = AncestorAside &^ doc.ancestors
		case "blockquote":
			ancestorMask = AncestorBlockquote &^ doc.ancestors
		case "ul", "ol":
			ancestorMask = AncestorList &^ doc.ancestors
		}
		// Add our mask to the ancestor bitmask.
		doc.ancestors |= ancestorMask
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			doc.parseBody(c)
		}
		// Remove our mask from the ancestor bitmask.
		doc.ancestors &^= ancestorMask
	case html.TextNode:
		if chunk, err := NewChunk(doc, n); err == nil {
			doc.Chunks = append(doc.Chunks, chunk)
		}
	}
}

type TextStat struct {
	Words     int
	Sentences int
	Count     int
}

// GetClassStats groups the document chunks by their classes (defined by the
// class attribute of HTML nodes) and calculates TextStats for each class.
func (doc *Document) GetClassStats() map[string]*TextStat {
	result := make(map[string]*TextStat)
	for _, chunk := range doc.Chunks {
		for _, class := range chunk.Classes {
			if stat, ok := result[class]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				result[class] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
		}
	}
	return result
}

// GetClusterStats groups the document chunks by common ancestors and
// calculates TextStats for each group of chunks.
func (doc *Document) GetClusterStats() map[*Chunk]*TextStat {
	// Don't ascend further than this constant.
	const maxAncestors = 3

	// Count TextStats for Chunk ancestors.
	ancestorStat := make(map[*html.Node]*TextStat)
	for _, chunk := range doc.Chunks {
		node, count := chunk.Block, 0
		for node != nil && count < maxAncestors {
			if stat, ok := ancestorStat[node]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				ancestorStat[node] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
			node, count = node.Parent, count+1
		}
	}

	// Generate result.
	result := make(map[*Chunk]*TextStat)
	for _, chunk := range doc.Chunks {
		node := chunk.Block
		if node == nil {
			continue
		}
		// Start with the parent's TextStat. Then ascend and check if the
		// current chunk has an ancestor with better stats. Use the best stat
		// as result.
		stat := ancestorStat[node]
		for {
			if node = node.Parent; node == nil {
				break
			}
			if statPrev, ok := ancestorStat[node]; ok {
				if stat.Count < statPrev.Count {
					stat = statPrev
				}
			} else {
				break
			}
		}
		result[chunk] = stat
	}
	return result
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"strings"

	"github.com/Unknwon/com"
	docker "github.com/fsouza/go-dockerclient"
	"github.com/gorilla/mux"
)

func RouteIndex(ctx *Context) error {
	return executeTemplate(ctx, "install.html", 200, map[string]interface{}{
		"Section": "install",
	})
}

func execInContainer(client *docker.Client,
	id string,
	command []string) (string, string, error) {

	exec, err := client.CreateExec(docker.CreateExecOptions{
		AttachStderr: true,
		AttachStdin:  false,
		AttachStdout: true,
		Tty:          false,
		Cmd:          command,
		Container:    id,
	})

	if err != nil {
		return "", "", err
	}

	var outputBytes []byte
	outputWriter := bytes.NewBuffer(outputBytes)
	var errorBytes []byte
	errorWriter := bytes.NewBuffer(errorBytes)

	err = client.StartExec(exec.ID, docker.StartExecOptions{
		OutputStream: outputWriter,
		ErrorStream:  errorWriter,
	})

	return outputWriter.String(), errorWriter.String(), err
}

func RouteApiV1CodetainerTTY(ctx *Context) error {
	if ctx.R.Method == "POST" {
		return RouteApiV1CodetainerUpdateCurrentTTY(ctx)
	} else {
		return RouteApiV1CodetainerGetCurrentTTY(ctx)
	}
}

func RouteApiV1CodetainerUpdateCurrentTTY(ctx *Context) error {
	vars := mux.Vars(ctx.R)
	id := vars["id"]
	if id == "" {
		return errors.New("id is required")
	}

	endpoint := GlobalConfig.GetDockerEndpoint()
	client, err := docker.NewClient(endpoint)
	if err != nil {
		return err
	}

	height := com.StrTo(ctx.R.FormValue("height")).MustInt()

	if height == 0 {
		return errors.New("height is required")
	}

	width := com.StrTo(ctx.R.FormValue("width")).MustInt()

	if width == 0 {
		return errors.New("width is required")
	}

	err = client.ResizeContainerTTY(id, height, width)
	Log.Info("ERROR", err, height, width, " ", id)
	return err
}

//
// Get TTY size
//
func RouteApiV1CodetainerGetCurrentTTY(ctx *Context) error {

	vars := mux.Vars(ctx.R)
	id := vars["id"]
	if id == "" {
		return errors.New("id is required")
	}

	endpoint := GlobalConfig.GetDockerEndpoint()
	client, err := docker.NewClient(endpoint)
	if err != nil {
		return err
	}
	col, _, err := execInContainer(client, id, []string{"tput", "cols"})
	col = strings.Trim(col, "\n")
	if err != nil {
		return err
	}
	lines, _, err := execInContainer(client, id, []string{"tput", "lines"})
	lines = strings.Trim(lines, "\n")
	if err != nil {
		return err
	}

	return renderJson(map[string]interface{}{
		"col":  col,
		"rows": lines,
	}, ctx.W)

}

//
// Stop a codetainer
//
func RouteApiV1CodetainerStop(ctx *Context) error {

	if ctx.R.Method != "POST" {
		return errors.New("POST only")
	}

	vars := mux.Vars(ctx.R)
	id := vars["id"]
	endpoint := GlobalConfig.GetDockerEndpoint()
	client, err := docker.NewClient(endpoint)
	if err != nil {
		return err
	}

	err = client.StopContainer(id, 30)

	if err != nil {
		return err
	}

	return nil
}

//
// List files in a codetainer
//
func RouteApiV1CodetainerListFiles(ctx *Context) error {

	vars := mux.Vars(ctx.R)
	id := vars["id"]
	if id == "" {
		return errors.New("id is required")
	}

	path := ctx.R.FormValue("path")
	if path == "" {
		return errors.New("path is required")
	}

	endpoint := GlobalConfig.GetDockerEndpoint()
	client, err := docker.NewClient(endpoint)
	if err != nil {
		return err
	}

	exec, err := client.CreateExec(docker.CreateExecOptions{
		AttachStderr: true,
		AttachStdin:  false,
		AttachStdout: true,
		Tty:          false,
		Cmd:          []string{"ls", path},
		Container:    id,
	})

	if err != nil {
		return err
	}

	var outputBytes []byte
	outputWriter := bytes.NewBuffer(outputBytes)
	var errorBytes []byte
	errorWriter := bytes.NewBuffer(errorBytes)

	// TODO fetch config for codetainer
	err = client.StartExec(exec.ID, docker.StartExecOptions{
		OutputStream: outputWriter,
		ErrorStream:  errorWriter,
	})

	if err != nil {
		return err
	}

	files := strings.Split(outputWriter.String(), "\n")

	// TODO: parse into string
	return renderJson(map[string]interface{}{
		"files": files,
		"error": errorWriter.String(),
	}, ctx.W)

}

//
// Start a stopped codetainer
//
func RouteApiV1CodetainerStart(ctx *Context) error {

	if ctx.R.Method != "POST" {
		return errors.New("POST only")
	}
	vars := mux.Vars(ctx.R)
	id := vars["id"]
	endpoint := GlobalConfig.GetDockerEndpoint()
	client, err := docker.NewClient(endpoint)
	if err != nil {
		return err
	}

	// TODO fetch config for codetainer
	err = client.StartContainer(id, &docker.HostConfig{})

	if err != nil {
		return err
	}

	return nil
}

//
// List all running codetainers
//
func RouteApiV1CodetainerList(ctx *Context) error {
	endpoint := GlobalConfig.GetDockerEndpoint()
	client, err := docker.NewClient(endpoint)
	if err != nil {
		return err
	}
	containers, err := client.ListContainers(docker.ListContainersOptions{})

	if err != nil {
		return err
	}
	return renderJson(map[string]interface{}{
		"containers": containers,
	}, ctx.W)
}

//
// Attach to a codetainer
//
func RouteApiV1CodetainerAttach(ctx *Context) error {
	vars := mux.Vars(ctx.R)
	id := vars["id"]

	if id == "" {
		return errors.New("ID of container must be provided")
	}

	if ctx.WS == nil {
		return errors.New("No websocket connection for web client")
	}

	connection := &ContainerConnection{id: id, web: ctx.WS}

	connection.Start()

	return nil
}

//
// View codetainer
//
func RouteApiV1CodetainerView(ctx *Context) error {
	vars := mux.Vars(ctx.R)
	id := vars["id"]

	if id == "" {
		return errors.New("ID of container must be provided")
	}

	return executeRaw(ctx, "view.html", 200, map[string]interface{}{
		"Section":             "ContainerView",
		"PageIsContainerView": true,
		"ContainerId":         id,
	})
}
<commit_msg>add return value<commit_after>package main

import (
	"bytes"
	"errors"
	"strings"

	"github.com/Unknwon/com"
	docker "github.com/fsouza/go-dockerclient"
	"github.com/gorilla/mux"
)

func RouteIndex(ctx *Context) error {
	return executeTemplate(ctx, "install.html", 200, map[string]interface{}{
		"Section": "install",
	})
}

func execInContainer(client *docker.Client,
	id string,
	command []string) (string, string, error) {

	exec, err := client.CreateExec(docker.CreateExecOptions{
		AttachStderr: true,
		AttachStdin:  false,
		AttachStdout: true,
		Tty:          false,
		Cmd:          command,
		Container:    id,
	})

	if err != nil {
		return "", "", err
	}

	var outputBytes []byte
	outputWriter := bytes.NewBuffer(outputBytes)
	var errorBytes []byte
	errorWriter := bytes.NewBuffer(errorBytes)

	err = client.StartExec(exec.ID, docker.StartExecOptions{
		OutputStream: outputWriter,
		ErrorStream:  errorWriter,
	})

	return outputWriter.String(), errorWriter.String(), err
}

func RouteApiV1CodetainerTTY(ctx *Context) error {
	if ctx.R.Method == "POST" {
		return RouteApiV1CodetainerUpdateCurrentTTY(ctx)
	} else {
		return RouteApiV1CodetainerGetCurrentTTY(ctx)
	}
}

func RouteApiV1CodetainerUpdateCurrentTTY(ctx *Context) error {
	vars := mux.Vars(ctx.R)
	id := vars["id"]
	if id == "" {
		return errors.New("id is required")
	}

	endpoint := GlobalConfig.GetDockerEndpoint()
	client, err := docker.NewClient(endpoint)
	if err != nil {
		return err
	}

	height := com.StrTo(ctx.R.FormValue("height")).MustInt()

	if height == 0 {
		return errors.New("height is required")
	}

	width := com.StrTo(ctx.R.FormValue("width")).MustInt()

	if width == 0 {
		return errors.New("width is required")
	}

	err = client.ResizeContainerTTY(id, height, width)

	if err != nil {
		return err
	}
	return renderJson(map[string]interface{}{
		"success": true,
	}, ctx.W)
}

//
// Get TTY size
//
func RouteApiV1CodetainerGetCurrentTTY(ctx *Context) error {

	vars := mux.Vars(ctx.R)
	id := vars["id"]
	if id == "" {
		return errors.New("id is required")
	}

	endpoint := GlobalConfig.GetDockerEndpoint()
	client, err := docker.NewClient(endpoint)
	if err != nil {
		return err
	}
	col, _, err := execInContainer(client, id, []string{"tput", "cols"})
	col = strings.Trim(col, "\n")
	if err != nil {
		return err
	}
	lines, _, err := execInContainer(client, id, []string{"tput", "lines"})
	lines = strings.Trim(lines, "\n")
	if err != nil {
		return err
	}

	return renderJson(map[string]interface{}{
		"col":  col,
		"rows": lines,
	}, ctx.W)

}

//
// Stop a codetainer
//
func RouteApiV1CodetainerStop(ctx *Context) error {

	if ctx.R.Method != "POST" {
		return errors.New("POST only")
	}

	vars := mux.Vars(ctx.R)
	id := vars["id"]
	endpoint := GlobalConfig.GetDockerEndpoint()
	client, err := docker.NewClient(endpoint)
	if err != nil {
		return err
	}

	err = client.StopContainer(id, 30)

	if err != nil {
		return err
	}

	return nil
}

//
// List files in a codetainer
//
func RouteApiV1CodetainerListFiles(ctx *Context) error {

	vars := mux.Vars(ctx.R)
	id := vars["id"]
	if id == "" {
		return errors.New("id is required")
	}

	path := ctx.R.FormValue("path")
	if path == "" {
		return errors.New("path is required")
	}

	endpoint := GlobalConfig.GetDockerEndpoint()
	client, err := docker.NewClient(endpoint)
	if err != nil {
		return err
	}

	exec, err := client.CreateExec(docker.CreateExecOptions{
		AttachStderr: true,
		AttachStdin:  false,
		AttachStdout: true,
		Tty:          false,
		Cmd:          []string{"ls", path},
		Container:    id,
	})

	if err != nil {
		return err
	}

	var outputBytes []byte
	outputWriter := bytes.NewBuffer(outputBytes)
	var errorBytes []byte
	errorWriter := bytes.NewBuffer(errorBytes)

	// TODO fetch config for codetainer
	err = client.StartExec(exec.ID, docker.StartExecOptions{
		OutputStream: outputWriter,
		ErrorStream:  errorWriter,
	})

	if err != nil {
		return err
	}

	files := strings.Split(outputWriter.String(), "\n")

	// TODO: parse into string
	return renderJson(map[string]interface{}{
		"files": files,
		"error": errorWriter.String(),
	}, ctx.W)

}

//
// Start a stopped codetainer
//
func RouteApiV1CodetainerStart(ctx *Context) error {

	if ctx.R.Method != "POST" {
		return errors.New("POST only")
	}
	vars := mux.Vars(ctx.R)
	id := vars["id"]
	endpoint := GlobalConfig.GetDockerEndpoint()
	client, err := docker.NewClient(endpoint)
	if err != nil {
		return err
	}

	// TODO fetch config for codetainer
	err = client.StartContainer(id, &docker.HostConfig{})

	if err != nil {
		return err
	}

	return nil
}

//
// List all running codetainers
//
func RouteApiV1CodetainerList(ctx *Context) error {
	endpoint := GlobalConfig.GetDockerEndpoint()
	client, err := docker.NewClient(endpoint)
	if err != nil {
		return err
	}
	containers, err := client.ListContainers(docker.ListContainersOptions{})

	if err != nil {
		return err
	}
	return renderJson(map[string]interface{}{
		"containers": containers,
	}, ctx.W)
}

//
// Attach to a codetainer
//
func RouteApiV1CodetainerAttach(ctx *Context) error {
	vars := mux.Vars(ctx.R)
	id := vars["id"]

	if id == "" {
		return errors.New("ID of container must be provided")
	}

	if ctx.WS == nil {
		return errors.New("No websocket connection for web client")
	}

	connection := &ContainerConnection{id: id, web: ctx.WS}

	connection.Start()

	return nil
}

//
// View codetainer
//
func RouteApiV1CodetainerView(ctx *Context) error {
	vars := mux.Vars(ctx.R)
	id := vars["id"]

	if id == "" {
		return errors.New("ID of container must be provided")
	}

	return executeRaw(ctx, "view.html", 200, map[string]interface{}{
		"Section":             "ContainerView",
		"PageIsContainerView": true,
		"ContainerId":         id,
	})
}
<|endoftext|>


<commit_before>package gitter

import (
	"github.com/esoui/lexicon/bot"
	gitter "github.com/sromku/go-gitter"
	"log"
	"os"
)

type message struct {
	room   string
	text   string
	sender string
}

func (m *message) Text() string {
	return m.text
}

func (m *message) Sender() string {
	return m.sender
}

type adapter struct {
	name     string
	api      *gitter.Gitter
	incoming chan *message
	user *gitter.User
}

func New(name string) *adapter {
	a := &adapter{
		name: name,
		api:  gitter.New(os.Getenv("GITTER_TOKEN")),
		incoming: make(chan *message),
	}
	var err error
	a.user, err = a.api.GetUser()
	if err != nil {
		log.Println("Error on getting API user")
	}
	go a.Listen("588506b2d73408ce4f45407f")
	return a
}

func (a *adapter) Listen(room string) {
	stream := a.api.Stream(room)
	go a.api.Listen(stream)
	log.Printf("Gitter is now listening to room %s\n", room)
	for {
		event := <-stream.Event
		switch e := event.Data.(type) {
		case *gitter.MessageReceived:
			if e.Message.From.ID == a.user.ID {
				continue
			}
			m := &message{
				sender: e.Message.From.Username,
				text:   e.Message.Text,
				room:   room,
			}
			a.incoming <- m
		case *gitter.GitterConnectionClosed:
			log.Println("Stream connection closed")
			return
		}
	}
}

func (a *adapter) Receive() bot.Message {
	m := <-a.incoming
	log.Printf("Message received %+v\n", m)
	return m
}

func (a *adapter) Reply(m bot.Message, reply string) {
	n, _ := m.(*message)
	a.api.SendMessage(n.room, reply)
}
<commit_msg>Go fmt<commit_after>package gitter

import (
	"github.com/esoui/lexicon/bot"
	gitter "github.com/sromku/go-gitter"
	"log"
	"os"
)

type message struct {
	room   string
	text   string
	sender string
}

func (m *message) Text() string {
	return m.text
}

func (m *message) Sender() string {
	return m.sender
}

type adapter struct {
	name     string
	api      *gitter.Gitter
	incoming chan *message
	user     *gitter.User
}

func New(name string) *adapter {
	a := &adapter{
		name:     name,
		api:      gitter.New(os.Getenv("GITTER_TOKEN")),
		incoming: make(chan *message),
	}
	var err error
	a.user, err = a.api.GetUser()
	if err != nil {
		log.Println("Error on getting API user")
	}
	go a.Listen("588506b2d73408ce4f45407f")
	return a
}

func (a *adapter) Listen(room string) {
	stream := a.api.Stream(room)
	go a.api.Listen(stream)
	log.Printf("Gitter is now listening to room %s\n", room)
	for {
		event := <-stream.Event
		switch e := event.Data.(type) {
		case *gitter.MessageReceived:
			if e.Message.From.ID == a.user.ID {
				continue
			}
			m := &message{
				sender: e.Message.From.Username,
				text:   e.Message.Text,
				room:   room,
			}
			a.incoming <- m
		case *gitter.GitterConnectionClosed:
			log.Println("Stream connection closed")
			return
		}
	}
}

func (a *adapter) Receive() bot.Message {
	m := <-a.incoming
	log.Printf("Message received %+v\n", m)
	return m
}

func (a *adapter) Reply(m bot.Message, reply string) {
	n, _ := m.(*message)
	a.api.SendMessage(n.room, reply)
}
<|endoftext|>