<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logger

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"sync"
	"time"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
	pzsyslog "github.com/venicegeo/pz-gocommon/syslog"
)

type Service struct {
	sync.Mutex

	stats  Stats
	origin string
	async  bool

	logWriter   pzsyslog.Writer
	auditWriter pzsyslog.Writer

	esIndex elasticsearch.IIndex
}

func (service *Service) Init(sys *piazza.SystemConfig, logWriter pzsyslog.Writer, auditWriter pzsyslog.Writer, esi elasticsearch.IIndex, asyncLogging bool) error {
	service.stats.CreatedOn = time.Now()
	service.stats.NumMessagesByApplication = map[string]int{}

	service.logWriter = logWriter
	service.auditWriter = auditWriter

	service.esIndex = esi

	service.origin = string(sys.Name)

	service.async = asyncLogging

	return nil
}

func (service *Service) newInternalErrorResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusInternalServerError,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) newBadRequestResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusBadRequest,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) incrementStats(application string) {
	service.Lock()
	service.stats.NumMessages++
	service.stats.NumMessagesByApplication[application]++
	service.Unlock()
}

func (service *Service) GetRoot() *piazza.JsonResponse {
	resp := &piazza.JsonResponse{
		StatusCode: 200,
		Data:       "Hi. I'm pz-logger.",
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetStats() *piazza.JsonResponse {
	service.Lock()
	t := service.stats
	service.Unlock()

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       t,
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func createQueryDslAsString(
	pagination *piazza.JsonPagination,
	params *piazza.HttpQueryParams) (string, error) {

	must := []map[string]interface{}{}

	service, err := params.GetAsString("service", "")
	if err != nil {
		return "", err
	}

	contains, err := params.GetAsString("contains", "")
	if err != nil {
		return "", err
	}

	before, err := params.GetBefore(time.Time{})
	if err != nil {
		return "", err
	}

	after, err := params.GetAfter(time.Time{})
	if err != nil {
		return "", err
	}

	if service != "" {
		must = append(must, map[string]interface{}{
			"match": map[string]interface{}{
				"application": service,
			},
		})
	}
	if contains != "" {
		should := []map[string]interface{}{}
		should = append(should, map[string]interface{}{
			"query": map[string]interface{}{
				"wildcard": map[string]interface{}{
					"hostName": map[string]interface{}{
						"value": "*" + contains + "*",
					},
				},
			},
		})
		should = append(should, map[string]interface{}{
			"query": map[string]interface{}{
				"wildcard": map[string]interface{}{
					"application": map[string]interface{}{
						"value": "*" + contains + "*",
					},
				},
			},
		})
		should = append(should, map[string]interface{}{
			"query": map[string]interface{}{
				"wildcard": map[string]interface{}{
					"process": map[string]interface{}{
						"value": "*" + contains + "*",
					},
				},
			},
		})
		should = append(should, map[string]interface{}{
			"query": map[string]interface{}{
				"wildcard": map[string]interface{}{
					"messageId": map[string]interface{}{
						"value": "*" + contains + "*",
					},
				},
			},
		})
		should = append(should, map[string]interface{}{
			"query": map[string]interface{}{
				"wildcard": map[string]interface{}{
					"message": map[string]interface{}{
						"value": "*" + contains + "*",
					},
				},
			},
		})
		queries := make([]interface{}, len(should))
		for i, s := range should {
			queries[i] = s
		}
		must = append(must, map[string]interface{}{
			"filtered": map[string]interface{}{
				"query": map[string]interface{}{
					"match_all": map[string]interface{}{},
				},
				"filter": map[string]interface{}{
					"bool": map[string]interface{}{
						"should": queries,
					},
				},
			},
		})
	}

	if !after.IsZero() || !before.IsZero() {
		rangeParams := map[string]time.Time{}

		if !after.IsZero() {
			rangeParams["gte"] = after
		}

		if !before.IsZero() {
			rangeParams["lte"] = before
		}

		must = append(must, map[string]interface{}{
			"range": map[string]interface{}{
				"timeStamp": rangeParams,
			},
		})
	}

	if len(must) == 0 {
		return "", nil
	}

	dsl := map[string]interface{}{
		"query": map[string]interface{}{
			"filtered": map[string]interface{}{
				"query": map[string]interface{}{
					"bool": map[string]interface{}{
						"must": must,
					},
				},
			},
		},
		"size": pagination.PerPage,
		"from": pagination.PerPage * pagination.Page,
	}

	dsl["sort"] = map[string]string{
		pagination.SortBy: string(pagination.Order),
	}

	output, err := json.Marshal(dsl)
	if err != nil {
		return "", err
	}

	return string(output), nil
}

func (service *Service) PostSyslog(mNew *pzsyslog.Message) *piazza.JsonResponse {
	err := mNew.Validate()
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	err = service.postSyslog(mNew)
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	service.incrementStats(mNew.Application)

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
	}
	return resp
}

func (service *Service) postSyslog(mssg *pzsyslog.Message) error {
	var err error
	isAudit := mssg.AuditData != nil

	if service.logWriter != nil {
		err = service.logWriter.Write(mssg, service.async)
		if err != nil {
			return fmt.Errorf("syslog.Service.postSyslog: %s", err.Error())
		}
	}

	if isAudit && service.auditWriter != nil {
		err = service.auditWriter.Write(mssg, service.async)
		if err != nil {
			return fmt.Errorf("syslog.Service.postSyslog (audit): %s", err.Error())
		}
	}

	return nil
}

func (service *Service) getMessageCommon(params *piazza.HttpQueryParams) (*elasticsearch.SearchResult, *piazza.JsonPagination, *piazza.JsonResponse) {
	pagination, err := piazza.NewJsonPagination(params)
	if err != nil {
		return nil, nil, service.newBadRequestResponse(err)
	}

	paginationCreatedOnToTimeStamp(pagination)

	dsl, err := createQueryDslAsString(pagination, params)
	if err != nil {
		return nil, pagination, service.newBadRequestResponse(err)
	}

	var searchResult *elasticsearch.SearchResult

	if dsl == "" {
		searchResult, err = service.esIndex.FilterByMatchAll(pzsyslog.LoggerType, pagination)
	} else {
		searchResult, err = service.esIndex.SearchByJSON(pzsyslog.LoggerType, dsl)
	}
	if err != nil {
		return nil, pagination, service.newInternalErrorResponse(err)
	}

	return searchResult, pagination, nil
}

func (service *Service) GetSyslog(params *piazza.HttpQueryParams) *piazza.JsonResponse {
	var err error

	format, err := params.GetAsString("format", "json")
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	searchResult, pagination, jErr := service.getMessageCommon(params)
	if jErr != nil {
		return jErr
	}

	lines, err := extractFromSearchResult(searchResult)
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	var data interface{} = lines

	if format == "string" {
		t := make([]string, len(lines))
		for i := 0; i < len(lines); i++ {
			t[i] = lines[i].String()
		}
		data = t
	}

	pagination.Count = int(searchResult.TotalHits())
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       data,
		Pagination: pagination,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func extractFromSearchResult(searchResult *elasticsearch.SearchResult) ([]pzsyslog.Message, error) {

	var lines = make([]pzsyslog.Message, 0, len(*searchResult.GetHits()))

	if searchResult == nil || searchResult.GetHits() == nil {
		return lines, nil
	}

	for _, hit := range *searchResult.GetHits() {
		if hit.Source == nil {
			log.Printf("null source hit")
			continue
		}

		msg := pzsyslog.Message{}
		err := json.Unmarshal(*hit.Source, &msg)
		if err != nil {
			log.Printf("UNABLE TO PARSE: %s", string(*hit.Source))
			continue
		}

		// just in case
		/*err = msg.Validate()
		if err != nil {
			log.Printf("UNABLE TO VALIDATE: %s", string(*hit.Source))
			continue
		}*/

		lines = append(lines, msg)
	}

	return lines, nil
}

func (service *Service) PostQuery(params *piazza.HttpQueryParams, jsnQuery string) *piazza.JsonResponse {

	format, err := piazza.NewJsonPagination(params)
	if err != nil {
		return service.newBadRequestResponse(err)
	}
	paginationCreatedOnToTimeStamp(format)

	if jsnQuery, err = syncPagination(jsnQuery, *format); err != nil {
		return service.newBadRequestResponse(err)
	}

	searchResult, err := service.esIndex.SearchByJSON(pzsyslog.LoggerType, jsnQuery)
	fmt.Println(searchResult, err)
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	lines, err := extractFromSearchResult(searchResult)
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	format.Count = int(searchResult.TotalHits())
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       lines,
		Pagination: format,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func syncPagination(dslString string, format piazza.JsonPagination) (string, error) {
	// Overwrite any from/size in dsl with what's in the params
	b := []byte(dslString)
	var f interface{}
	err := json.Unmarshal(b, &f)
	if err != nil {
		return "", err
	}
	dsl := f.(map[string]interface{})
	dsl["from"] = format.Page * format.PerPage
	dsl["size"] = format.PerPage
	if dsl["sort"] == nil {
		// Since ES has more fine grained sorting allow their sorting to take precedence
		// If sorting wasn't specified in the DSL, put in sorting from Piazza
		bts := []byte("[{\"" + format.SortBy + "\":\"" + string(format.Order) + "\"}]")
		var g interface{}
		if err = json.Unmarshal(bts, &g); err != nil {
			return "", err
		}
		sortDsl := g.([]interface{})
		dsl["sort"] = sortDsl
	}
	byteArray, err := json.Marshal(dsl)
	if err != nil {
		return "", err
	}
	return string(byteArray), nil
}
<commit_msg>Update Service.go<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logger

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"sync"
	"time"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
	pzsyslog "github.com/venicegeo/pz-gocommon/syslog"
)

type Service struct {
	sync.Mutex

	stats  Stats
	origin string
	async  bool

	logWriter   pzsyslog.Writer
	auditWriter pzsyslog.Writer

	esIndex elasticsearch.IIndex
}

func (service *Service) Init(sys *piazza.SystemConfig, logWriter pzsyslog.Writer, auditWriter pzsyslog.Writer, esi elasticsearch.IIndex, asyncLogging bool) error {
	service.stats.CreatedOn = time.Now()
	service.stats.NumMessagesByApplication = map[string]int{}

	service.logWriter = logWriter
	service.auditWriter = auditWriter

	service.esIndex = esi

	service.origin = string(sys.Name)

	service.async = asyncLogging

	return nil
}

func (service *Service) newInternalErrorResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusInternalServerError,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) newBadRequestResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusBadRequest,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) incrementStats(application string) {
	service.Lock()
	service.stats.NumMessages++
	service.stats.NumMessagesByApplication[application]++
	service.Unlock()
}

func (service *Service) GetRoot() *piazza.JsonResponse {
	resp := &piazza.JsonResponse{
		StatusCode: 200,
		Data:       "Hi. I'm pz-logger.",
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetStats() *piazza.JsonResponse {
	service.Lock()
	t := service.stats
	service.Unlock()

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       t,
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func createQueryDslAsString(
	pagination *piazza.JsonPagination,
	params *piazza.HttpQueryParams) (string, error) {

	must := []map[string]interface{}{}

	service, err := params.GetAsString("service", "")
	if err != nil {
		return "", err
	}

	contains, err := params.GetAsString("contains", "")
	if err != nil {
		return "", err
	}

	before, err := params.GetBefore(time.Time{})
	if err != nil {
		return "", err
	}

	after, err := params.GetAfter(time.Time{})
	if err != nil {
		return "", err
	}

	if service != "" {
		must = append(must, map[string]interface{}{
			"match": map[string]interface{}{
				"application": service,
			},
		})
	}
	if contains != "" {
		should := []map[string]interface{}{}
		should = append(should, map[string]interface{}{
			"query": map[string]interface{}{
				"wildcard": map[string]interface{}{
					"hostName": map[string]interface{}{
						"value": "*" + contains + "*",
					},
				},
			},
		})
		should = append(should, map[string]interface{}{
			"query": map[string]interface{}{
				"wildcard": map[string]interface{}{
					"application": map[string]interface{}{
						"value": "*" + contains + "*",
					},
				},
			},
		})
		should = append(should, map[string]interface{}{
			"query": map[string]interface{}{
				"wildcard": map[string]interface{}{
					"process": map[string]interface{}{
						"value": "*" + contains + "*",
					},
				},
			},
		})
		should = append(should, map[string]interface{}{
			"query": map[string]interface{}{
				"wildcard": map[string]interface{}{
					"messageId": map[string]interface{}{
						"value": "*" + contains + "*",
					},
				},
			},
		})
		should = append(should, map[string]interface{}{
			"query": map[string]interface{}{
				"wildcard": map[string]interface{}{
					"message": map[string]interface{}{
						"value": "*" + contains + "*",
					},
				},
			},
		})
		queries := make([]interface{}, len(should))
		for i, s := range should {
			queries[i] = s
		}
		must = append(must, map[string]interface{}{
			"filtered": map[string]interface{}{
				"query": map[string]interface{}{
					"match_all": map[string]interface{}{},
				},
				"filter": map[string]interface{}{
					"bool": map[string]interface{}{
						"should": queries,
					},
				},
			},
		})
	}

	if !after.IsZero() || !before.IsZero() {
		rangeParams := map[string]time.Time{}

		if !after.IsZero() {
			rangeParams["gte"] = after
		}

		if !before.IsZero() {
			rangeParams["lte"] = before
		}

		must = append(must, map[string]interface{}{
			"range": map[string]interface{}{
				"timeStamp": rangeParams,
			},
		})
	}

	if len(must) == 0 {
		return "", nil
	}

	dsl := map[string]interface{}{
		"query": map[string]interface{}{
			"filtered": map[string]interface{}{
				"query": map[string]interface{}{
					"bool": map[string]interface{}{
						"must": must,
					},
				},
			},
		},
		"size": pagination.PerPage,
		"from": pagination.PerPage * pagination.Page,
	}

	dsl["sort"] = map[string]string{
		pagination.SortBy: string(pagination.Order),
	}

	output, err := json.Marshal(dsl)
	if err != nil {
		return "", err
	}

	return string(output), nil
}

func (service *Service) PostSyslog(mNew *pzsyslog.Message) *piazza.JsonResponse {
	err := mNew.Validate()
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	err = service.postSyslog(mNew)
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	service.incrementStats(mNew.Application)

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
	}
	return resp
}

func (service *Service) postSyslog(mssg *pzsyslog.Message) error {
	var err error
	isAudit := mssg.AuditData != nil

	if service.logWriter != nil {
		err = service.logWriter.Write(mssg, service.async)
		if err != nil {
			return fmt.Errorf("syslog.Service.postSyslog: %s", err.Error())
		}
	}

	if isAudit && service.auditWriter != nil {
		err = service.auditWriter.Write(mssg, service.async)
		if err != nil {
			return fmt.Errorf("syslog.Service.postSyslog (audit): %s", err.Error())
		}
	}

	return nil
}

func (service *Service) getMessageCommon(params *piazza.HttpQueryParams) (*elasticsearch.SearchResult, *piazza.JsonPagination, *piazza.JsonResponse) {
	pagination, err := piazza.NewJsonPagination(params)
	if err != nil {
		return nil, nil, service.newBadRequestResponse(err)
	}

	paginationCreatedOnToTimeStamp(pagination)

	dsl, err := createQueryDslAsString(pagination, params)
	if err != nil {
		return nil, pagination, service.newBadRequestResponse(err)
	}

	var searchResult *elasticsearch.SearchResult

	if dsl == "" {
		searchResult, err = service.esIndex.FilterByMatchAll(pzsyslog.LoggerType, pagination)
	} else {
		searchResult, err = service.esIndex.SearchByJSON(pzsyslog.LoggerType, dsl)
	}
	if err != nil {
		return nil, pagination, service.newInternalErrorResponse(err)
	}

	return searchResult, pagination, nil
}

func (service *Service) GetSyslog(params *piazza.HttpQueryParams) *piazza.JsonResponse {
	var err error

	format, err := params.GetAsString("format", "json")
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	searchResult, pagination, jErr := service.getMessageCommon(params)
	if jErr != nil {
		return jErr
	}

	lines, err := extractFromSearchResult(searchResult)
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	var data interface{} = lines

	if format == "string" {
		t := make([]string, len(lines))
		for i := 0; i < len(lines); i++ {
			t[i] = lines[i].String()
		}
		data = t
	}

	pagination.Count = int(searchResult.TotalHits())
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       data,
		Pagination: pagination,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func extractFromSearchResult(searchResult *elasticsearch.SearchResult) ([]pzsyslog.Message, error) {

	var lines = make([]pzsyslog.Message, 0, len(*searchResult.GetHits()))

	if searchResult == nil || searchResult.GetHits() == nil {
		return lines, nil
	}

	for _, hit := range *searchResult.GetHits() {
		if hit.Source == nil {
			log.Printf("null source hit")
			continue
		}

		msg := pzsyslog.Message{}
		err := json.Unmarshal(*hit.Source, &msg)
		if err != nil {
			log.Printf("UNABLE TO PARSE: %s", string(*hit.Source))
			continue
		}

		// just in case
		/*err = msg.Validate()
		if err != nil {
			log.Printf("UNABLE TO VALIDATE: %s", string(*hit.Source))
			continue
		}*/

		lines = append(lines, msg)
	}

	return lines, nil
}

func (service *Service) PostQuery(params *piazza.HttpQueryParams, jsnQuery string) *piazza.JsonResponse {

	format, err := piazza.NewJsonPagination(params)
	if err != nil {
		return service.newBadRequestResponse(err)
	}
	paginationCreatedOnToTimeStamp(format)

	if jsnQuery, err = syncPagination(jsnQuery, *format); err != nil {
		return service.newBadRequestResponse(err)
	}

	searchResult, err := service.esIndex.SearchByJSON(pzsyslog.LoggerType, jsnQuery)
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	lines, err := extractFromSearchResult(searchResult)
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	format.Count = int(searchResult.TotalHits())
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       lines,
		Pagination: format,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func syncPagination(dslString string, format piazza.JsonPagination) (string, error) {
	// Overwrite any from/size in dsl with what's in the params
	b := []byte(dslString)
	var f interface{}
	err := json.Unmarshal(b, &f)
	if err != nil {
		return "", err
	}
	dsl := f.(map[string]interface{})
	dsl["from"] = format.Page * format.PerPage
	dsl["size"] = format.PerPage
	if dsl["sort"] == nil {
		// Since ES has more fine grained sorting allow their sorting to take precedence
		// If sorting wasn't specified in the DSL, put in sorting from Piazza
		bts := []byte("[{\"" + format.SortBy + "\":\"" + string(format.Order) + "\"}]")
		var g interface{}
		if err = json.Unmarshal(bts, &g); err != nil {
			return "", err
		}
		sortDsl := g.([]interface{})
		dsl["sort"] = sortDsl
	}
	byteArray, err := json.Marshal(dsl)
	if err != nil {
		return "", err
	}
	return string(byteArray), nil
}
<|endoftext|>


<commit_before>// Copyright 2013, Cong Ding. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// author: Cong Ding <dinggnu@gmail.com>
//
package logging

import (
	"fmt"
	"os"
	"runtime"
	"sync/atomic"
	"time"
)

type log struct {
	level           Level
	name            string
	nextSeqid       uint64
	levelno         int
	levelname       string
	pathname        string
	filename        string
	module          string
	lineno          int
	funcName        string
	created         int64
	asctime         string
	msecs           int64
	relativeCreated int64
	thread          int
	threadName      string
	process         int
	message         string
	timestamp       int64
}

type field func(*logging, *log) interface{}

var fields = map[string]field{
	"name":            (*logging).lname,
	"nextSeqid":       (*logging).nextSeqid,
	"levelno":         (*logging).levelno,
	"levelname":       (*logging).levelname,
	"pathname":        (*logging).pathname,
	"filename":        (*logging).filename,
	"module":          (*logging).module,
	"lineno":          (*logging).lineno,
	"funcName":        (*logging).funcName,
	"created":         (*logging).created,
	"asctime":         (*logging).asctime,
	"msecs":           (*logging).msecs,
	"relativeCreated": (*logging).relativeCreated,
	"thread":          (*logging).thread,
	"threadName":      (*logging).threadName,
	"process":         (*logging).process,
	"message":         (*logging).message,
	"timestamp":       (*logging).timestamp,
}

const errString = "???"

// GetGoId returns the id of goroutine, which is defined in ./get_go_id.c
func GetGoId() int32

func init() {
}

// generate the runtime information, including pathname, function name,
// filename, line number.
func genRuntime(l *log) {
	calldepth := 5
	pc, file, line, ok := runtime.Caller(calldepth)
	if ok {
		// generate short filename
		short := file
		for i := len(file) - 1; i > 0; i-- {
			if file[i] == '/' {
				short = file[i+1:]
				break
			}
		}

		l.pathname = file
		l.funcName = runtime.FuncForPC(pc).Name()
		l.filename = short
		l.lineno = line
	} else {
		l.pathname = errString
		l.funcName = errString
		l.filename = errString
		l.lineno = 0
	}
}

func (logger *logging) lname(l *log) interface{} {
	return logger.name
}

func (logger *logging) nextSeqid(l *log) interface{} {
	return atomic.AddUint64(&(logger.seqid), 1)
}

func (logger *logging) levelno(l *log) interface{} {
	return int(l.level)
}

func (logger *logging) levelname(l *log) interface{} {
	return levelNames[l.level]
}

func (logger *logging) pathname(l *log) interface{} {
	if l.pathname == "" {
		genRuntime(l)
	}
	return l.pathname
}

func (logger *logging) filename(l *log) interface{} {
	if l.filename == "" {
		genRuntime(l)
	}
	return l.filename
}

func (logger *logging) module(l *log) interface{} {
	return ""
}

func (logger *logging) lineno(l *log) interface{} {
	if l.lineno == 0 {
		genRuntime(l)
	}
	return l.lineno
}

func (logger *logging) funcName(l *log) interface{} {
	if l.funcName == "" {
		genRuntime(l)
	}
	return l.funcName
}

func (logger *logging) created(l *log) interface{} {
	return logger.startTime
}

func (logger *logging) asctime(l *log) interface{} {
	if l.asctime == "" {
		l.asctime = time.Now().String()
	}
	return l.asctime
}

func (logger *logging) msecs(l *log) interface{} {
	if l.msecs == 0 {
		l.msecs = logger.startTime % 1000
	}
	return l.msecs
}

func (logger *logging) timestamp(l *log) interface{} {
	if l.timestamp == 0 {
		l.timestamp = time.Now().UnixNano()
	}
	return l.timestamp
}

func (logger *logging) relativeCreated(l *log) interface{} {
	if l.relativeCreated == 0 {
		l.relativeCreated = time.Now().UnixNano() - logger.startTime
	}
	return l.relativeCreated
}

func (logger *logging) thread(l *log) interface{} {
	if l.thread == 0 {
		l.thread = int(GetGoId())
	}
	return l.thread
}

func (logger *logging) threadName(l *log) interface{} {
	if l.threadName == "" {
		l.threadName = fmt.Sprintf("Thread-%d", GetGoId())
	}
	return l.threadName
}

// Process ID
func (logger *logging) process(l *log) interface{} {
	if l.process == 0 {
		l.process = os.Getpid()
	}
	return l.process
}

func (logger *logging) message(l *log) interface{} {
	return l.message
}
<commit_msg>generate short function name<commit_after>// Copyright 2013, Cong Ding. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// author: Cong Ding <dinggnu@gmail.com>
//
package logging

import (
	"fmt"
	"os"
	"runtime"
	"sync/atomic"
	"time"
)

type log struct {
	level           Level
	name            string
	nextSeqid       uint64
	levelno         int
	levelname       string
	pathname        string
	filename        string
	module          string
	lineno          int
	funcName        string
	created         int64
	asctime         string
	msecs           int64
	relativeCreated int64
	thread          int
	threadName      string
	process         int
	message         string
	timestamp       int64
}

type field func(*logging, *log) interface{}

var fields = map[string]field{
	"name":            (*logging).lname,
	"nextSeqid":       (*logging).nextSeqid,
	"levelno":         (*logging).levelno,
	"levelname":       (*logging).levelname,
	"pathname":        (*logging).pathname,
	"filename":        (*logging).filename,
	"module":          (*logging).module,
	"lineno":          (*logging).lineno,
	"funcName":        (*logging).funcName,
	"created":         (*logging).created,
	"asctime":         (*logging).asctime,
	"msecs":           (*logging).msecs,
	"relativeCreated": (*logging).relativeCreated,
	"thread":          (*logging).thread,
	"threadName":      (*logging).threadName,
	"process":         (*logging).process,
	"message":         (*logging).message,
	"timestamp":       (*logging).timestamp,
}

const errString = "???"

// GetGoId returns the id of goroutine, which is defined in ./get_go_id.c
func GetGoId() int32

func init() {
}

// generate the runtime information, including pathname, function name,
// filename, line number.
func genRuntime(l *log) {
	calldepth := 5
	pc, file, line, ok := runtime.Caller(calldepth)
	if ok {
		// generate short filename
		short := file
		for i := len(file) - 1; i > 0; i-- {
			if file[i] == '/' {
				short = file[i+1:]
				break
			}
		}

		// generate short function name
		fname := runtime.FuncForPC(pc).Name()
		fshort := fname
		for i := len(fname) - 1; i > 0; i-- {
			if fname[i] == '.' {
				fshort = fname[i+1:]
				break
			}
		}

		l.pathname = file
		l.funcName = fshort
		l.filename = short
		l.lineno = line
	} else {
		l.pathname = errString
		l.funcName = errString
		l.filename = errString
		l.lineno = 0
	}
}

func (logger *logging) lname(l *log) interface{} {
	return logger.name
}

func (logger *logging) nextSeqid(l *log) interface{} {
	return atomic.AddUint64(&(logger.seqid), 1)
}

func (logger *logging) levelno(l *log) interface{} {
	return int(l.level)
}

func (logger *logging) levelname(l *log) interface{} {
	return levelNames[l.level]
}

func (logger *logging) pathname(l *log) interface{} {
	if l.pathname == "" {
		genRuntime(l)
	}
	return l.pathname
}

func (logger *logging) filename(l *log) interface{} {
	if l.filename == "" {
		genRuntime(l)
	}
	return l.filename
}

func (logger *logging) module(l *log) interface{} {
	return ""
}

func (logger *logging) lineno(l *log) interface{} {
	if l.lineno == 0 {
		genRuntime(l)
	}
	return l.lineno
}

func (logger *logging) funcName(l *log) interface{} {
	if l.funcName == "" {
		genRuntime(l)
	}
	return l.funcName
}

func (logger *logging) created(l *log) interface{} {
	return logger.startTime
}

func (logger *logging) asctime(l *log) interface{} {
	if l.asctime == "" {
		l.asctime = time.Now().String()
	}
	return l.asctime
}

func (logger *logging) msecs(l *log) interface{} {
	if l.msecs == 0 {
		l.msecs = logger.startTime % 1000
	}
	return l.msecs
}

func (logger *logging) timestamp(l *log) interface{} {
	if l.timestamp == 0 {
		l.timestamp = time.Now().UnixNano()
	}
	return l.timestamp
}

func (logger *logging) relativeCreated(l *log) interface{} {
	if l.relativeCreated == 0 {
		l.relativeCreated = time.Now().UnixNano() - logger.startTime
	}
	return l.relativeCreated
}

func (logger *logging) thread(l *log) interface{} {
	if l.thread == 0 {
		l.thread = int(GetGoId())
	}
	return l.thread
}

func (logger *logging) threadName(l *log) interface{} {
	if l.threadName == "" {
		l.threadName = fmt.Sprintf("Thread-%d", GetGoId())
	}
	return l.threadName
}

// Process ID
func (logger *logging) process(l *log) interface{} {
	if l.process == 0 {
		l.process = os.Getpid()
	}
	return l.process
}

func (logger *logging) message(l *log) interface{} {
	return l.message
}
<|endoftext|>


<commit_before>package cas

import (
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"golang.org/x/net/html"
)

var (
	urlCleanParameters     = []string{"gateway", "renew", "service", "ticket"}
	redirectAttemptedError = errors.New("redirect")
	InvalidCredentials     = errors.New("Bad username or password")
)

// sanitisedURL cleans a URL of CAS specific parameters
func sanitisedURL(unclean *url.URL) *url.URL {
	// Shouldn't be any errors parsing an existing *url.URL
	u, _ := url.Parse(unclean.String())
	q := u.Query()

	for _, param := range urlCleanParameters {
		q.Del(param)
	}

	u.RawQuery = q.Encode()
	return u
}

// sanitisedURLString cleans a URL and returns its string value
func sanitisedURLString(unclean *url.URL) string {
	return sanitisedURL(unclean).String()
}

// requestURL determines an absolute URL from the http.Request.
func requestURL(r *http.Request) (*url.URL, error) {
	u, err := url.Parse(r.URL.String())
	if err != nil {
		return nil, err
	}

	u.Host = r.Host
	u.Scheme = "http"

	if scheme := r.Header.Get("X-Forwarded-Proto"); scheme != "" {
		u.Scheme = scheme
	} else if r.TLS != nil {
		u.Scheme = "https"
	}

	return u, nil
}

type Client struct {
	URL *url.URL
}

func (c *Client) AuthenticateUser(username, password string, r *http.Request) (*AuthenticationResponse, error) {
	lt, et, jsession, err := c.getLoginToken(r)
	if err != nil {
		return nil, err
	}
	if lt == "" {
		return nil, errors.New("Couldn't get a login token")
	}
	if jsession == nil {
		return nil, errors.New("Couldn't get server session cookie")
	}

	form := url.Values{}
	form.Add("username", username)
	form.Add("password", password)
	form.Add("lt", lt)
	form.Add("_eventId", "submit") // Not sure why this is needed, it's not in the spec
	form.Add("execution", et)

	client := &http.Client{}
	// Force the client to never follow redirects
	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		return redirectAttemptedError
	}
	reqUrl, _ := c.loginUrlForRequestor(r)
	req, err := http.NewRequest("POST", reqUrl, strings.NewReader(form.Encode()))
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.AddCookie(jsession) // The login token is tied to our session
	if err != nil {
		return nil, err
	}

	resp, err := client.Do(req)
	defer resp.Body.Close()
	// Filter out our fake redirect error
	if urlError, ok := err.(*url.Error); ok && urlError.Err == redirectAttemptedError {
		err = nil
	}
	if err != nil {
		return nil, err
	}

	location, err := resp.Location()
	if err != nil {
		return nil, InvalidCredentials
	}
	return c.validateTicket(location.Query().Get("ticket"), r)
}

func (c *Client) validateTicket(ticket string, r *http.Request) (*AuthenticationResponse, error) {
	// Validate the ticket
	reqUrl, _ := c.serviceValidateUrlForRequest(ticket, r)
	resp, err := http.Get(reqUrl)
	defer resp.Body.Close()

	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, errors.New("Failed to verify ticket")
	}

	body, _ := ioutil.ReadAll(resp.Body)
	return ParseServiceResponse(body)
}

func (c *Client) getLoginToken(r *http.Request) (string, string, *http.Cookie, error) {
	reqUrl, _ := c.loginUrlForRequestor(r)
	resp, err := http.Get(reqUrl)
	if err != nil {
		return "", "", nil, err
	}

	var jsession *http.Cookie
	cookies := resp.Cookies()
	for _, cookie := range cookies {
		if cookie.Name == "JSESSIONID" {
			jsession = cookie
			break
		}
	}

	b := resp.Body
	defer b.Close()
	loginToken := ""
	executionToken := ""

	z := html.NewTokenizer(b)
tokenLoop:
	for {
		tt := z.Next()

		switch {
		case tt == html.ErrorToken:
			// End of the document, we're done
			return "", "", nil, nil
		case tt == html.SelfClosingTagToken:
			t := z.Token()

			// Check if the token is an <input> tag
			isInput := t.Data == "input"
			if !isInput {
				continue
			}

			// Iterate over all of the Token's attributes until we find an "id", "name", or "value"
			tokenName := ""
			tokenVal := ""
			for _, a := range t.Attr {
				if a.Key == "name" {
					tokenName = a.Val
				} else if a.Key == "value" {
					tokenVal = a.Val
				}
			}

			switch tokenName {
			case "lt":
				loginToken = tokenVal
			case "execution":
				executionToken = tokenVal
			}

			if loginToken != "" && executionToken != "" {
				break tokenLoop
			}
		}
	}
	return loginToken, executionToken, jsession, nil
}

func (c *Client) loginUrlForRequestor(r *http.Request) (string, error) {
	u, err := c.URL.Parse("login")
	if err != nil {
		return "", err
	}

	service, err := requestURL(r)
	if err != nil {
		return "", err
	}

	q := u.Query()
	q.Add("service", sanitisedURLString(service))
	u.RawQuery = q.Encode()

	return u.String(), nil
}

func (c *Client) validateUrlForRequest(ticket string, r *http.Request) (string, error) {
	u, err := c.URL.Parse("validate")
	if err != nil {
		return "", err
	}

	service, err := requestURL(r)
	if err != nil {
		return "", err
	}

	q := u.Query()
	q.Add("service", sanitisedURLString(service))
	q.Add("ticket", ticket)
	u.RawQuery = q.Encode()

	return u.String(), nil
}

func (c *Client) serviceValidateUrlForRequest(ticket string, r *http.Request) (string, error) {
	u, err := c.URL.Parse("serviceValidate")
	if err != nil {
		return "", err
	}

	service, err := requestURL(r)
	if err != nil {
		return "", err
	}

	q := u.Query()
	q.Add("service", sanitisedURLString(service))
	q.Add("ticket", ticket)
	u.RawQuery = q.Encode()

	return u.String(), nil
}
<commit_msg>Fixed merge conflict<commit_after>package cas

import (
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"golang.org/x/net/html"
)

var (
	urlCleanParameters     = []string{"gateway", "renew", "service", "ticket"}
	redirectAttemptedError = errors.New("redirect")
	InvalidCredentials     = errors.New("Bad username or password")
)

// sanitisedURL cleans a URL of CAS specific parameters
func sanitisedURL(unclean *url.URL) *url.URL {
	// Shouldn't be any errors parsing an existing *url.URL
	u, _ := url.Parse(unclean.String())
	q := u.Query()

	for _, param := range urlCleanParameters {
		q.Del(param)
	}

	u.RawQuery = q.Encode()
	return u
}

// sanitisedURLString cleans a URL and returns its string value
func sanitisedURLString(unclean *url.URL) string {
	return sanitisedURL(unclean).String()
}

// requestURL determines an absolute URL from the http.Request.
func requestURL(r *http.Request) (*url.URL, error) {
	u, err := url.Parse(r.URL.String())
	if err != nil {
		return nil, err
	}

	u.Host = r.Host
	u.Scheme = "http"

	if scheme := r.Header.Get("X-Forwarded-Proto"); scheme != "" {
		u.Scheme = scheme
	} else if r.TLS != nil {
		u.Scheme = "https"
	}

	return u, nil
}

type Client struct {
	URL *url.URL
}

func (c *Client) AuthenticateUser(username, password string, r *http.Request) (*AuthenticationResponse, error) {
	lt, et, jsession, err := c.getLoginToken(r)
	if err != nil {
		return nil, err
	}
	if lt == "" {
		return nil, errors.New("Couldn't get a login token")
	}
	if jsession == nil {
		return nil, errors.New("Couldn't get server session cookie")
	}

	form := url.Values{}
	form.Add("username", username)
	form.Add("password", password)
	form.Add("lt", lt)
	form.Add("_eventId", "submit") // Not sure why this is needed, it's not in the spec
	if et != "" {
		form.Add("execution", et)
	}

	client := &http.Client{}
	// Force the client to never follow redirects
	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		return redirectAttemptedError
	}
	reqUrl, _ := c.loginUrlForRequestor(r)
	req, err := http.NewRequest("POST", reqUrl, strings.NewReader(form.Encode()))
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.AddCookie(jsession) // The login token is tied to our session
	if err != nil {
		return nil, err
	}

	resp, err := client.Do(req)
	defer resp.Body.Close()
	// Filter out our fake redirect error
	if urlError, ok := err.(*url.Error); ok && urlError.Err == redirectAttemptedError {
		err = nil
	}
	if err != nil {
		return nil, err
	}

	location, err := resp.Location()
	if err != nil {
		return nil, InvalidCredentials
	}
	return c.validateTicket(location.Query().Get("ticket"), r)
}

func (c *Client) validateTicket(ticket string, r *http.Request) (*AuthenticationResponse, error) {
	// Validate the ticket
	reqUrl, _ := c.serviceValidateUrlForRequest(ticket, r)
	resp, err := http.Get(reqUrl)
	defer resp.Body.Close()

	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, errors.New("Failed to verify ticket")
	}

	body, _ := ioutil.ReadAll(resp.Body)
	return ParseServiceResponse(body)
}

func (c *Client) getLoginToken(r *http.Request) (string, string, *http.Cookie, error) {
	reqUrl, _ := c.loginUrlForRequestor(r)
	resp, err := http.Get(reqUrl)
	if err != nil {
		return "", "", nil, err
	}

	var jsession *http.Cookie
	cookies := resp.Cookies()
	for _, cookie := range cookies {
		if cookie.Name == "JSESSIONID" {
			jsession = cookie
			break
		}
	}

	b := resp.Body
	defer b.Close()
	loginToken := ""
	executionToken := ""

	z := html.NewTokenizer(b)
tokenLoop:
	for {
		tt := z.Next()

		switch {
		case tt == html.ErrorToken:
			// End of the document, we're done
			break tokenLoop
		case tt == html.SelfClosingTagToken:
			t := z.Token()

			// Check if the token is an <input> tag
			isInput := t.Data == "input"
			if !isInput {
				continue
			}

			// Iterate over all of the Token's attributes until we find an "id", "name", or "value"
			tokenName := ""
			tokenVal := ""
			for _, a := range t.Attr {
				if a.Key == "name" {
					tokenName = a.Val
				} else if a.Key == "value" {
					tokenVal = a.Val
				}
			}

			switch tokenName {
			case "lt":
				loginToken = tokenVal
			case "execution":
				executionToken = tokenVal
			}

			if loginToken != "" && executionToken != "" {
				break tokenLoop
			}
		}
	}
	return loginToken, executionToken, jsession, nil
}

func (c *Client) loginUrlForRequestor(r *http.Request) (string, error) {
	u, err := c.URL.Parse("login")
	if err != nil {
		return "", err
	}

	service, err := requestURL(r)
	if err != nil {
		return "", err
	}

	q := u.Query()
	q.Add("service", sanitisedURLString(service))
	u.RawQuery = q.Encode()

	return u.String(), nil
}

func (c *Client) validateUrlForRequest(ticket string, r *http.Request) (string, error) {
	u, err := c.URL.Parse("validate")
	if err != nil {
		return "", err
	}

	service, err := requestURL(r)
	if err != nil {
		return "", err
	}

	q := u.Query()
	q.Add("service", sanitisedURLString(service))
	q.Add("ticket", ticket)
	u.RawQuery = q.Encode()

	return u.String(), nil
}

func (c *Client) serviceValidateUrlForRequest(ticket string, r *http.Request) (string, error) {
	u, err := c.URL.Parse("serviceValidate")
	if err != nil {
		return "", err
	}

	service, err := requestURL(r)
	if err != nil {
		return "", err
	}

	q := u.Query()
	q.Add("service", sanitisedURLString(service))
	q.Add("ticket", ticket)
	u.RawQuery = q.Encode()

	return u.String(), nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package scanner implements a scanner for gcfg configuration text.
// It takes a []byte as source which can then be tokenized
// through repeated calls to the Scan method.
//
// Note that the API for the scanner package may change to accommodate new
// features or implementation changes in gcfg.
//
package scanner

import (
	"fmt"
	"path/filepath"
	"unicode"
	"unicode/utf8"

	"gopkg.in/gcfg.v1/token"
)

var RelaxedScannerMode = false

// An ErrorHandler may be provided to Scanner.Init. If a syntax error is
// encountered and a handler was installed, the handler is called with a
// position and an error message. The position points to the beginning of
// the offending token.
//
type ErrorHandler func(pos token.Position, msg string)

// A Scanner holds the scanner's internal state while processing
// a given text.  It can be allocated as part of another data
// structure but must be initialized via Init before use.
//
type Scanner struct {
	// immutable state
	file *token.File  // source file handle
	dir  string       // directory portion of file.Name()
	src  []byte       // source
	err  ErrorHandler // error reporting; or nil
	mode Mode         // scanning mode

	// scanning state
	ch         rune // current character
	offset     int  // character offset
	rdOffset   int  // reading offset (position after current character)
	lineOffset int  // current line offset
	nextVal    bool // next token is expected to be a value

	// public state - ok to modify
	ErrorCount int // number of errors encountered
}

// Read the next Unicode char into s.ch.
// s.ch < 0 means end-of-file.
//
func (s *Scanner) next() {
	if s.rdOffset < len(s.src) {
		s.offset = s.rdOffset
		if s.ch == '\n' {
			s.lineOffset = s.offset
			s.file.AddLine(s.offset)
		}
		r, w := rune(s.src[s.rdOffset]), 1
		switch {
		case r == 0:
			s.error(s.offset, "illegal character NUL")
		case r >= 0x80:
			// not ASCII
			r, w = utf8.DecodeRune(s.src[s.rdOffset:])
			if r == utf8.RuneError && w == 1 {
				s.error(s.offset, "illegal UTF-8 encoding")
			}
		}
		s.rdOffset += w
		s.ch = r
	} else {
		s.offset = len(s.src)
		if s.ch == '\n' {
			s.lineOffset = s.offset
			s.file.AddLine(s.offset)
		}
		s.ch = -1 // eof
	}
}

// A mode value is a set of flags (or 0).
// They control scanner behavior.
//
type Mode uint

const (
	ScanComments Mode = 1 << iota // return comments as COMMENT tokens
)

// Init prepares the scanner s to tokenize the text src by setting the
// scanner at the beginning of src. The scanner uses the file set file
// for position information and it adds line information for each line.
// It is ok to re-use the same file when re-scanning the same file as
// line information which is already present is ignored. Init causes a
// panic if the file size does not match the src size.
//
// Calls to Scan will invoke the error handler err if they encounter a
// syntax error and err is not nil. Also, for each error encountered,
// the Scanner field ErrorCount is incremented by one. The mode parameter
// determines how comments are handled.
//
// Note that Init may call err if there is an error in the first character
// of the file.
//
func (s *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode Mode) {
	// Explicitly initialize all fields since a scanner may be reused.
	if file.Size() != len(src) {
		panic(fmt.Sprintf("file size (%d) does not match src len (%d)", file.Size(), len(src)))
	}
	s.file = file
	s.dir, _ = filepath.Split(file.Name())
	s.src = src
	s.err = err
	s.mode = mode

	s.ch = ' '
	s.offset = 0
	s.rdOffset = 0
	s.lineOffset = 0
	s.ErrorCount = 0
	s.nextVal = false

	s.next()
}

func (s *Scanner) error(offs int, msg string) {
	if s.err != nil {
		s.err(s.file.Position(s.file.Pos(offs)), msg)
	}
	s.ErrorCount++
}

func (s *Scanner) scanComment() string {
	// initial [;#] already consumed
	offs := s.offset - 1 // position of initial [;#]

	for s.ch != '\n' && s.ch >= 0 {
		s.next()
	}
	return string(s.src[offs:s.offset])
}

func isLetter(ch rune) bool {
	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch >= 0x80 && unicode.IsLetter(ch)
}

func isDigit(ch rune) bool {
	return '0' <= ch && ch <= '9' || ch >= 0x80 && unicode.IsDigit(ch)
}

func (s *Scanner) scanIdentifier() string {
	offs := s.offset
	for isLetter(s.ch) || isDigit(s.ch) || s.ch == '-' {
		s.next()
	}
	return string(s.src[offs:s.offset])
}

func (s *Scanner) scanEscape(val bool) {
	offs := s.offset
	ch := s.ch
	s.next() // always make progress
	switch ch {
	case '\\', '"':
		// ok
	case 'n', 't':
		if val {
			break // ok
		}
		fallthrough
	default:
		s.error(offs, "unknown escape sequence")
	}
}

func (s *Scanner) scanString() string {
	// '"' opening already consumed
	offs := s.offset - 1

	for s.ch != '"' {
		ch := s.ch
		s.next()
		if ch == '\n' || ch < 0 {
			s.error(offs, "string not terminated")
			break
		}
		if ch == '\\' {
			s.scanEscape(false)
		}
	}

	s.next()

	return string(s.src[offs:s.offset])
}

func stripCR(b []byte) []byte {
	c := make([]byte, len(b))
	i := 0
	for _, ch := range b {
		if ch != '\r' {
			c[i] = ch
			i++
		}
	}
	return c[:i]
}

func (s *Scanner) scanValString() string {
	offs := s.offset

	hasCR := false
	end := offs
	inQuote := false
loop:
	for inQuote || s.ch >= 0 && s.ch != '\n' && s.ch != ';' && s.ch != '#' {
		ch := s.ch
		s.next()
		switch {
		case inQuote && ch == '\\':
			s.scanEscape(true)
		case !inQuote && ch == '\\':
			if s.ch == '\r' {
				hasCR = true
				s.next()
			}
			if s.ch != '\n' && !RelaxedScannerMode {
				s.error(offs, "unquoted '\\' must be followed by new line")
				break loop
			}
			s.next()
		case ch == '"':
			inQuote = !inQuote
		case ch == '\r':
			hasCR = true
		case ch < 0 || inQuote && ch == '\n':
			s.error(offs, "string not terminated")
			break loop
		}
		if inQuote || !isWhiteSpace(ch) {
			end = s.offset
		}
	}

	lit := s.src[offs:end]
	if hasCR {
		lit = stripCR(lit)
	}

	return string(lit)
}

func isWhiteSpace(ch rune) bool {
	return ch == ' ' || ch == '\t' || ch == '\r'
}

func (s *Scanner) skipWhitespace() {
	for isWhiteSpace(s.ch) {
		s.next()
	}
}

// Scan scans the next token and returns the token position, the token,
// and its literal string if applicable. The source end is indicated by
// token.EOF.
//
// If the returned token is a literal (token.IDENT, token.STRING) or
// token.COMMENT, the literal string has the corresponding value.
//
// If the returned token is token.ILLEGAL, the literal string is the
// offending character.
//
// In all other cases, Scan returns an empty literal string.
//
// For more tolerant parsing, Scan will return a valid token if
// possible even if a syntax error was encountered. Thus, even
// if the resulting token sequence contains no illegal tokens,
// a client may not assume that no error occurred. Instead it
// must check the scanner's ErrorCount or the number of calls
// of the error handler, if there was one installed.
//
// Scan adds line information to the file added to the file
// set with Init. Token positions are relative to that file
// and thus relative to the file set.
//
func (s *Scanner) Scan() (pos token.Pos, tok token.Token, lit string) {
scanAgain:
	s.skipWhitespace()

	// current token start
	pos = s.file.Pos(s.offset)

	// determine token value
	switch ch := s.ch; {
	case s.nextVal:
		lit = s.scanValString()
		tok = token.STRING
		s.nextVal = false
	case isLetter(ch):
		lit = s.scanIdentifier()
		tok = token.IDENT
	default:
		s.next() // always make progress
		switch ch {
		case -1:
			tok = token.EOF
		case '\n':
			tok = token.EOL
		case '"':
			tok = token.STRING
			lit = s.scanString()
		case '[':
			tok = token.LBRACK
		case ']':
			tok = token.RBRACK
		case ';', '#':
			// comment
			lit = s.scanComment()
			if s.mode&ScanComments == 0 {
				// skip comment
				goto scanAgain
			}
			tok = token.COMMENT
		case '=':
			tok = token.ASSIGN
			s.nextVal = true
		default:
			s.error(s.file.Offset(pos), fmt.Sprintf("illegal character %#U", ch))
			tok = token.ILLEGAL
			lit = string(ch)
		}
	}

	return
}
<commit_msg>config file allows underscore variables<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package scanner implements a scanner for gcfg configuration text.
// It takes a []byte as source which can then be tokenized
// through repeated calls to the Scan method.
//
// Note that the API for the scanner package may change to accommodate new
// features or implementation changes in gcfg.
//
package scanner

import (
	"fmt"
	"path/filepath"
	"unicode"
	"unicode/utf8"

	"gopkg.in/gcfg.v1/token"
)

var RelaxedScannerMode = false

// An ErrorHandler may be provided to Scanner.Init. If a syntax error is
// encountered and a handler was installed, the handler is called with a
// position and an error message. The position points to the beginning of
// the offending token.
//
type ErrorHandler func(pos token.Position, msg string)

// A Scanner holds the scanner's internal state while processing
// a given text.  It can be allocated as part of another data
// structure but must be initialized via Init before use.
//
type Scanner struct {
	// immutable state
	file *token.File  // source file handle
	dir  string       // directory portion of file.Name()
	src  []byte       // source
	err  ErrorHandler // error reporting; or nil
	mode Mode         // scanning mode

	// scanning state
	ch         rune // current character
	offset     int  // character offset
	rdOffset   int  // reading offset (position after current character)
	lineOffset int  // current line offset
	nextVal    bool // next token is expected to be a value

	// public state - ok to modify
	ErrorCount int // number of errors encountered
}

// Read the next Unicode char into s.ch.
// s.ch < 0 means end-of-file.
//
func (s *Scanner) next() {
	if s.rdOffset < len(s.src) {
		s.offset = s.rdOffset
		if s.ch == '\n' {
			s.lineOffset = s.offset
			s.file.AddLine(s.offset)
		}
		r, w := rune(s.src[s.rdOffset]), 1
		switch {
		case r == 0:
			s.error(s.offset, "illegal character NUL")
		case r >= 0x80:
			// not ASCII
			r, w = utf8.DecodeRune(s.src[s.rdOffset:])
			if r == utf8.RuneError && w == 1 {
				s.error(s.offset, "illegal UTF-8 encoding")
			}
		}
		s.rdOffset += w
		s.ch = r
	} else {
		s.offset = len(s.src)
		if s.ch == '\n' {
			s.lineOffset = s.offset
			s.file.AddLine(s.offset)
		}
		s.ch = -1 // eof
	}
}

// A mode value is a set of flags (or 0).
// They control scanner behavior.
//
type Mode uint

const (
	ScanComments Mode = 1 << iota // return comments as COMMENT tokens
)

// Init prepares the scanner s to tokenize the text src by setting the
// scanner at the beginning of src. The scanner uses the file set file
// for position information and it adds line information for each line.
// It is ok to re-use the same file when re-scanning the same file as
// line information which is already present is ignored. Init causes a
// panic if the file size does not match the src size.
//
// Calls to Scan will invoke the error handler err if they encounter a
// syntax error and err is not nil. Also, for each error encountered,
// the Scanner field ErrorCount is incremented by one. The mode parameter
// determines how comments are handled.
//
// Note that Init may call err if there is an error in the first character
// of the file.
//
func (s *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode Mode) {
	// Explicitly initialize all fields since a scanner may be reused.
	if file.Size() != len(src) {
		panic(fmt.Sprintf("file size (%d) does not match src len (%d)", file.Size(), len(src)))
	}
	s.file = file
	s.dir, _ = filepath.Split(file.Name())
	s.src = src
	s.err = err
	s.mode = mode

	s.ch = ' '
	s.offset = 0
	s.rdOffset = 0
	s.lineOffset = 0
	s.ErrorCount = 0
	s.nextVal = false

	s.next()
}

func (s *Scanner) error(offs int, msg string) {
	if s.err != nil {
		s.err(s.file.Position(s.file.Pos(offs)), msg)
	}
	s.ErrorCount++
}

func (s *Scanner) scanComment() string {
	// initial [;#] already consumed
	offs := s.offset - 1 // position of initial [;#]

	for s.ch != '\n' && s.ch >= 0 {
		s.next()
	}
	return string(s.src[offs:s.offset])
}

func isLetter(ch rune) bool {
	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= 0x80 && unicode.IsLetter(ch)
}

func isDigit(ch rune) bool {
	return '0' <= ch && ch <= '9' || ch >= 0x80 && unicode.IsDigit(ch)
}

func (s *Scanner) scanIdentifier() string {
	offs := s.offset
	for isLetter(s.ch) || isDigit(s.ch) || s.ch == '-' {
		s.next()
	}
	return string(s.src[offs:s.offset])
}

func (s *Scanner) scanEscape(val bool) {
	offs := s.offset
	ch := s.ch
	s.next() // always make progress
	switch ch {
	case '\\', '"':
		// ok
	case 'n', 't':
		if val {
			break // ok
		}
		fallthrough
	default:
		s.error(offs, "unknown escape sequence")
	}
}

func (s *Scanner) scanString() string {
	// '"' opening already consumed
	offs := s.offset - 1

	for s.ch != '"' {
		ch := s.ch
		s.next()
		if ch == '\n' || ch < 0 {
			s.error(offs, "string not terminated")
			break
		}
		if ch == '\\' {
			s.scanEscape(false)
		}
	}

	s.next()

	return string(s.src[offs:s.offset])
}

func stripCR(b []byte) []byte {
	c := make([]byte, len(b))
	i := 0
	for _, ch := range b {
		if ch != '\r' {
			c[i] = ch
			i++
		}
	}
	return c[:i]
}

func (s *Scanner) scanValString() string {
	offs := s.offset

	hasCR := false
	end := offs
	inQuote := false
loop:
	for inQuote || s.ch >= 0 && s.ch != '\n' && s.ch != ';' && s.ch != '#' {
		ch := s.ch
		s.next()
		switch {
		case inQuote && ch == '\\':
			s.scanEscape(true)
		case !inQuote && ch == '\\':
			if s.ch == '\r' {
				hasCR = true
				s.next()
			}
			if s.ch != '\n' && !RelaxedScannerMode {
				s.error(offs, "unquoted '\\' must be followed by new line")
				break loop
			}
			s.next()
		case ch == '"':
			inQuote = !inQuote
		case ch == '\r':
			hasCR = true
		case ch < 0 || inQuote && ch == '\n':
			s.error(offs, "string not terminated")
			break loop
		}
		if inQuote || !isWhiteSpace(ch) {
			end = s.offset
		}
	}

	lit := s.src[offs:end]
	if hasCR {
		lit = stripCR(lit)
	}

	return string(lit)
}

func isWhiteSpace(ch rune) bool {
	return ch == ' ' || ch == '\t' || ch == '\r'
}

func (s *Scanner) skipWhitespace() {
	for isWhiteSpace(s.ch) {
		s.next()
	}
}

// Scan scans the next token and returns the token position, the token,
// and its literal string if applicable. The source end is indicated by
// token.EOF.
//
// If the returned token is a literal (token.IDENT, token.STRING) or
// token.COMMENT, the literal string has the corresponding value.
//
// If the returned token is token.ILLEGAL, the literal string is the
// offending character.
//
// In all other cases, Scan returns an empty literal string.
//
// For more tolerant parsing, Scan will return a valid token if
// possible even if a syntax error was encountered. Thus, even
// if the resulting token sequence contains no illegal tokens,
// a client may not assume that no error occurred. Instead it
// must check the scanner's ErrorCount or the number of calls
// of the error handler, if there was one installed.
//
// Scan adds line information to the file added to the file
// set with Init. Token positions are relative to that file
// and thus relative to the file set.
//
func (s *Scanner) Scan() (pos token.Pos, tok token.Token, lit string) {
scanAgain:
	s.skipWhitespace()

	// current token start
	pos = s.file.Pos(s.offset)

	// determine token value
	switch ch := s.ch; {
	case s.nextVal:
		lit = s.scanValString()
		tok = token.STRING
		s.nextVal = false
	case isLetter(ch):
		lit = s.scanIdentifier()
		tok = token.IDENT
	default:
		s.next() // always make progress
		switch ch {
		case -1:
			tok = token.EOF
		case '\n':
			tok = token.EOL
		case '"':
			tok = token.STRING
			lit = s.scanString()
		case '[':
			tok = token.LBRACK
		case ']':
			tok = token.RBRACK
		case ';', '#':
			// comment
			lit = s.scanComment()
			if s.mode&ScanComments == 0 {
				// skip comment
				goto scanAgain
			}
			tok = token.COMMENT
		case '=':
			tok = token.ASSIGN
			s.nextVal = true
		default:
			s.error(s.file.Offset(pos), fmt.Sprintf("illegal character %#U", ch))
			tok = token.ILLEGAL
			lit = string(ch)
		}
	}

	return
}
<|endoftext|>


<commit_before>package preloadlist

import (
	"bufio"
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"strings"
	"time"
	"unicode"
)

const (
	// ForceHTTPS indicates that all requests should be upgraded from HTTP to
	// HTTPS using the HSTS mechanism (https://tools.ietf.org/html/rfc6797).
	ForceHTTPS = "force-https"
)

// PreloadList contains a parsed form of the Chromium Preload list.
//
// The full list contains information about more than just HSTS, but only
// HSTS-related contents are currently exposed in this struct.
type PreloadList struct {
	Entries []Entry `json:"entries"`
}

// A Entry contains the data from an entry in the Chromium
// Preload list.
//
// - Name: The domain name.
//
// - Mode: The only valid non-empty value is ForceHTTPS
//
// - IncludeSubDomains: If Mode == ForceHTTPS, forces HSTS to apply to
//   all subdomains.
type Entry struct {
	Name              string `json:"name"`
	Mode              string `json:"mode"`
	IncludeSubDomains bool   `json:"include_subdomains"`
}

// IndexedEntries is case-insensitive index of
// the entries from the given PreloadList.
type IndexedEntries struct {
	index map[string]Entry
}

// Index creates an index out of the given list.
func (p PreloadList) Index() (idx IndexedEntries) {
	m := make(map[string]Entry)
	for _, entry := range p.Entries {
		d := strings.ToLower(string(entry.Name))
		m[d] = entry
	}
	return IndexedEntries{
		index: m,
	}
}

// Get acts similar to map access: it returns an entry from the index preload
// list (if it is present), along with a boolean indicating if the entry is
// present.
func (idx IndexedEntries) Get(domain string) (Entry, bool) {
	entry, ok := idx.index[strings.ToLower(domain)]
	return entry, ok
}

const (
	// LatestChromiumURL is the URL of the latest preload list in the Chromium source.
	LatestChromiumURL = "https://chromium.googlesource.com/chromium/src/+/master/net/http/transport_security_state_static.json?format=TEXT"
)

// Parse reads a preload list in JSON format (with certain possible comments)
// and returns a parsed version.
func Parse(r io.Reader) (PreloadList, error) {
	var list PreloadList

	jsonBytes, err := removeComments(r)
	if err != nil {
		return list, errors.New("could not decode body")
	}

	if err := json.Unmarshal(jsonBytes, &list); err != nil {
		return list, err
	}

	return list, nil
}

// removeComments reads the contents of |r| and removes any lines beginning
// with optional whitespace followed by "//"
func removeComments(r io.Reader) ([]byte, error) {
	var buf bytes.Buffer

	sc := bufio.NewScanner(r)
	for sc.Scan() {
		line := sc.Text()
		if isCommentLine(line) {
			fmt.Fprintln(&buf, line)
		}
	}
	if err := sc.Err(); err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

func isCommentLine(line string) bool {
	trimmed := strings.TrimLeftFunc(line, unicode.IsSpace)
	return !strings.HasPrefix(trimmed, "//")
}

// NewFromChromiumURL retrieves the PreloadList from a URL that returns the list
// in base 64.
func NewFromChromiumURL(u string) (PreloadList, error) {
	var list PreloadList

	client := http.Client{
		Timeout: time.Second * 10,
	}

	resp, err := client.Get(u)
	if err != nil {
		return list, err
	}

	if resp.StatusCode != 200 {
		return list, fmt.Errorf("status code %d", resp.StatusCode)
	}

	body := base64.NewDecoder(base64.StdEncoding, resp.Body)

	return Parse(body)
}

// NewFromLatest retrieves the latest PreloadList from the Chromium source at
// https://chromium.googlesource.com/chromium/src/+/master/net/http/transport_security_state_static.json
//
// Note that this list may be up to 12 weeks fresher than the list used
// by the current stable version of Chrome. See
// https://www.chromium.org/developers/calendar for a calendar of releases.
func NewFromLatest() (PreloadList, error) {
	return NewFromChromiumURL(LatestChromiumURL)
}

// NewFromFile reads a PreloadList from a JSON file.
//
// In a Chromium checkout, the file is at
// src/net/http/transport_security_state_static.json
func NewFromFile(fileName string) (PreloadList, error) {
	b, err := os.Open(fileName)
	if err != nil {
		return PreloadList{}, err
	}

	return Parse(b)
}
<commit_msg>First step in making ancestor preload work<commit_after>package preloadlist

import (
	"bufio"
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"strings"
	"time"
	"unicode"
)

const (
	// ForceHTTPS indicates that all requests should be upgraded from HTTP to
	// HTTPS using the HSTS mechanism (https://tools.ietf.org/html/rfc6797).
	ForceHTTPS = "force-https"
)

type HstsPreloadStatus int

const (
	NotFound           HstsPreloadStatus = 1 << iota
	ExactEntryFound
	AncestorEntryFound
)

// PreloadList contains a parsed form of the Chromium Preload list.
//
// The full list contains information about more than just HSTS, but only
// HSTS-related contents are currently exposed in this struct.
type PreloadList struct {
	Entries []Entry `json:"entries"`
}

// A Entry contains the data from an entry in the Chromium
// Preload list.
//
// - Name: The domain name.
//
// - Mode: The only valid non-empty value is ForceHTTPS
//
// - IncludeSubDomains: If Mode == ForceHTTPS, forces HSTS to apply to
//   all subdomains.
type Entry struct {
	Name              string `json:"name"`
	Mode              string `json:"mode"`
	IncludeSubDomains bool   `json:"include_subdomains"`
}

// IndexedEntries is case-insensitive index of
// the entries from the given PreloadList.
type IndexedEntries struct {
	index map[string]Entry
}

// Index creates an index out of the given list.
func (p PreloadList) Index() (idx IndexedEntries) {
	m := make(map[string]Entry)
	for _, entry := range p.Entries {
		d := strings.ToLower(string(entry.Name))
		m[d] = entry
	}
	return IndexedEntries{
		index: m,
	}
}

// Get acts similar to map access: it returns an entry from the index preload
// list (if it is present), along with a boolean indicating if the entry is
// present.
func (idx IndexedEntries) Get(domain string) (Entry, HstsPreloadStatus) {
	domain = strings.ToLower(domain)
	entry, ok := idx.index[domain]
	if ok {
		return entry, ExactEntryFound
	} else {
		for i := strings.Index(domain, "."); i != -1; domain = domain[i:] {
			entry, ok = idx.index[domain]
			if ok && entry.IncludeSubDomains {
				return entry, AncestorEntryFound
			}
		}
	}
	return Entry{"", "", false}, NotFound
}

const (
	// LatestChromiumURL is the URL of the latest preload list in the Chromium source.
	LatestChromiumURL = "https://chromium.googlesource.com/chromium/src/+/master/net/http/transport_security_state_static.json?format=TEXT"
)

// Parse reads a preload list in JSON format (with certain possible comments)
// and returns a parsed version.
func Parse(r io.Reader) (PreloadList, error) {
	var list PreloadList

	jsonBytes, err := removeComments(r)
	if err != nil {
		return list, errors.New("could not decode body")
	}

	if err := json.Unmarshal(jsonBytes, &list); err != nil {
		return list, err
	}

	return list, nil
}

// removeComments reads the contents of |r| and removes any lines beginning
// with optional whitespace followed by "//"
func removeComments(r io.Reader) ([]byte, error) {
	var buf bytes.Buffer

	sc := bufio.NewScanner(r)
	for sc.Scan() {
		line := sc.Text()
		if isCommentLine(line) {
			fmt.Fprintln(&buf, line)
		}
	}
	if err := sc.Err(); err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

func isCommentLine(line string) bool {
	trimmed := strings.TrimLeftFunc(line, unicode.IsSpace)
	return !strings.HasPrefix(trimmed, "//")
}

// NewFromChromiumURL retrieves the PreloadList from a URL that returns the list
// in base 64.
func NewFromChromiumURL(u string) (PreloadList, error) {
	var list PreloadList

	client := http.Client{
		Timeout: time.Second * 10,
	}

	resp, err := client.Get(u)
	if err != nil {
		return list, err
	}

	if resp.StatusCode != 200 {
		return list, fmt.Errorf("status code %d", resp.StatusCode)
	}

	body := base64.NewDecoder(base64.StdEncoding, resp.Body)

	return Parse(body)
}

// NewFromLatest retrieves the latest PreloadList from the Chromium source at
// https://chromium.googlesource.com/chromium/src/+/master/net/http/transport_security_state_static.json
//
// Note that this list may be up to 12 weeks fresher than the list used
// by the current stable version of Chrome. See
// https://www.chromium.org/developers/calendar for a calendar of releases.
func NewFromLatest() (PreloadList, error) {
	return NewFromChromiumURL(LatestChromiumURL)
}

// NewFromFile reads a PreloadList from a JSON file.
//
// In a Chromium checkout, the file is at
// src/net/http/transport_security_state_static.json
func NewFromFile(fileName string) (PreloadList, error) {
	b, err := os.Open(fileName)
	if err != nil {
		return PreloadList{}, err
	}

	return Parse(b)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"errors"
	"flag"
	"github.com/miekg/dns"
	"github.com/rrva/dockerclient"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

func registerShutdownSignalHandlers() {
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
	for s := range sigChan {
		log.Fatalf("Signal (%s) received, shutting down", s)
	}
}

func matches(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}

type lookupResponse struct {
	response *string
	err      error
}

type lookupFn func(string) (*string, error)

func lookupWithTimeout(n string, t time.Duration, lookup lookupFn) (*string, error) {

	lookupResponses := make(chan lookupResponse)
	go func(n string) {
		name, err := lookup(n)
		lookupResponses <- lookupResponse{response: name, err: err}
	}(n)

	timeouts := make(chan bool, 1)
	go func() {
		<-time.After(t)
		timeouts <- true
	}()

	for {
		select {
		case r := <-lookupResponses:
			return r.response, r.err
		case <-timeouts:
			return nil, errors.New("Timeout occurred talking to docker")
		}
	}
}

func findContainerNameByIPAddress(n string) (*string, error) {
	containers, err := docker.ListContainers(false, false, "")
	if err != nil {
		return nil, err
	}
	for _, container := range containers {
		info, _ := docker.InspectContainer(container.Id)
		if info.NetworkSettings.IpAddress == n {
			if len(container.Names) > 0 {
				str := container.Names[0][1:]
				return &str, nil
			}
			return nil, nil
		}
	}
	return nil, nil
}

func findContainerIPAddressByName(n string) (*string, error) {
	containers, err := docker.ListContainers(false, false, "")
	if err != nil {
		return nil, err
	}
	for _, container := range containers {
		if matches(container.Names, n) {
			info, err := docker.InspectContainer(container.Id)
			if err != nil {
				return nil, err
			}
			if info != nil {
				return &info.NetworkSettings.IpAddress, nil
			}
		}
	}
	return nil, nil
}

func writeDNSResponse(w dns.ResponseWriter, m *dns.Msg) {
	err := w.WriteMsg(m)
	if err != nil {
		log.Printf("Failed writing DNS response: %s", err)
	}
}

func myselfAsResolverErrorResponse(w dns.ResponseWriter, m *dns.Msg, name string) {
	log.Printf("Oh my! I'm pointing to myself!")
	rr := new(dns.A)
	rr.Hdr = dns.RR_Header{Name: name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 1}
	rr.A = net.ParseIP("0.0.0.0").To4()
	m.Answer = append(m.Answer, rr)
	writeDNSResponse(w, m)
}

func loopbackSanityCheckResponse(w dns.ResponseWriter, req *dns.Msg) {
	m := new(dns.Msg)
	m.SetReply(req)

	for _, question := range req.Question {

		if question.Name == "dockerdns.loopbacktest." {
			myselfAsResolverErrorResponse(w, m, question.Name)
			return
		}
	}

	writeDNSResponse(w, m)
}

func localLookupServer(w dns.ResponseWriter, req *dns.Msg) {
	m := new(dns.Msg)
	m.SetReply(req)

	for _, question := range req.Question {

		if question.Qtype == dns.TypeA || question.Qtype == dns.TypeAAAA {
			IPAddresses, err := net.LookupHost(strings.TrimSuffix(question.Name, "."))

			if err != nil {
				log.Printf("Local lookup error: %s", err)
				m.SetRcode(req, dns.RcodeServerFailure)
				writeDNSResponse(w, m)
				return
			}

			for _, IPAddress := range IPAddresses {
				writeIPAddressResponse(&IPAddress, m, question.Name, question.Qtype)
			}
		}

	}

	writeDNSResponse(w, m)
}

func writeNameResponse(m *dns.Msg, ip string, name *string) {
	rr := new(dns.PTR)
	str := *name + "." + domain + "."
	rr.Hdr = dns.RR_Header{Name: ip, Rrtype: dns.TypePTR, Class: dns.ClassINET, Ttl: 1}
	rr.Ptr = str
	m.Answer = append(m.Answer, rr)
}

func writeIPAddressResponse(IPAddress *string, m *dns.Msg, name string, qtype uint16) {
	ip := net.ParseIP(*IPAddress)
	ip4 := ip.To4()
	if ip4 != nil && qtype == dns.TypeA {
		rr := new(dns.A)
		rr.Hdr = dns.RR_Header{Name: name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 1}

		rr.A = ip4
		m.Answer = append(m.Answer, rr)
	} else if ip4 == nil && qtype == dns.TypeAAAA {
		rr := new(dns.AAAA)
		rr.Hdr = dns.RR_Header{Name: name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 1}

		rr.AAAA = ip.To16()
		m.Answer = append(m.Answer, rr)
	}
}

func inAddrArpaToIpAddress(n string) string {
	ips := strings.Split(strings.Replace(n, ".in-addr.arpa.", "", 1), ".")
	s := ""
	last := len(ips)
	for i := range ips {
		s = s + ips[last-1-i]
		if i < last-1 {
			s = s + "."
		}
	}
	return s
}

func reverseLookupServer(w dns.ResponseWriter, req *dns.Msg) {
	m := new(dns.Msg)
	m.SetReply(req)

	for _, question := range req.Question {

		if question.Qtype != dns.TypePTR {
			continue
		}

		ip := inAddrArpaToIpAddress(question.Name)

		name, err := lookupWithTimeout(ip, 1*time.Second, findContainerNameByIPAddress)

		if err != nil {
			log.Printf("Container lookup error: %s", err)
			m.SetRcode(req, dns.RcodeServerFailure)
			writeDNSResponse(w, m)
			return
		}

		if name != nil {
			writeNameResponse(m, question.Name, name)
		} else {
			m.SetRcode(req, dns.RcodeRefused)
			log.Printf("No match found for %s sending empty reply", question.Name)
		}
	}

	writeDNSResponse(w, m)

}

func dockerNameLookupServer(w dns.ResponseWriter, req *dns.Msg) {
	m := new(dns.Msg)
	m.SetReply(req)

	for _, question := range req.Question {
		rr := new(dns.A)
		rr.Hdr = dns.RR_Header{Name: question.Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 1}

		name := "/" + strings.TrimSuffix(question.Name, "."+domain+".")

		IPAddress, err := lookupWithTimeout(name, 1*time.Second, findContainerIPAddressByName)

		if err != nil {
			log.Printf("Container lookup error: %s", err)
			callMeMaybe := "127.0.53.53"
			writeIPAddressResponse(&callMeMaybe, m, question.Name, question.Qtype)
			writeDNSResponse(w, m)
			return
		}

		if IPAddress != nil {
			writeIPAddressResponse(IPAddress, m, question.Name, question.Qtype)
		} else {
			log.Printf("No match found for %s sending empty reply", question.Name)
		}
	}

	writeDNSResponse(w, m)

}

func checkIfResolverIsMyself() {

	IPAddresses, _ := net.LookupHost("dockerdns.loopbacktest")

	if IPAddresses != nil && IPAddresses[0] == "0.0.0.0" {
		log.Fatal("Configuration error detected, I am pointing to myself as resolver, that won't work")
	}

}

func dnsStarted() {
	log.Printf("DNS server started")

	go checkIfResolverIsMyself()
}

func startDNS(addr string, errors chan error) {

	log.Printf("Trying to listen on %s", addr)
	server := &dns.Server{Addr: addr, Net: "udp", NotifyStartedFunc: dnsStarted}

	err := server.ListenAndServe()

	if err != nil {
		errors <- err
	}
}

var docker *dockerclient.DockerClient
var domain string

func newHTTPClient(host string, tlsConfig *tls.Config, timeout time.Duration) (*http.Client, *url.URL, error) {
	u, err := url.Parse(host)
	if err != nil {
		return nil, nil, err
	}

	httpTransport := &http.Transport{
		TLSClientConfig:       tlsConfig,
		ResponseHeaderTimeout: timeout,
	}

	switch u.Scheme {
	default:
		httpTransport.Dial = func(proto, addr string) (net.Conn, error) {
			return net.DialTimeout(proto, addr, timeout)
		}
	case "tcp":
		u.Scheme = "http"
	case "unix":
		socketPath := u.Path
		unixDial := func(proto, addr string) (net.Conn, error) {
			return net.DialTimeout("unix", socketPath, timeout)
		}
		httpTransport.Dial = unixDial
		u.Scheme = "http"
		u.Host = "unix.sock"
		u.Path = ""
	}
	return &http.Client{Transport: httpTransport}, u, nil
}

func main() {

	domain = *flag.String("domain", "dev", "domain")

	defaultDockerHost := os.Getenv("DOCKER_HOST")
	if defaultDockerHost == "" {
		defaultDockerHost = "tcp://docker:2375"
	}
	var dockerHost = *flag.String("docker-host", defaultDockerHost, "docker host url, or set DOCKER_HOST")
	var listenAddr = flag.String("listen-addr", ":53", "Listen address for DNS")
	var otherLookupsToLocalResolver = *flag.Bool("local-resolver", true, "Perform local gethostbyname queries for other domains")
	flag.Parse()

	var err error
	httpClient, url, err := newHTTPClient(dockerHost, nil, 100*time.Millisecond)
	if err != nil {
		log.Fatalf("Got error %s", err)
	}
	docker, err = dockerclient.NewDockerClientFromHttpClient(url, httpClient, nil)

	_, err = docker.Version()
	if err != nil {
		log.Fatalf("Failed reading docker version: %s specify host with -docker-host", err)
	}

	go registerShutdownSignalHandlers()

	log.Printf("Resolving %s domain against docker container names at %s", domain, dockerHost)
	if otherLookupsToLocalResolver {
		log.Println("Resolving other domains through system resolver")
		dns.HandleFunc(".", localLookupServer)
		dns.HandleFunc("loopbacktest", loopbackSanityCheckResponse)
	}
	dns.HandleFunc(domain, dockerNameLookupServer)
	dns.HandleFunc("arpa", reverseLookupServer)

	dnsListenErrors := make(chan error)
	go startDNS(*listenAddr, dnsListenErrors)
	err = <-dnsListenErrors
	log.Fatalf("Got error %s", err)
}
<commit_msg>Faster container lookup, better default TTL and timeout, fix AAAA queries<commit_after>package main

import (
	"crypto/tls"
	"errors"
	"flag"
	"github.com/miekg/dns"
	"github.com/rrva/dockerclient"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

func registerShutdownSignalHandlers() {
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
	for s := range sigChan {
		log.Fatalf("Signal (%s) received, shutting down", s)
	}
}

func matches(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}

type lookupResponse struct {
	response *string
	err      error
}

type lookupFn func(string) (*string, error)

func lookupWithTimeout(n string, t time.Duration, lookup lookupFn) (*string, error) {

	lookupResponses := make(chan lookupResponse)
	go func(n string) {
		name, err := lookup(n)
		lookupResponses <- lookupResponse{response: name, err: err}
	}(n)

	timeouts := make(chan bool, 1)
	go func() {
		<-time.After(t)
		timeouts <- true
	}()

	for {
		select {
		case r := <-lookupResponses:
			return r.response, r.err
		case <-timeouts:
			return nil, errors.New("Timeout occurred talking to docker")
		}
	}
}

func findContainerNameByIPAddress(n string) (*string, error) {
	containers, err := docker.ListContainers(false, false, "")
	if err != nil {
		return nil, err
	}
	for _, container := range containers {
		info, _ := docker.InspectContainer(container.Id)
		if info.NetworkSettings.IpAddress == n {
			if len(container.Names) > 0 {
				str := container.Names[0][1:]
				return &str, nil
			}
			return nil, nil
		}
	}
	return nil, nil
}

func findContainerIPAddressByName(n string) (*string, error) {
	info, err := docker.InspectContainer(n)
	if err != nil {
		return nil, err
	}
	if info != nil {
		return &info.NetworkSettings.IpAddress, nil
	}
	return nil, nil
}

func writeDNSResponse(w dns.ResponseWriter, m *dns.Msg) {
	err := w.WriteMsg(m)
	if err != nil {
		log.Printf("Failed writing DNS response: %s", err)
	}
}

func myselfAsResolverErrorResponse(w dns.ResponseWriter, m *dns.Msg, name string) {
	log.Printf("Oh my! I'm pointing to myself!")
	rr := new(dns.A)
	rr.Hdr = dns.RR_Header{Name: name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 5}
	rr.A = net.ParseIP("0.0.0.0").To4()
	m.Answer = append(m.Answer, rr)
	writeDNSResponse(w, m)
}

func loopbackSanityCheckResponse(w dns.ResponseWriter, req *dns.Msg) {
	m := new(dns.Msg)
	m.SetReply(req)

	for _, question := range req.Question {

		if question.Name == "dockerdns.loopbacktest." {
			myselfAsResolverErrorResponse(w, m, question.Name)
			return
		}
	}

	writeDNSResponse(w, m)
}

func localLookupServer(w dns.ResponseWriter, req *dns.Msg) {
	m := new(dns.Msg)
	m.SetReply(req)

	for _, question := range req.Question {

		if question.Qtype == dns.TypeA || question.Qtype == dns.TypeAAAA {
			IPAddresses, err := net.LookupHost(strings.TrimSuffix(question.Name, "."))

			if err != nil {
				log.Printf("Local lookup error: %s", err)
				m.SetRcode(req, dns.RcodeServerFailure)
				writeDNSResponse(w, m)
				return
			}

			for _, IPAddress := range IPAddresses {
				writeIPAddressResponse(&IPAddress, m, question.Name, question.Qtype)
			}
		}

	}

	writeDNSResponse(w, m)
}

func writeNameResponse(m *dns.Msg, ip string, name *string) {
	rr := new(dns.PTR)
	str := *name + "." + domain + "."
	rr.Hdr = dns.RR_Header{Name: ip, Rrtype: dns.TypePTR, Class: dns.ClassINET, Ttl: 5}
	rr.Ptr = str
	m.Answer = append(m.Answer, rr)
}

func writeIPAddressResponse(IPAddress *string, m *dns.Msg, name string, qtype uint16) {
	ip := net.ParseIP(*IPAddress)
	ip4 := ip.To4()
	if ip4 != nil && qtype == dns.TypeA {
		rr := new(dns.A)
		rr.Hdr = dns.RR_Header{Name: name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 5}

		rr.A = ip4
		m.Answer = append(m.Answer, rr)
	} else if ip4 == nil && qtype == dns.TypeAAAA {
		rr := new(dns.AAAA)
		rr.Hdr = dns.RR_Header{Name: name, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 5}

		rr.AAAA = ip.To16()
		m.Answer = append(m.Answer, rr)
	}
}

func inAddrArpaToIpAddress(n string) string {
	ips := strings.Split(strings.Replace(n, ".in-addr.arpa.", "", 1), ".")
	s := ""
	last := len(ips)
	for i := range ips {
		s = s + ips[last-1-i]
		if i < last-1 {
			s = s + "."
		}
	}
	return s
}

func reverseLookupServer(w dns.ResponseWriter, req *dns.Msg) {
	m := new(dns.Msg)
	m.SetReply(req)

	for _, question := range req.Question {

		if question.Qtype != dns.TypePTR {
			continue
		}

		ip := inAddrArpaToIpAddress(question.Name)

		name, err := lookupWithTimeout(ip, 5*time.Second, findContainerNameByIPAddress)

		if err != nil {
			log.Printf("Container lookup error: %s", err)
			m.SetRcode(req, dns.RcodeServerFailure)
			writeDNSResponse(w, m)
			return
		}

		if name != nil {
			writeNameResponse(m, question.Name, name)
		} else {
			m.SetRcode(req, dns.RcodeRefused)
			log.Printf("No match found for %s sending empty reply", question.Name)
		}
	}

	writeDNSResponse(w, m)

}

func dockerNameLookupServer(w dns.ResponseWriter, req *dns.Msg) {
	m := new(dns.Msg)
	m.SetReply(req)

	for _, question := range req.Question {
		rr := new(dns.A)
		rr.Hdr = dns.RR_Header{Name: question.Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 5}

		name := "/" + strings.TrimSuffix(question.Name, "."+domain+".")

		IPAddress, err := lookupWithTimeout(name, 5*time.Second, findContainerIPAddressByName)

		if err != nil {
			log.Printf("Container lookup error: %s", err)
			callMeMaybe := "127.0.53.53"
			writeIPAddressResponse(&callMeMaybe, m, question.Name, question.Qtype)
			writeDNSResponse(w, m)
			return
		}

		if IPAddress != nil {
			writeIPAddressResponse(IPAddress, m, question.Name, question.Qtype)
		} else {
			log.Printf("No match found for %s sending empty reply", question.Name)
		}
	}

	writeDNSResponse(w, m)

}

func checkIfResolverIsMyself() {

	IPAddresses, _ := net.LookupHost("dockerdns.loopbacktest")

	if IPAddresses != nil && IPAddresses[0] == "0.0.0.0" {
		log.Fatal("Configuration error detected, I am pointing to myself as resolver, that won't work")
	}

}

func dnsStarted() {
	log.Printf("DNS server started")

	go checkIfResolverIsMyself()
}

func startDNS(addr string, errors chan error) {

	log.Printf("Trying to listen on %s", addr)
	server := &dns.Server{Addr: addr, Net: "udp", NotifyStartedFunc: dnsStarted}

	err := server.ListenAndServe()

	if err != nil {
		errors <- err
	}
}

var docker *dockerclient.DockerClient
var domain string

func newHTTPClient(host string, tlsConfig *tls.Config, timeout time.Duration) (*http.Client, *url.URL, error) {
	u, err := url.Parse(host)
	if err != nil {
		return nil, nil, err
	}

	httpTransport := &http.Transport{
		TLSClientConfig:       tlsConfig,
		ResponseHeaderTimeout: timeout,
	}

	switch u.Scheme {
	default:
		httpTransport.Dial = func(proto, addr string) (net.Conn, error) {
			return net.DialTimeout(proto, addr, timeout)
		}
	case "tcp":
		u.Scheme = "http"
	case "unix":
		socketPath := u.Path
		unixDial := func(proto, addr string) (net.Conn, error) {
			return net.DialTimeout("unix", socketPath, timeout)
		}
		httpTransport.Dial = unixDial
		u.Scheme = "http"
		u.Host = "unix.sock"
		u.Path = ""
	}
	return &http.Client{Transport: httpTransport}, u, nil
}

func main() {

	domain = *flag.String("domain", "dev", "domain")

	defaultDockerHost := os.Getenv("DOCKER_HOST")
	if defaultDockerHost == "" {
		defaultDockerHost = "tcp://docker:2375"
	}
	var dockerHost = *flag.String("docker-host", defaultDockerHost, "docker host url, or set DOCKER_HOST")
	var listenAddr = flag.String("listen-addr", ":53", "Listen address for DNS")
	var otherLookupsToLocalResolver = *flag.Bool("local-resolver", true, "Perform local gethostbyname queries for other domains")
	flag.Parse()

	var err error
	httpClient, url, err := newHTTPClient(dockerHost, nil, 100*time.Millisecond)
	if err != nil {
		log.Fatalf("Got error %s", err)
	}
	docker, err = dockerclient.NewDockerClientFromHttpClient(url, httpClient, nil)

	_, err = docker.Version()
	if err != nil {
		log.Fatalf("Failed reading docker version: %s specify host with -docker-host", err)
	}

	go registerShutdownSignalHandlers()

	log.Printf("Resolving %s domain against docker container names at %s", domain, dockerHost)
	if otherLookupsToLocalResolver {
		log.Println("Resolving other domains through system resolver")
		dns.HandleFunc(".", localLookupServer)
		dns.HandleFunc("loopbacktest", loopbackSanityCheckResponse)
	}
	dns.HandleFunc(domain, dockerNameLookupServer)
	dns.HandleFunc("arpa", reverseLookupServer)

	dnsListenErrors := make(chan error)
	go startDNS(*listenAddr, dnsListenErrors)
	err = <-dnsListenErrors
	log.Fatalf("Got error %s", err)
}
<|endoftext|>