<commit_before>// Code generated by go generate. DO NOT EDIT.
// Source: vendor/istio.io/api/mixer/v1/global_dictionary.yaml

package attribute

func GlobalList() ([]string) {
    tmp := make([]string, len(globalList))
    copy(tmp, globalList)
    return tmp
}

var (
    globalList = []string{
		"source.ip",
		"source.port",
		"source.name",
		"source.uid",
		"source.namespace",
		"source.labels",
		"source.user",
		"target.ip",
		"target.port",
		"target.service",
		"target.name",
		"target.uid",
		"target.namespace",
		"target.labels",
		"target.user",
		"request.headers",
		"request.id",
		"request.path",
		"request.host",
		"request.method",
		"request.reason",
		"request.referer",
		"request.scheme",
		"request.size",
		"request.time",
		"request.useragent",
		"response.headers",
		"response.size",
		"response.time",
		"response.duration",
		"response.code",
		":authority",
		":method",
		":path",
		":scheme",
		":status",
		"access-control-allow-origin",
		"access-control-allow-methods",
		"access-control-allow-headers",
		"access-control-max-age",
		"access-control-request-method",
		"access-control-request-headers",
		"accept-charset",
		"accept-encoding",
		"accept-language",
		"accept-ranges",
		"accept",
		"access-control-allow",
		"age",
		"allow",
		"authorization",
		"cache-control",
		"content-disposition",
		"content-encoding",
		"content-language",
		"content-length",
		"content-location",
		"content-range",
		"content-type",
		"cookie",
		"date",
		"etag",
		"expect",
		"expires",
		"from",
		"host",
		"if-match",
		"if-modified-since",
		"if-none-match",
		"if-range",
		"if-unmodified-since",
		"keep-alive",
		"last-modified",
		"link",
		"location",
		"max-forwards",
		"proxy-authenticate",
		"proxy-authorization",
		"range",
		"referer",
		"refresh",
		"retry-after",
		"server",
		"set-cookie",
		"strict-transport-sec",
		"transfer-encoding",
		"user-agent",
		"vary",
		"via",
		"www-authenticate",
		"GET",
		"POST",
		"http",
		"envoy",
		"'200'",
		"Keep-Alive",
		"chunked",
		"x-envoy-service-time",
		"x-forwarded-for",
		"x-forwarded-host",
		"x-forwarded-proto",
		"x-http-method-override",
		"x-request-id",
		"x-requested-with",
		"application/json",
		"application/xml",
		"gzip",
		"text/html",
		"text/html; charset=utf-8",
		"text/plain",
		"text/plain; charset=utf-8",
		"'0'",
		"'1'",
		"true",
		"false",
		"gzip, deflate",
		"max-age=0",
		"x-envoy-upstream-service-time",
		"x-envoy-internal",
		"x-envoy-expected-rq-timeout-ms",
		"x-ot-span-context",
		"x-b3-traceid",
		"x-b3-sampled",
		"x-b3-spanid",
		"tcp",
		"connection.id",
		"connection.received.bytes",
		"connection.received.bytes_total",
		"connection.sent.bytes",
		"connection.sent.bytes_total",
		"connection.duration",
		"context.protocol",
		"context.timestamp",
		"context.time",
		"0",
		"1",
		"200",
		"302",
		"400",
		"401",
		"403",
		"404",
		"409",
		"429",
		"499",
		"500",
		"501",
		"502",
		"503",
		"504",
		"destination.ip",
		"destination.port",
		"destination.service",
		"destination.name",
		"destination.uid",
		"destination.namespace",
		"destination.labels",
		"destination.user",
		"source.service",
		"api.service",
		"api.version",
		"api.operation",
		"api.protocol",
		"request.auth.principal",
		"request.auth.audiences",
		"request.auth.presenter",
		"request.api_key",
		"check.error_code",
		"check.error_message",
		"request.total_size",
		"response.total_size",
		"connection.event",
		"check.cache_hit",
		"quota.cache_hit",
		"source.principal",
		"request.auth.claims",
		"request.auth.raw_claims",
		"connection.mtls",
		"source.metadata",
		"source.owner",
		"source.services",
		"source.workload.uid",
		"source.workload.name",
		"source.workload.namespace",
		"destination.metadata",
		"destination.owner",
		"destination.principal",
		"destination.workload.uid",
		"destination.workload.name",
		"destination.workload.namespace",
		"destination.service.uid",
		"destination.service.name",
		"destination.service.namespace",
		"destination.service.host",
		"destination.container.name",
		"destination.container.image",
		"context.reporter.local",
		"context.reporter.uid",
		"response.grpc_status",
		"response.grpc_message",
		"context.reporter.type",
		"context.reporter.kind",
		"connection.requested_server_name",
		"request.url_path",
		"request.query_params",
		"context.proxy_error_code",
		"UH",
		"UF",
		"UO",
		"NR",
		"LH",
		"UT",
		"LR",
		"UR",
		"UC",
		"DI",
		"FI",
		"RL",
		"\"-\"",
		"inbound",
		"outbound",
    }
)
<commit_msg>update linters (#10809)<commit_after>// Code generated by go generate. DO NOT EDIT.
// Source: vendor/istio.io/api/mixer/v1/global_dictionary.yaml

package attribute

func GlobalList() []string {
	tmp := make([]string, len(globalList))
	copy(tmp, globalList)
	return tmp
}

var (
	globalList = []string{
		"source.ip",
		"source.port",
		"source.name",
		"source.uid",
		"source.namespace",
		"source.labels",
		"source.user",
		"target.ip",
		"target.port",
		"target.service",
		"target.name",
		"target.uid",
		"target.namespace",
		"target.labels",
		"target.user",
		"request.headers",
		"request.id",
		"request.path",
		"request.host",
		"request.method",
		"request.reason",
		"request.referer",
		"request.scheme",
		"request.size",
		"request.time",
		"request.useragent",
		"response.headers",
		"response.size",
		"response.time",
		"response.duration",
		"response.code",
		":authority",
		":method",
		":path",
		":scheme",
		":status",
		"access-control-allow-origin",
		"access-control-allow-methods",
		"access-control-allow-headers",
		"access-control-max-age",
		"access-control-request-method",
		"access-control-request-headers",
		"accept-charset",
		"accept-encoding",
		"accept-language",
		"accept-ranges",
		"accept",
		"access-control-allow",
		"age",
		"allow",
		"authorization",
		"cache-control",
		"content-disposition",
		"content-encoding",
		"content-language",
		"content-length",
		"content-location",
		"content-range",
		"content-type",
		"cookie",
		"date",
		"etag",
		"expect",
		"expires",
		"from",
		"host",
		"if-match",
		"if-modified-since",
		"if-none-match",
		"if-range",
		"if-unmodified-since",
		"keep-alive",
		"last-modified",
		"link",
		"location",
		"max-forwards",
		"proxy-authenticate",
		"proxy-authorization",
		"range",
		"referer",
		"refresh",
		"retry-after",
		"server",
		"set-cookie",
		"strict-transport-sec",
		"transfer-encoding",
		"user-agent",
		"vary",
		"via",
		"www-authenticate",
		"GET",
		"POST",
		"http",
		"envoy",
		"'200'",
		"Keep-Alive",
		"chunked",
		"x-envoy-service-time",
		"x-forwarded-for",
		"x-forwarded-host",
		"x-forwarded-proto",
		"x-http-method-override",
		"x-request-id",
		"x-requested-with",
		"application/json",
		"application/xml",
		"gzip",
		"text/html",
		"text/html; charset=utf-8",
		"text/plain",
		"text/plain; charset=utf-8",
		"'0'",
		"'1'",
		"true",
		"false",
		"gzip, deflate",
		"max-age=0",
		"x-envoy-upstream-service-time",
		"x-envoy-internal",
		"x-envoy-expected-rq-timeout-ms",
		"x-ot-span-context",
		"x-b3-traceid",
		"x-b3-sampled",
		"x-b3-spanid",
		"tcp",
		"connection.id",
		"connection.received.bytes",
		"connection.received.bytes_total",
		"connection.sent.bytes",
		"connection.sent.bytes_total",
		"connection.duration",
		"context.protocol",
		"context.timestamp",
		"context.time",
		"0",
		"1",
		"200",
		"302",
		"400",
		"401",
		"403",
		"404",
		"409",
		"429",
		"499",
		"500",
		"501",
		"502",
		"503",
		"504",
		"destination.ip",
		"destination.port",
		"destination.service",
		"destination.name",
		"destination.uid",
		"destination.namespace",
		"destination.labels",
		"destination.user",
		"source.service",
		"api.service",
		"api.version",
		"api.operation",
		"api.protocol",
		"request.auth.principal",
		"request.auth.audiences",
		"request.auth.presenter",
		"request.api_key",
		"check.error_code",
		"check.error_message",
		"request.total_size",
		"response.total_size",
		"connection.event",
		"check.cache_hit",
		"quota.cache_hit",
		"source.principal",
		"request.auth.claims",
		"request.auth.raw_claims",
		"connection.mtls",
		"source.metadata",
		"source.owner",
		"source.services",
		"source.workload.uid",
		"source.workload.name",
		"source.workload.namespace",
		"destination.metadata",
		"destination.owner",
		"destination.principal",
		"destination.workload.uid",
		"destination.workload.name",
		"destination.workload.namespace",
		"destination.service.uid",
		"destination.service.name",
		"destination.service.namespace",
		"destination.service.host",
		"destination.container.name",
		"destination.container.image",
		"context.reporter.local",
		"context.reporter.uid",
		"response.grpc_status",
		"response.grpc_message",
		"context.reporter.type",
		"context.reporter.kind",
		"connection.requested_server_name",
		"request.url_path",
		"request.query_params",
		"context.proxy_error_code",
		"UH",
		"UF",
		"UO",
		"NR",
		"LH",
		"UT",
		"LR",
		"UR",
		"UC",
		"DI",
		"FI",
		"RL",
		"\"-\"",
		"inbound",
		"outbound",
	}
)
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net"
	"strconv"

	baps3 "github.com/UniversityRadioYork/baps3-go"
)

type clientAndMessage struct {
	c   *Client
	msg baps3.Message
}

// Maintains communications with the downstream service and connected clients.
// Also does any processing needed with the commands.
type hub struct {
	// All current clients.
	clients map[*Client]bool

	// Downstream service state
	downstreamState baps3.ServiceState

	// Playlist instance
	pl *Playlist

	// For communication with the downstream service.
	cReqCh chan<- baps3.Message
	cResCh <-chan baps3.Message

	// Where new requests from clients come through.
	reqCh chan clientAndMessage

	// Handlers for adding/removing connections.
	addCh chan *Client
	rmCh  chan *Client
	Quit  chan bool
}

// Handles a new client connection.
// conn is the new connection object.
func (h *hub) handleNewConnection(conn net.Conn) {
	defer conn.Close()
	client := &Client{
		conn:  conn,
		resCh: make(chan baps3.Message),
		tok:   baps3.NewTokeniser(),
	}

	// Register user
	h.addCh <- client

	go client.Read(h.reqCh, h.rmCh)
	client.Write(client.resCh, h.rmCh)
}

// Appends the downstream service's version (from the OHAI) to the listd version.
func (h *hub) makeRsOhai() *baps3.Message {
	return baps3.NewMessage(baps3.RsOhai).AddArg("listd " + LD_VERSION + "/" + h.downstreamState.Identifier)
}

// Crafts the features message by adding listd's features to the downstream service's and removing
// features listd intercepts.
func (h *hub) makeRsFeatures() (msg *baps3.Message) {
	features := h.downstreamState.Features
	features.DelFeature(baps3.FtFileLoad) // 'Mask' the features listd intercepts
	features.AddFeature(baps3.FtPlaylist)
	features.AddFeature(baps3.FtPlaylistTextItems)
	msg = features.ToMessage()
	return
}

// Collates all the responses that comprise a dump response.
// Exists as this is used by the dump response handler /and/ is sent on client connection
func (h *hub) makeDumpResponses() (msgs []*baps3.Message) {
	msgs = append(msgs, baps3.NewMessage(baps3.RsState).AddArg(h.downstreamState.State.String()))
	if h.downstreamState.State != baps3.StEjected {
		msgs = append(msgs, baps3.NewMessage(baps3.RsTime).AddArg(
			strconv.FormatInt(h.downstreamState.Time.Nanoseconds()/1000, 10)))
	}
	msgs = append(msgs, h.makeListResponses()...)
	return
}

// Collates all the responses that comprise a list reponse.
// Exists as this is used by the list response handler and makeDumpResponse.
func (h *hub) makeListResponses() (msgs []*baps3.Message) {
	msgs = append(msgs, baps3.NewMessage(baps3.RsCount).AddArg(strconv.Itoa(len(h.pl.items))))
	for i, item := range h.pl.items {
		typeStr := "file"
		if !item.IsFile {
			typeStr = "text"
		}
		msgs = append(msgs, baps3.NewMessage(baps3.RsItem).AddArg(strconv.Itoa(i)).AddArg(item.Hash).AddArg(typeStr).AddArg(item.Data))
	}
	return
}

func sendInvalidCmd(c *Client, errRes baps3.Message, oldCmd baps3.Message) {
	for _, w := range oldCmd.AsSlice() {
		errRes.AddArg(w)
	}
	c.resCh <- errRes
}

func (h *hub) processReqDequeue(req baps3.Message) (resps []*baps3.Message) {
	args := req.AsSlice()[1:]
	if len(args) != 2 {
		return append(resps, baps3.NewMessage(baps3.RsWhat).AddArg("Bad command"))
	}
	iStr, hash := args[0], args[1]

	i, err := strconv.Atoi(iStr)
	if err != nil {
		return append(resps, baps3.NewMessage(baps3.RsWhat).AddArg("Bad index"))
	}

	oldSelection := h.pl.selection
	rmIdx, rmHash, err := h.pl.Dequeue(i, hash)
	if err != nil {
		return append(resps, baps3.NewMessage(baps3.RsFail).AddArg(err.Error()))
	}
	if oldSelection != h.pl.selection {
		if !h.pl.HasSelection() {
			resps = append(resps, baps3.NewMessage(baps3.RsSelect))
		} else {
			resps = append(resps, baps3.NewMessage(baps3.RsSelect).AddArg(strconv.Itoa(h.pl.selection)).AddArg(h.pl.items[h.pl.selection].Hash))
		}
	}
	return append(resps, baps3.NewMessage(baps3.RsDequeue).AddArg(strconv.Itoa(rmIdx)).AddArg(rmHash))
}

func (h *hub) processReqEnqueue(req baps3.Message) (resps []*baps3.Message) {
	args := req.AsSlice()[1:]
	if len(args) != 4 {
		return append(resps, baps3.NewMessage(baps3.RsWhat).AddArg("Bad command"))
	}
	iStr, hash, itemType, data := args[0], args[1], args[2], args[3]

	i, err := strconv.Atoi(iStr)
	if err != nil {
		return append(resps, baps3.NewMessage(baps3.RsWhat).AddArg("Bad index"))
	}

	if itemType != "file" && itemType != "text" {
		return append(resps, baps3.NewMessage(baps3.RsWhat).AddArg("Bad item type"))
	}

	oldSelection := h.pl.selection
	item := &PlaylistItem{Data: data, Hash: hash, IsFile: itemType == "file"}
	newIdx, err := h.pl.Enqueue(i, item)
	if err != nil {
		return append(resps, baps3.NewMessage(baps3.RsFail).AddArg(err.Error()))
	}
	if oldSelection != h.pl.selection {
		resps = append(resps, baps3.NewMessage(baps3.RsSelect).AddArg(strconv.Itoa(h.pl.selection)).AddArg(h.pl.items[h.pl.selection].Hash))
	}
	return append(resps, baps3.NewMessage(baps3.RsEnqueue).AddArg(strconv.Itoa(newIdx)).AddArg(item.Hash).AddArg(itemType).AddArg(item.Data))
}

func (h *hub) processReqSelect(req baps3.Message) (resps []*baps3.Message) {
	args := req.AsSlice()[1:]
	if len(args) == 0 {
		if h.pl.HasSelection() {
			// Remove current selection
			h.pl.selection = -1
			resps = append(resps, baps3.NewMessage(baps3.RsSelect))
		} else {
			// TODO: Should we care about there not being an existing selection?
			resps = append(resps, baps3.NewMessage(baps3.RsFail).AddArg("No selection to remove"))
		}
	} else if len(args) == 2 {
		iStr, hash := args[0], args[1]

		i, err := strconv.Atoi(iStr)
		if err != nil {
			return append(resps, baps3.NewMessage(baps3.RsWhat).AddArg("Bad index"))
		}

		newIdx, newHash, err := h.pl.Select(i, hash)
		if err != nil {
			return append(resps, baps3.NewMessage(baps3.RsFail).AddArg(err.Error()))
		}

		resps = append(resps, baps3.NewMessage(baps3.RsSelect).AddArg(strconv.Itoa(newIdx)).AddArg(newHash))
	} else {
		resps = append(resps, baps3.NewMessage(baps3.RsWhat).AddArg("Bad command"))
	}
	return
}

func (h *hub) processReqList(req baps3.Message) (resps []*baps3.Message) {
	resps = h.makeListResponses()
	return
}

func (h *hub) processReqLoadEject(req baps3.Message) (resps []*baps3.Message) {
	return append(resps, baps3.NewMessage(baps3.RsWhat).AddArg("Bad command"))
}

func (h *hub) processReqDump(req baps3.Message) (msgs []*baps3.Message) {
	msgs = h.makeDumpResponses()
	return
}

var REQ_FUNC_MAP = map[baps3.MessageWord]func(*hub, baps3.Message) []*baps3.Message{
	baps3.RqEnqueue: (*hub).processReqEnqueue,
	baps3.RqDequeue: (*hub).processReqDequeue,
	baps3.RqSelect:  (*hub).processReqSelect,
	baps3.RqList:    (*hub).processReqList,
	baps3.RqLoad:    (*hub).processReqLoadEject,
	baps3.RqEject:   (*hub).processReqLoadEject,
	baps3.RqDump:    (*hub).processReqDump,
}

// Handles a request from a client.
// Falls through to the connector cReqCh if command is "not understood".
func (h *hub) processRequest(c *Client, req baps3.Message) {
	log.Println("New request:", req.String())
	if reqFunc, ok := REQ_FUNC_MAP[req.Word()]; ok {
		responses := reqFunc(h, req)
		// Of course one of them is special...
		if req.Word() == baps3.RqSelect {
			if h.pl.HasSelection() {
				h.cReqCh <- *baps3.NewMessage(baps3.RqLoad).AddArg(h.pl.items[h.pl.selection].Data)
			} else {
				h.cReqCh <- *baps3.NewMessage(baps3.RqEject)
			}
		}
		for _, resp := range responses {
			// TODO: Add a "is fail word" func to baps3-go?
			if resp.Word() == baps3.RsFail || resp.Word() == baps3.RsWhat {
				// failures only go to sender
				sendInvalidCmd(c, *resp, req)
			} else {
				h.broadcast(*resp)
			}
		}
	} else {
		h.cReqCh <- req
	}
}

// Processes a response from the downstream service.
func (h *hub) processResponse(res baps3.Message) {
	log.Println("New response:", res.String())
	switch res.Word() {
	case baps3.RsTime, baps3.RsState: // Broadcast _AND_ update state
		h.broadcast(res)
		fallthrough
	case baps3.RsOhai, baps3.RsFeatures: // Just update state
		if err := h.downstreamState.Update(res); err != nil {
			log.Fatal("Error updating state: " + err.Error())
		}
	default:
		h.broadcast(res)
	}
}

// Send a response message to all clients.
func (h *hub) broadcast(res baps3.Message) {
	for c, _ := range h.clients {
		c.resCh <- res
	}
}

// Listens for new connections on addr:port and spins up the relevant goroutines.
func (h *hub) runListener(addr string, port string) {
	netListener, err := net.Listen("tcp", addr+":"+port)
	if err != nil {
		log.Println("Listening error:", err.Error())
		return
	}

	// Get new connections
	go func() {
		for {
			conn, err := netListener.Accept()
			if err != nil {
				log.Println("Error accepting connection:", err.Error())
				continue
			}

			go h.handleNewConnection(conn)
		}
	}()

	for {
		select {
		case msg := <-h.cResCh:
			h.processResponse(msg)
		case data := <-h.reqCh:
			h.processRequest(data.c, data.msg)
		case client := <-h.addCh:
			h.clients[client] = true
			client.resCh <- *h.makeRsOhai()
			client.resCh <- *h.makeRsFeatures()
			for _, msg := range h.makeDumpResponses() {
				client.resCh <- *msg
			}
			log.Println("New connection from", client.conn.RemoteAddr())
		case client := <-h.rmCh:
			close(client.resCh)
			delete(h.clients, client)
			log.Println("Closed connection from", client.conn.RemoteAddr())
		case <-h.Quit:
			log.Println("Closing all connections")
			for c, _ := range h.clients {
				close(c.resCh)
				delete(h.clients, c)
			}
			//			h.Quit <- true
		}
	}
}

// Sets up the connector channels for the hub object.
func (h *hub) setConnector(cReqCh chan<- baps3.Message, cResCh <-chan baps3.Message) {
	h.cReqCh = cReqCh
	h.cResCh = cResCh
}
<commit_msg>Use Args() rather than AsSlice()<commit_after>package main

import (
	"log"
	"net"
	"strconv"

	baps3 "github.com/UniversityRadioYork/baps3-go"
)

type clientAndMessage struct {
	c   *Client
	msg baps3.Message
}

// Maintains communications with the downstream service and connected clients.
// Also does any processing needed with the commands.
type hub struct {
	// All current clients.
	clients map[*Client]bool

	// Downstream service state
	downstreamState baps3.ServiceState

	// Playlist instance
	pl *Playlist

	// For communication with the downstream service.
	cReqCh chan<- baps3.Message
	cResCh <-chan baps3.Message

	// Where new requests from clients come through.
	reqCh chan clientAndMessage

	// Handlers for adding/removing connections.
	addCh chan *Client
	rmCh  chan *Client
	Quit  chan bool
}

// Handles a new client connection.
// conn is the new connection object.
func (h *hub) handleNewConnection(conn net.Conn) {
	defer conn.Close()
	client := &Client{
		conn:  conn,
		resCh: make(chan baps3.Message),
		tok:   baps3.NewTokeniser(),
	}

	// Register user
	h.addCh <- client

	go client.Read(h.reqCh, h.rmCh)
	client.Write(client.resCh, h.rmCh)
}

// Appends the downstream service's version (from the OHAI) to the listd version.
func (h *hub) makeRsOhai() *baps3.Message {
	return baps3.NewMessage(baps3.RsOhai).AddArg("listd " + LD_VERSION + "/" + h.downstreamState.Identifier)
}

// Crafts the features message by adding listd's features to the downstream service's and removing
// features listd intercepts.
func (h *hub) makeRsFeatures() (msg *baps3.Message) {
	features := h.downstreamState.Features
	features.DelFeature(baps3.FtFileLoad) // 'Mask' the features listd intercepts
	features.AddFeature(baps3.FtPlaylist)
	features.AddFeature(baps3.FtPlaylistTextItems)
	msg = features.ToMessage()
	return
}

// Collates all the responses that comprise a dump response.
// Exists as this is used by the dump response handler /and/ is sent on client connection
func (h *hub) makeDumpResponses() (msgs []*baps3.Message) {
	msgs = append(msgs, baps3.NewMessage(baps3.RsState).AddArg(h.downstreamState.State.String()))
	if h.downstreamState.State != baps3.StEjected {
		msgs = append(msgs, baps3.NewMessage(baps3.RsTime).AddArg(
			strconv.FormatInt(h.downstreamState.Time.Nanoseconds()/1000, 10)))
	}
	msgs = append(msgs, h.makeListResponses()...)
	return
}

// Collates all the responses that comprise a list reponse.
// Exists as this is used by the list response handler and makeDumpResponse.
func (h *hub) makeListResponses() (msgs []*baps3.Message) {
	msgs = append(msgs, baps3.NewMessage(baps3.RsCount).AddArg(strconv.Itoa(len(h.pl.items))))
	for i, item := range h.pl.items {
		typeStr := "file"
		if !item.IsFile {
			typeStr = "text"
		}
		msgs = append(msgs, baps3.NewMessage(baps3.RsItem).AddArg(strconv.Itoa(i)).AddArg(item.Hash).AddArg(typeStr).AddArg(item.Data))
	}
	return
}

func sendInvalidCmd(c *Client, errRes baps3.Message, oldCmd baps3.Message) {
	for _, w := range oldCmd.AsSlice() {
		errRes.AddArg(w)
	}
	c.resCh <- errRes
}

func (h *hub) processReqDequeue(req baps3.Message) (resps []*baps3.Message) {
	if len(req.Args()) != 2 {
		return append(resps, baps3.NewMessage(baps3.RsWhat).AddArg("Bad command"))
	}
	iStr, hash := req.Args()[0], req.Args()[1]

	i, err := strconv.Atoi(iStr)
	if err != nil {
		return append(resps, baps3.NewMessage(baps3.RsWhat).AddArg("Bad index"))
	}

	oldSelection := h.pl.selection
	rmIdx, rmHash, err := h.pl.Dequeue(i, hash)
	if err != nil {
		return append(resps, baps3.NewMessage(baps3.RsFail).AddArg(err.Error()))
	}
	if oldSelection != h.pl.selection {
		if !h.pl.HasSelection() {
			resps = append(resps, baps3.NewMessage(baps3.RsSelect))
		} else {
			resps = append(resps, baps3.NewMessage(baps3.RsSelect).AddArg(strconv.Itoa(h.pl.selection)).AddArg(h.pl.items[h.pl.selection].Hash))
		}
	}
	return append(resps, baps3.NewMessage(baps3.RsDequeue).AddArg(strconv.Itoa(rmIdx)).AddArg(rmHash))
}

func (h *hub) processReqEnqueue(req baps3.Message) (resps []*baps3.Message) {
	args := req.Args()
	if len(args) != 4 {
		return append(resps, baps3.NewMessage(baps3.RsWhat).AddArg("Bad command"))
	}
	iStr, hash, itemType, data := args[0], args[1], args[2], args[3]

	i, err := strconv.Atoi(iStr)
	if err != nil {
		return append(resps, baps3.NewMessage(baps3.RsWhat).AddArg("Bad index"))
	}

	if itemType != "file" && itemType != "text" {
		return append(resps, baps3.NewMessage(baps3.RsWhat).AddArg("Bad item type"))
	}

	oldSelection := h.pl.selection
	item := &PlaylistItem{Data: data, Hash: hash, IsFile: itemType == "file"}
	newIdx, err := h.pl.Enqueue(i, item)
	if err != nil {
		return append(resps, baps3.NewMessage(baps3.RsFail).AddArg(err.Error()))
	}
	if oldSelection != h.pl.selection {
		resps = append(resps, baps3.NewMessage(baps3.RsSelect).AddArg(strconv.Itoa(h.pl.selection)).AddArg(h.pl.items[h.pl.selection].Hash))
	}
	return append(resps, baps3.NewMessage(baps3.RsEnqueue).AddArg(strconv.Itoa(newIdx)).AddArg(item.Hash).AddArg(itemType).AddArg(item.Data))
}

func (h *hub) processReqSelect(req baps3.Message) (resps []*baps3.Message) {
	if len(req.Args()) == 0 {
		if h.pl.HasSelection() {
			// Remove current selection
			h.pl.selection = -1
			resps = append(resps, baps3.NewMessage(baps3.RsSelect))
		} else {
			// TODO: Should we care about there not being an existing selection?
			resps = append(resps, baps3.NewMessage(baps3.RsFail).AddArg("No selection to remove"))
		}
	} else if len(req.Args()) == 2 {
		iStr, hash := req.Args()[0], req.Args()[1]

		i, err := strconv.Atoi(iStr)
		if err != nil {
			return append(resps, baps3.NewMessage(baps3.RsWhat).AddArg("Bad index"))
		}

		newIdx, newHash, err := h.pl.Select(i, hash)
		if err != nil {
			return append(resps, baps3.NewMessage(baps3.RsFail).AddArg(err.Error()))
		}

		resps = append(resps, baps3.NewMessage(baps3.RsSelect).AddArg(strconv.Itoa(newIdx)).AddArg(newHash))
	} else {
		resps = append(resps, baps3.NewMessage(baps3.RsWhat).AddArg("Bad command"))
	}
	return
}

func (h *hub) processReqList(req baps3.Message) (resps []*baps3.Message) {
	resps = h.makeListResponses()
	return
}

func (h *hub) processReqLoadEject(req baps3.Message) (resps []*baps3.Message) {
	return append(resps, baps3.NewMessage(baps3.RsWhat).AddArg("Bad command"))
}

func (h *hub) processReqDump(req baps3.Message) (msgs []*baps3.Message) {
	msgs = h.makeDumpResponses()
	return
}

var REQ_FUNC_MAP = map[baps3.MessageWord]func(*hub, baps3.Message) []*baps3.Message{
	baps3.RqEnqueue: (*hub).processReqEnqueue,
	baps3.RqDequeue: (*hub).processReqDequeue,
	baps3.RqSelect:  (*hub).processReqSelect,
	baps3.RqList:    (*hub).processReqList,
	baps3.RqLoad:    (*hub).processReqLoadEject,
	baps3.RqEject:   (*hub).processReqLoadEject,
	baps3.RqDump:    (*hub).processReqDump,
}

// Handles a request from a client.
// Falls through to the connector cReqCh if command is "not understood".
func (h *hub) processRequest(c *Client, req baps3.Message) {
	log.Println("New request:", req.String())
	if reqFunc, ok := REQ_FUNC_MAP[req.Word()]; ok {
		responses := reqFunc(h, req)
		// Of course one of them is special...
		if req.Word() == baps3.RqSelect {
			if h.pl.HasSelection() {
				h.cReqCh <- *baps3.NewMessage(baps3.RqLoad).AddArg(h.pl.items[h.pl.selection].Data)
			} else {
				h.cReqCh <- *baps3.NewMessage(baps3.RqEject)
			}
		}
		for _, resp := range responses {
			// TODO: Add a "is fail word" func to baps3-go?
			if resp.Word() == baps3.RsFail || resp.Word() == baps3.RsWhat {
				// failures only go to sender
				sendInvalidCmd(c, *resp, req)
			} else {
				h.broadcast(*resp)
			}
		}
	} else {
		h.cReqCh <- req
	}
}

// Processes a response from the downstream service.
func (h *hub) processResponse(res baps3.Message) {
	log.Println("New response:", res.String())
	switch res.Word() {
	case baps3.RsTime, baps3.RsState: // Broadcast _AND_ update state
		h.broadcast(res)
		fallthrough
	case baps3.RsOhai, baps3.RsFeatures: // Just update state
		if err := h.downstreamState.Update(res); err != nil {
			log.Fatal("Error updating state: " + err.Error())
		}
	default:
		h.broadcast(res)
	}
}

// Send a response message to all clients.
func (h *hub) broadcast(res baps3.Message) {
	for c, _ := range h.clients {
		c.resCh <- res
	}
}

// Listens for new connections on addr:port and spins up the relevant goroutines.
func (h *hub) runListener(addr string, port string) {
	netListener, err := net.Listen("tcp", addr+":"+port)
	if err != nil {
		log.Println("Listening error:", err.Error())
		return
	}

	// Get new connections
	go func() {
		for {
			conn, err := netListener.Accept()
			if err != nil {
				log.Println("Error accepting connection:", err.Error())
				continue
			}

			go h.handleNewConnection(conn)
		}
	}()

	for {
		select {
		case msg := <-h.cResCh:
			h.processResponse(msg)
		case data := <-h.reqCh:
			h.processRequest(data.c, data.msg)
		case client := <-h.addCh:
			h.clients[client] = true
			client.resCh <- *h.makeRsOhai()
			client.resCh <- *h.makeRsFeatures()
			for _, msg := range h.makeDumpResponses() {
				client.resCh <- *msg
			}
			log.Println("New connection from", client.conn.RemoteAddr())
		case client := <-h.rmCh:
			close(client.resCh)
			delete(h.clients, client)
			log.Println("Closed connection from", client.conn.RemoteAddr())
		case <-h.Quit:
			log.Println("Closing all connections")
			for c, _ := range h.clients {
				close(c.resCh)
				delete(h.clients, c)
			}
			//			h.Quit <- true
		}
	}
}

// Sets up the connector channels for the hub object.
func (h *hub) setConnector(cReqCh chan<- baps3.Message, cResCh <-chan baps3.Message) {
	h.cReqCh = cReqCh
	h.cResCh = cResCh
}
<|endoftext|>


<commit_before>package lmdb

/*
#include <stdlib.h>
#include <stdio.h>
#include "lmdb.h"
*/
import "C"

import (
	"reflect"
	"unsafe"
)

// Multi is a wrapper for a contiguous page of sorted, fixed-length values
// passed to Cursor.PutMulti or retrieved using Cursor.Get with the
// GetMultiple/NextMultiple flag.
//
// Multi values are only useful in databases opened with DupSort|DupFixed.
type Multi struct {
	page   []byte
	stride int
}

// WrapMulti converts a page of contiguous values with stride size into a
// Multi.  WrapMulti panics if len(page) is not a multiple of stride.
//
//		_, val, _ := cursor.Get(nil, nil, lmdb.FirstDup)
//		_, page, _ := cursor.Get(nil, nil, lmdb.GetMultiple)
//		multi := lmdb.WrapMulti(page, len(val))
//
// See mdb_cursor_get and MDB_GET_MULTIPLE.
func WrapMulti(page []byte, stride int) *Multi {
	if len(page)%stride != 0 {
		panic("incongruent arguments")
	}
	return &Multi{page: page, stride: stride}
}

// Vals returns a slice containing the values in m.  The returned slice has
// length m.Len() and each item has length m.Stride().
func (m *Multi) Vals() [][]byte {
	n := m.Len()
	ps := make([][]byte, n)
	for i := 0; i < n; i++ {
		ps[i] = m.Val(i)
	}
	return ps
}

// Val returns the value at index i.  Val panics if i is out of range.
func (m *Multi) Val(i int) []byte {
	off := i * m.stride
	return m.page[off : off+m.stride]
}

// Len returns the number of values in the Multi.
func (m *Multi) Len() int {
	return len(m.page) / m.stride
}

// Stride returns the length of an individual value in the m.
func (m *Multi) Stride() int {
	return m.stride
}

// Size returns the total size of the Multi data and is equal to
//
//		m.Len()*m.Stride()
//
func (m *Multi) Size() int {
	return len(m.page)
}

// Page returns the Multi page data as a raw slice of bytes with length
// m.Size().
func (m *Multi) Page() []byte {
	return m.page[:len(m.page):len(m.page)]
}

var eb = []byte{0}

func valBytes(b []byte) ([]byte, int) {
	if len(b) == 0 {
		return eb, 0
	}
	return b, len(b)
}

func wrapVal(b []byte) *C.MDB_val {
	p, n := valBytes(b)
	return &C.MDB_val{
		mv_data: unsafe.Pointer(&p[0]),
		mv_size: C.size_t(n),
	}
}

func getBytes(val *C.MDB_val) []byte {
	hdr := reflect.SliceHeader{
		Data: uintptr(unsafe.Pointer(val.mv_data)),
		Len:  int(val.mv_size),
		Cap:  int(val.mv_size),
	}
	return *(*[]byte)(unsafe.Pointer(&hdr))
}

func getBytesCopy(val *C.MDB_val) []byte {
	return C.GoBytes(val.mv_data, C.int(val.mv_size))
}
<commit_msg>lmdb: use a more reliable conversion to []byte from MDB_val<commit_after>package lmdb

/*
#include <stdlib.h>
#include <stdio.h>
#include "lmdb.h"
*/
import "C"

import "unsafe"

const valMaxSize = 1<<32 - 1

// Multi is a wrapper for a contiguous page of sorted, fixed-length values
// passed to Cursor.PutMulti or retrieved using Cursor.Get with the
// GetMultiple/NextMultiple flag.
//
// Multi values are only useful in databases opened with DupSort|DupFixed.
type Multi struct {
	page   []byte
	stride int
}

// WrapMulti converts a page of contiguous values with stride size into a
// Multi.  WrapMulti panics if len(page) is not a multiple of stride.
//
//		_, val, _ := cursor.Get(nil, nil, lmdb.FirstDup)
//		_, page, _ := cursor.Get(nil, nil, lmdb.GetMultiple)
//		multi := lmdb.WrapMulti(page, len(val))
//
// See mdb_cursor_get and MDB_GET_MULTIPLE.
func WrapMulti(page []byte, stride int) *Multi {
	if len(page)%stride != 0 {
		panic("incongruent arguments")
	}
	return &Multi{page: page, stride: stride}
}

// Vals returns a slice containing the values in m.  The returned slice has
// length m.Len() and each item has length m.Stride().
func (m *Multi) Vals() [][]byte {
	n := m.Len()
	ps := make([][]byte, n)
	for i := 0; i < n; i++ {
		ps[i] = m.Val(i)
	}
	return ps
}

// Val returns the value at index i.  Val panics if i is out of range.
func (m *Multi) Val(i int) []byte {
	off := i * m.stride
	return m.page[off : off+m.stride]
}

// Len returns the number of values in the Multi.
func (m *Multi) Len() int {
	return len(m.page) / m.stride
}

// Stride returns the length of an individual value in the m.
func (m *Multi) Stride() int {
	return m.stride
}

// Size returns the total size of the Multi data and is equal to
//
//		m.Len()*m.Stride()
//
func (m *Multi) Size() int {
	return len(m.page)
}

// Page returns the Multi page data as a raw slice of bytes with length
// m.Size().
func (m *Multi) Page() []byte {
	return m.page[:len(m.page):len(m.page)]
}

var eb = []byte{0}

func valBytes(b []byte) ([]byte, int) {
	if len(b) == 0 {
		return eb, 0
	}
	return b, len(b)
}

func wrapVal(b []byte) *C.MDB_val {
	p, n := valBytes(b)
	return &C.MDB_val{
		mv_data: unsafe.Pointer(&p[0]),
		mv_size: C.size_t(n),
	}
}

func getBytes(val *C.MDB_val) []byte {
	return (*[valMaxSize]byte)(unsafe.Pointer(val.mv_data))[:val.mv_size:val.mv_size]
}

func getBytesCopy(val *C.MDB_val) []byte {
	return C.GoBytes(val.mv_data, C.int(val.mv_size))
}
<|endoftext|>


<commit_before>package blorg

import (
	"fmt"
	"io/ioutil"
	"os/exec"
	"strings"
	"testing"
)

func TestBlorg(t *testing.T) {
	config, err := ReadConfig("testdata/blorg.org")
	if err != nil {
		t.Errorf("Could not read config: %s", err)
		return
	}
	committedHashBs, err := ioutil.ReadFile("testdata/public.md5")
	if err != nil {
		t.Errorf("Could not read hash bytes: %s", err)
		return
	}
	if err := config.Render(); err != nil {
		t.Errorf("Could not render: %s", err)
		return
	}
	renderedHashBs, err := exec.Command("bash", "-c", fmt.Sprintf("find %s -type f | sort -u | xargs md5sum", config.PublicDir)).Output()
	if err != nil {
		t.Errorf("Could not hash PublicDir: %s", err)
		return
	}
	rendered, committed := strings.TrimSpace(string(renderedHashBs)), strings.TrimSpace(string(committedHashBs))
	if rendered != committed {
		t.Errorf("PublicDir hashes do not match: '%s' -> '%s'", committed, rendered)
		return
	}
}
<commit_msg>Rewrite blorg tests without external md5sum<commit_after>package blorg

import (
	"bufio"
	"crypto/md5"
	"encoding/hex"
	"io/fs"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

func TestBlorg(t *testing.T) {
	// Re-generate this file with `find testdata/public -type f | sort -u | xargs md5sum > testdata/public.md5`
	hashFile, err := os.Open("testdata/public.md5")
	if err != nil {
		t.Errorf("Could not open hash file: %s", err)
		return
	}
	defer hashFile.Close()
	scanner := bufio.NewScanner(hashFile)
	committedHashes := make(map[string]string)
	for scanner.Scan() {
		parts := strings.Fields(scanner.Text())
		if len(parts) != 2 {
			t.Errorf("Could not split hash entry line in 2: len(parts)=%d", len(parts))
			return
		}
		hash := parts[0]
		fileName := parts[1]
		committedHashes[fileName] = hash
	}
	if err := scanner.Err(); err != nil {
		t.Errorf("Failed to read hash file: %s", err)
		return
	}

	config, err := ReadConfig("testdata/blorg.org")
	if err != nil {
		t.Errorf("Could not read config: %s", err)
		return
	}
	if err := config.Render(); err != nil {
		t.Errorf("Could not render: %s", err)
		return
	}

	renderedFileHashes := make(map[string]string)
	err = filepath.WalkDir(config.PublicDir, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			return nil
		}
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		hash := md5.Sum(data)
		renderedFileHashes[path] = hex.EncodeToString(hash[:])
		return nil
	})
	if err != nil {
		t.Errorf("Could not determine hashes of rendered files: %s", err)
		return
	}

	for file, rendered := range renderedFileHashes {
		if _, ok := committedHashes[file]; !ok {
			t.Errorf("New file %s does not have a committed hash", file)
			continue
		}
		committed := committedHashes[file]
		committedHashes[file] = "" // To check if there are missing files later.
		if rendered != committed {
			t.Errorf("PublicDir hashes do not match for %s: '%s' -> '%s'", file, committed, rendered)
		}
	}
	for file, committed := range committedHashes {
		if committed != "" {
			t.Errorf("Missing file %s has a committed hash, but was not rendered", file)
		}
	}
}
<|endoftext|>


<commit_before>package srnd

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
)

type varnishHandler struct {
	cache *VarnishCache
}

func (self *varnishHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path
	_, file := filepath.Split(path)

	isjson := strings.HasSuffix(file, "json")

	if strings.HasPrefix(path, "/t/") {
		// thread handler
		hash := strings.Trim(path[3:], "/")
		hash = strings.Replace(hash, "json", "", -1)
		msg, err := self.cache.database.GetMessageIDByHash(hash)
		if err == nil {
			template.genThread(self.cache.attachments, msg, self.cache.prefix, self.cache.name, w, self.cache.database, isjson)
			return
		} else {
			goto notfound
		}
	}
	if strings.Trim(path, "/") == "overboard" {
		// generate ukko aka overboard
		template.genUkko(self.cache.prefix, self.cache.name, w, self.cache.database, isjson)
		return
	}

	if strings.HasPrefix(path, "/b/") {
		// board handler
		parts := strings.Split(path[3:], "/")
		page := 0
		group := parts[0]
		if len(parts) > 1 && parts[1] != "" {
			var err error
			page, err = strconv.Atoi(parts[1])
			if err != nil {
				goto notfound
			}
		}
		hasgroup := self.cache.database.HasNewsgroup(group)
		if !hasgroup {
			goto notfound
		}
		pages := self.cache.database.GetGroupPageCount(group)
		if page >= int(pages) {
			goto notfound
		}
		template.genBoardPage(self.cache.attachments, self.cache.prefix, self.cache.name, group, page, w, self.cache.database, isjson)
		return
	}

	if strings.HasPrefix(path, "/o/") {
		page_str := strings.Trim(path[3:], "/")
		page_str = strings.Replace(page_str, "json", "", -1)
		page := 0
		if page_str != "" {
			var err error
			page, err = strconv.Atoi(page_str)
			if err != nil {
				goto notfound
			}
		}
		template.genUkkoPaginated(self.cache.prefix, self.cache.name, w, self.cache.database, page, isjson)
		return
	}

	if len(file) == 0 || file == "index.html" {
		template.genFrontPage(10, self.cache.prefix, self.cache.name, w, ioutil.Discard, self.cache.database)
		return
	}

	if file == "index.json" {
		// TODO: index.json
		goto notfound
	}
	if strings.HasPrefix(file, "history.html") {
		template.genGraphs(self.cache.prefix, w, self.cache.database)
		return
	}
	if strings.HasPrefix(file, "boards.html") {
		template.genFrontPage(10, self.cache.prefix, self.cache.name, ioutil.Discard, w, self.cache.database)
		return
	}

	if strings.HasPrefix(file, "boards.json") {
		b := self.cache.database.GetAllNewsgroups()
		json.NewEncoder(w).Encode(b)
		return
	}

	if strings.HasPrefix(file, "ukko.html") {
		template.genUkko(self.cache.prefix, self.cache.name, w, self.cache.database, false)
		return
	}
	if strings.HasPrefix(file, "ukko.json") {
		template.genUkko(self.cache.prefix, self.cache.name, w, self.cache.database, true)
		return
	}

	if strings.HasPrefix(file, "ukko-") {
		page := getUkkoPage(file)
		template.genUkkoPaginated(self.cache.prefix, self.cache.name, w, self.cache.database, page, isjson)
		return
	}
	if strings.HasPrefix(file, "thread-") {
		hash := getThreadHash(file)
		if len(hash) == 0 {
			goto notfound
		}
		msg, err := self.cache.database.GetMessageIDByHash(hash)
		if err != nil {
			goto notfound
		}
		template.genThread(self.cache.attachments, msg, self.cache.prefix, self.cache.name, w, self.cache.database, isjson)
		return
	}
	if strings.HasPrefix(file, "catalog-") {
		group := getGroupForCatalog(file)
		if len(group) == 0 {
			goto notfound
		}
		hasgroup := self.cache.database.HasNewsgroup(group)
		if !hasgroup {
			goto notfound
		}
		template.genCatalog(self.cache.prefix, self.cache.name, group, w, self.cache.database)
		return
	} else {
		group, page := getGroupAndPage(file)
		if len(group) == 0 || page < 0 {
			goto notfound
		}
		hasgroup := self.cache.database.HasNewsgroup(group)
		if !hasgroup {
			goto notfound
		}
		pages := self.cache.database.GetGroupPageCount(group)
		if page >= int(pages) {
			goto notfound
		}
		template.genBoardPage(self.cache.attachments, self.cache.prefix, self.cache.name, group, page, w, self.cache.database, isjson)
		return
	}

notfound:
	template.renderNotFound(w, r, self.cache.prefix, self.cache.name)
}

type VarnishCache struct {
	database Database
	store    ArticleStore

	webroot_dir string
	name        string

	regen_threads int
	attachments   bool

	prefix          string
	varnish_url     string
	client          *http.Client
	regenThreadChan chan ArticleEntry
	regenGroupChan  chan groupRegenRequest
}

func (self *VarnishCache) invalidate(r string) {
	u, _ := url.Parse(r)
	resp, err := self.client.Do(&http.Request{
		Method: "PURGE",
		URL:    u,
	})
	if err == nil {
		resp.Body.Close()
	} else {
		log.Println("varnish cache error", err)
	}
}

func (self *VarnishCache) DeleteBoardMarkup(group string) {
	n, _ := self.database.GetPagesPerBoard(group)
	for n > 0 {
		go self.invalidate(fmt.Sprintf("%s%s%s-%d.html", self.varnish_url, self.prefix, group, n))
		go self.invalidate(fmt.Sprintf("%s%sb/%s/%d/", self.varnish_url, self.prefix, group, n))
		n--
	}
	self.invalidate(fmt.Sprintf("%s%sb/%s/", self.varnish_url, self.prefix, group))
}

// try to delete root post's page
func (self *VarnishCache) DeleteThreadMarkup(root_post_id string) {
	self.invalidate(fmt.Sprintf("%s%sthread-%s.html", self.varnish_url, self.prefix, HashMessageID(root_post_id)))
	self.invalidate(fmt.Sprintf("%s%st/%s/", self.varnish_url, self.prefix, HashMessageID(root_post_id)))
}

// regen every newsgroup
func (self *VarnishCache) RegenAll() {
	// we will do this as it's used by rengen on start for frontend
	groups := self.database.GetAllNewsgroups()
	for _, group := range groups {
		self.database.GetGroupThreads(group, self.regenThreadChan)
	}
}

func (self *VarnishCache) RegenFrontPage() {
	self.invalidate(fmt.Sprintf("%s%s", self.varnish_url, self.prefix))
}

func (self *VarnishCache) invalidateUkko() {
	// TODO: invalidate paginated ukko
	self.invalidate(fmt.Sprintf("%s%sukko.html", self.varnish_url, self.prefix))
	self.invalidate(fmt.Sprintf("%s%soverboard/", self.varnish_url, self.prefix))
}

func (self *VarnishCache) pollRegen() {
	for {
		select {
		// consume regen requests
		case ev := <-self.regenGroupChan:
			{
				self.invalidate(fmt.Sprintf("%s%s%s-%d.html", self.varnish_url, self.prefix, ev.group, ev.page))
				self.invalidate(fmt.Sprintf("%s%sb/%s/%d/", self.varnish_url, self.prefix, ev.group, ev.page))
				if ev.page == 0 {
					self.invalidate(fmt.Sprintf("%s%sb/%s/", self.varnish_url, self.prefix, ev.group))
				}
			}
		case ev := <-self.regenThreadChan:
			{
				self.Regen(ev)
			}
		}
	}
}

// regen every page of the board
func (self *VarnishCache) RegenerateBoard(group string) {
	n, _ := self.database.GetPagesPerBoard(group)
	for n > 0 {
		go self.invalidate(fmt.Sprintf("%s%s%s-%d.html", self.varnish_url, self.prefix, group, n))
		go self.invalidate(fmt.Sprintf("%s%s%s/%d/", self.varnish_url, self.prefix, group, n))
		n--
	}
	self.invalidate(fmt.Sprintf("%s%sb/%s/", self.varnish_url, self.prefix, group))
}

// regenerate pages after a mod event
func (self *VarnishCache) RegenOnModEvent(newsgroup, msgid, root string, page int) {
	self.regenGroupChan <- groupRegenRequest{newsgroup, page}
	self.regenThreadChan <- ArticleEntry{newsgroup, root}
}

func (self *VarnishCache) Start() {
	go self.pollRegen()
}

func (self *VarnishCache) Regen(msg ArticleEntry) {
	go self.invalidate(fmt.Sprintf("%s%s%s-%d.html", self.varnish_url, self.prefix, msg.Newsgroup(), 0))
	go self.invalidate(fmt.Sprintf("%s%s%s/%d/", self.varnish_url, self.prefix, msg.Newsgroup(), 0))
	go self.invalidate(fmt.Sprintf("%s%sthread-%s.html", self.varnish_url, self.prefix, HashMessageID(msg.MessageID())))
	go self.invalidate(fmt.Sprintf("%s%st/%s/", self.varnish_url, self.prefix, HashMessageID(msg.MessageID())))
	self.invalidateUkko()
}

func (self *VarnishCache) GetThreadChan() chan ArticleEntry {
	return self.regenThreadChan
}

func (self *VarnishCache) GetGroupChan() chan groupRegenRequest {
	return self.regenGroupChan
}

func (self *VarnishCache) GetHandler() http.Handler {
	return &varnishHandler{self}
}

func (self *VarnishCache) Close() {
	//nothig to do
}

func NewVarnishCache(varnish_url, bind_addr, prefix, webroot, name string, attachments bool, db Database, store ArticleStore) CacheInterface {
	cache := new(VarnishCache)
	cache.regenThreadChan = make(chan ArticleEntry, 16)
	cache.regenGroupChan = make(chan groupRegenRequest, 8)
	local_addr, err := net.ResolveTCPAddr("tcp", bind_addr)
	if err != nil {
		log.Fatalf("failed to resolve %s for varnish cache: %s", bind_addr, err)
	}
	cache.client = &http.Client{
		Transport: &http.Transport{
			Dial: func(network, addr string) (c net.Conn, err error) {
				var remote_addr *net.TCPAddr
				remote_addr, err = net.ResolveTCPAddr(network, addr)
				if err == nil {
					c, err = net.DialTCP(network, local_addr, remote_addr)
				}
				return
			},
		},
	}
	cache.prefix = prefix
	cache.webroot_dir = webroot
	cache.name = name
	cache.attachments = attachments
	cache.database = db
	cache.store = store
	cache.varnish_url = varnish_url
	return cache
}
<commit_msg>fix json<commit_after>package srnd

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
)

type varnishHandler struct {
	cache *VarnishCache
}

func (self *varnishHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path
	_, file := filepath.Split(path)

	isjson := strings.HasSuffix(path, "json")

	if strings.HasPrefix(path, "/t/") {
		// thread handler
		hash := strings.Trim(path[3:], "/")
		hash = strings.Replace(hash, "json", "", -1)
		msg, err := self.cache.database.GetMessageIDByHash(hash)
		if err == nil {
			template.genThread(self.cache.attachments, msg, self.cache.prefix, self.cache.name, w, self.cache.database, isjson)
			return
		} else {
			goto notfound
		}
	}
	if strings.Trim(path, "/") == "overboard" {
		// generate ukko aka overboard
		template.genUkko(self.cache.prefix, self.cache.name, w, self.cache.database, isjson)
		return
	}

	if strings.HasPrefix(path, "/b/") {
		// board handler
		parts := strings.Split(path[3:], "/")
		page := 0
		group := parts[0]
		if len(parts) > 1 && parts[1] != "" {
			var err error
			page, err = strconv.Atoi(parts[1])
			if err != nil {
				goto notfound
			}
		}
		hasgroup := self.cache.database.HasNewsgroup(group)
		if !hasgroup {
			goto notfound
		}
		pages := self.cache.database.GetGroupPageCount(group)
		if page >= int(pages) {
			goto notfound
		}
		template.genBoardPage(self.cache.attachments, self.cache.prefix, self.cache.name, group, page, w, self.cache.database, isjson)
		return
	}

	if strings.HasPrefix(path, "/o/") {
		page_str := strings.Trim(path[3:], "/")
		page_str = strings.Replace(page_str, "json", "", -1)
		page := 0
		if page_str != "" {
			var err error
			page, err = strconv.Atoi(page_str)
			if err != nil {
				goto notfound
			}
		}
		template.genUkkoPaginated(self.cache.prefix, self.cache.name, w, self.cache.database, page, isjson)
		return
	}

	if len(file) == 0 || file == "index.html" {
		template.genFrontPage(10, self.cache.prefix, self.cache.name, w, ioutil.Discard, self.cache.database)
		return
	}

	if file == "index.json" {
		// TODO: index.json
		goto notfound
	}
	if strings.HasPrefix(file, "history.html") {
		template.genGraphs(self.cache.prefix, w, self.cache.database)
		return
	}
	if strings.HasPrefix(file, "boards.html") {
		template.genFrontPage(10, self.cache.prefix, self.cache.name, ioutil.Discard, w, self.cache.database)
		return
	}

	if strings.HasPrefix(file, "boards.json") {
		b := self.cache.database.GetAllNewsgroups()
		json.NewEncoder(w).Encode(b)
		return
	}

	if strings.HasPrefix(file, "ukko.html") {
		template.genUkko(self.cache.prefix, self.cache.name, w, self.cache.database, false)
		return
	}
	if strings.HasPrefix(file, "ukko.json") {
		template.genUkko(self.cache.prefix, self.cache.name, w, self.cache.database, true)
		return
	}

	if strings.HasPrefix(file, "ukko-") {
		page := getUkkoPage(file)
		template.genUkkoPaginated(self.cache.prefix, self.cache.name, w, self.cache.database, page, isjson)
		return
	}
	if strings.HasPrefix(file, "thread-") {
		hash := getThreadHash(file)
		if len(hash) == 0 {
			goto notfound
		}
		msg, err := self.cache.database.GetMessageIDByHash(hash)
		if err != nil {
			goto notfound
		}
		template.genThread(self.cache.attachments, msg, self.cache.prefix, self.cache.name, w, self.cache.database, isjson)
		return
	}
	if strings.HasPrefix(file, "catalog-") {
		group := getGroupForCatalog(file)
		if len(group) == 0 {
			goto notfound
		}
		hasgroup := self.cache.database.HasNewsgroup(group)
		if !hasgroup {
			goto notfound
		}
		template.genCatalog(self.cache.prefix, self.cache.name, group, w, self.cache.database)
		return
	} else {
		group, page := getGroupAndPage(file)
		if len(group) == 0 || page < 0 {
			goto notfound
		}
		hasgroup := self.cache.database.HasNewsgroup(group)
		if !hasgroup {
			goto notfound
		}
		pages := self.cache.database.GetGroupPageCount(group)
		if page >= int(pages) {
			goto notfound
		}
		template.genBoardPage(self.cache.attachments, self.cache.prefix, self.cache.name, group, page, w, self.cache.database, isjson)
		return
	}

notfound:
	template.renderNotFound(w, r, self.cache.prefix, self.cache.name)
}

type VarnishCache struct {
	database Database
	store    ArticleStore

	webroot_dir string
	name        string

	regen_threads int
	attachments   bool

	prefix          string
	varnish_url     string
	client          *http.Client
	regenThreadChan chan ArticleEntry
	regenGroupChan  chan groupRegenRequest
}

func (self *VarnishCache) invalidate(r string) {
	u, _ := url.Parse(r)
	resp, err := self.client.Do(&http.Request{
		Method: "PURGE",
		URL:    u,
	})
	if err == nil {
		resp.Body.Close()
	} else {
		log.Println("varnish cache error", err)
	}
}

func (self *VarnishCache) DeleteBoardMarkup(group string) {
	n, _ := self.database.GetPagesPerBoard(group)
	for n > 0 {
		go self.invalidate(fmt.Sprintf("%s%s%s-%d.html", self.varnish_url, self.prefix, group, n))
		go self.invalidate(fmt.Sprintf("%s%sb/%s/%d/", self.varnish_url, self.prefix, group, n))
		n--
	}
	self.invalidate(fmt.Sprintf("%s%sb/%s/", self.varnish_url, self.prefix, group))
}

// try to delete root post's page
func (self *VarnishCache) DeleteThreadMarkup(root_post_id string) {
	self.invalidate(fmt.Sprintf("%s%sthread-%s.html", self.varnish_url, self.prefix, HashMessageID(root_post_id)))
	self.invalidate(fmt.Sprintf("%s%st/%s/", self.varnish_url, self.prefix, HashMessageID(root_post_id)))
}

// regen every newsgroup
func (self *VarnishCache) RegenAll() {
	// we will do this as it's used by rengen on start for frontend
	groups := self.database.GetAllNewsgroups()
	for _, group := range groups {
		self.database.GetGroupThreads(group, self.regenThreadChan)
	}
}

func (self *VarnishCache) RegenFrontPage() {
	self.invalidate(fmt.Sprintf("%s%s", self.varnish_url, self.prefix))
}

func (self *VarnishCache) invalidateUkko() {
	// TODO: invalidate paginated ukko
	self.invalidate(fmt.Sprintf("%s%sukko.html", self.varnish_url, self.prefix))
	self.invalidate(fmt.Sprintf("%s%soverboard/", self.varnish_url, self.prefix))
}

func (self *VarnishCache) pollRegen() {
	for {
		select {
		// consume regen requests
		case ev := <-self.regenGroupChan:
			{
				self.invalidate(fmt.Sprintf("%s%s%s-%d.html", self.varnish_url, self.prefix, ev.group, ev.page))
				self.invalidate(fmt.Sprintf("%s%sb/%s/%d/", self.varnish_url, self.prefix, ev.group, ev.page))
				if ev.page == 0 {
					self.invalidate(fmt.Sprintf("%s%sb/%s/", self.varnish_url, self.prefix, ev.group))
				}
			}
		case ev := <-self.regenThreadChan:
			{
				self.Regen(ev)
			}
		}
	}
}

// regen every page of the board
func (self *VarnishCache) RegenerateBoard(group string) {
	n, _ := self.database.GetPagesPerBoard(group)
	for n > 0 {
		go self.invalidate(fmt.Sprintf("%s%s%s-%d.html", self.varnish_url, self.prefix, group, n))
		go self.invalidate(fmt.Sprintf("%s%s%s/%d/", self.varnish_url, self.prefix, group, n))
		n--
	}
	self.invalidate(fmt.Sprintf("%s%sb/%s/", self.varnish_url, self.prefix, group))
}

// regenerate pages after a mod event
func (self *VarnishCache) RegenOnModEvent(newsgroup, msgid, root string, page int) {
	self.regenGroupChan <- groupRegenRequest{newsgroup, page}
	self.regenThreadChan <- ArticleEntry{newsgroup, root}
}

func (self *VarnishCache) Start() {
	go self.pollRegen()
}

func (self *VarnishCache) Regen(msg ArticleEntry) {
	go self.invalidate(fmt.Sprintf("%s%s%s-%d.html", self.varnish_url, self.prefix, msg.Newsgroup(), 0))
	go self.invalidate(fmt.Sprintf("%s%s%s/%d/", self.varnish_url, self.prefix, msg.Newsgroup(), 0))
	go self.invalidate(fmt.Sprintf("%s%sthread-%s.html", self.varnish_url, self.prefix, HashMessageID(msg.MessageID())))
	go self.invalidate(fmt.Sprintf("%s%st/%s/", self.varnish_url, self.prefix, HashMessageID(msg.MessageID())))
	self.invalidateUkko()
}

func (self *VarnishCache) GetThreadChan() chan ArticleEntry {
	return self.regenThreadChan
}

func (self *VarnishCache) GetGroupChan() chan groupRegenRequest {
	return self.regenGroupChan
}

func (self *VarnishCache) GetHandler() http.Handler {
	return &varnishHandler{self}
}

func (self *VarnishCache) Close() {
	//nothig to do
}

func NewVarnishCache(varnish_url, bind_addr, prefix, webroot, name string, attachments bool, db Database, store ArticleStore) CacheInterface {
	cache := new(VarnishCache)
	cache.regenThreadChan = make(chan ArticleEntry, 16)
	cache.regenGroupChan = make(chan groupRegenRequest, 8)
	local_addr, err := net.ResolveTCPAddr("tcp", bind_addr)
	if err != nil {
		log.Fatalf("failed to resolve %s for varnish cache: %s", bind_addr, err)
	}
	cache.client = &http.Client{
		Transport: &http.Transport{
			Dial: func(network, addr string) (c net.Conn, err error) {
				var remote_addr *net.TCPAddr
				remote_addr, err = net.ResolveTCPAddr(network, addr)
				if err == nil {
					c, err = net.DialTCP(network, local_addr, remote_addr)
				}
				return
			},
		},
	}
	cache.prefix = prefix
	cache.webroot_dir = webroot
	cache.name = name
	cache.attachments = attachments
	cache.database = db
	cache.store = store
	cache.varnish_url = varnish_url
	return cache
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"strings"
	"time"

	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
)

func nfdString(s string) string {
	result, _, _ := transform.String(norm.NFD, s)
	return result
}

func consumeNumSep(s string) (string, int) {
	consumed := 0
	for len(s) > 0 {
		c := s[0]
		if c == ' ' || c == '-' || c == '/' || c == ',' {
			consumed += 1
			s = s[1:]
			continue
		}
		if strings.HasPrefix(s, "ou") || strings.HasPrefix(s, "et") {
			consumed += 2
			s = s[2:]
			continue
		}
		break
	}
	return s, consumed
}

func isNum(c byte) bool {
	return c >= '0' && c <= '9'
}

// fixCountryNums split inputs like "23/45 - 52 ou 92" into country numbers.
func fixCountryNums(s string) []string {
	found := []string{}
	input, _ := consumeNumSep(s)
	var consumed int
	for {
		if len(input) >= 2 && isNum(input[0]) && isNum(input[1]) {
			found = append(found, input[:2])
			input = input[2:]
		} else if len(input) >= 1 && isNum(input[0]) {
			found = append(found, input[:1])
			input = input[1:]
		} else {
			break
		}
		input, consumed = consumeNumSep(input)
		if consumed <= 0 && input != "" {
			break
		}
		if input == "" {
			return found
		}
	}
	return []string{s}
}

var (
	locPrefixes = []string{
		nfdString("proche de"),
		nfdString("proche"),
		nfdString("dpts"),
		nfdString("dpt"),
		nfdString("départem."),
		nfdString("départements"),
		nfdString("agglo."),
		nfdString("agglo"),
		nfdString("agence de"),
		nfdString("agence"),
		nfdString("basé"),
		nfdString("régions"),
		nfdString("région"),
	}
)

func stripPrefixes(s string) []string {
	orig := nfdString(s)
	stripped := orig
	for _, p := range locPrefixes {
		if strings.HasPrefix(stripped, p) {
			stripped = strings.TrimSpace(stripped[len(p):])
		}
	}
	if stripped != orig {
		s = stripped
	}
	return []string{s}
}

func splitAlternatives(s string) []string {
	return strings.Split(s, " ou ")
}

func fixWellKnown(s string) []string {
	if s == "idf" {
		return []string{"ile-de-france"}
	}
	if strings.Contains(s, "boulogne b") {
		return []string{"boulogne billancourt"}
	}
	if strings.Contains(s, "velizy") {
		return []string{"velizy"}
	}
	return []string{s}
}

func apply(input []string, fn func(string) []string) []string {
	output := []string{}
	for _, s := range input {
		res := fn(s)
		for _, r := range res {
			r = strings.TrimSpace(r)
			if r != "" {
				output = append(output, r)
			}
		}
	}
	return output
}

func nfcString(s string) []string {
	result, _, _ := transform.String(norm.NFC, s)
	return []string{result}
}

func fixLocation(s string) []string {
	result := []string{strings.TrimSpace(strings.ToLower(s))}
	result = apply(result, splitAlternatives)
	result = apply(result, stripPrefixes)
	result = apply(result, fixWellKnown)
	result = apply(result, fixCountryNums)
	result = apply(result, nfcString)
	return result
}

// getOfferLocation returns a cached or live geocoded location, an updated
// "offline" boolean signaling whether live calls could proceed or not, and an
// error on failure.
func geocodeOffer(geocoder *Geocoder, location string, offline bool,
	minQuota int) (*Location, bool, bool, error) {

	candidates := fixLocation(location)
	for _, c := range candidates {
		// Resolve from cache
		pos, ok, err := geocoder.GetCachedLocation(c, "fr")
		if err != nil {
			return nil, false, offline, err
		}
		if pos != nil || ok {
			return pos, false, offline, nil
		}
		if offline {
			// Tolerate a lower quality geocoding for now
			continue
		}
		loc, err := geocoder.Geocode(c, "fr", false)
		if err != nil {
			if err != QuotaError {
				return nil, false, offline, err
			}
			offline = true
			continue
		}
		if loc.Rate.Remaining <= minQuota {
			// Try to preserve quota for test purpose. This is not
			// perfect as it consumes one geocoding token per function
			// call. I do not know how to query quota directly yet.
			offline = true
		}
		p := buildLocation(loc)
		result := "no result"
		if p != nil {
			result = loc.Results[0].Component.String()
		}
		log.Printf("geocoding %s => %s => %s (quota: %d/%d)\n",
			location, c, result, loc.Rate.Remaining, loc.Rate.Limit)
		time.Sleep(1 * time.Second)
		if p != nil {
			return p, true, offline, nil
		}
	}
	return nil, false, offline, nil
}

func geocodeOffers(store *Store, geocoder *Geocoder, offers []*Offer,
	minQuota int) (int, error) {

	rejected := 0
	offline := false
	for _, offer := range offers {
		pos, _, off, err := geocodeOffer(geocoder, offer.Location,
			offline, minQuota)
		if err != nil {
			return rejected, err
		}
		offline = off
		if !offline {
			err = store.PutLocation(offer.Id, pos, offer.Date)
			if err != nil {
				return rejected, err
			}
		}
		if pos == nil {
			rejected++
		}
	}
	return rejected, nil
}
<commit_msg>location: add special cases for common geocoding failures<commit_after>package main

import (
	"log"
	"strings"
	"time"

	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
)

func nfdString(s string) string {
	result, _, _ := transform.String(norm.NFD, s)
	return result
}

func consumeNumSep(s string) (string, int) {
	consumed := 0
	for len(s) > 0 {
		c := s[0]
		if c == ' ' || c == '-' || c == '/' || c == ',' {
			consumed += 1
			s = s[1:]
			continue
		}
		if strings.HasPrefix(s, "ou") || strings.HasPrefix(s, "et") {
			consumed += 2
			s = s[2:]
			continue
		}
		break
	}
	return s, consumed
}

func isNum(c byte) bool {
	return c >= '0' && c <= '9'
}

// fixCountryNums split inputs like "23/45 - 52 ou 92" into country numbers.
func fixCountryNums(s string) []string {
	found := []string{}
	input, _ := consumeNumSep(s)
	var consumed int
	for {
		if len(input) >= 2 && isNum(input[0]) && isNum(input[1]) {
			found = append(found, input[:2])
			input = input[2:]
		} else if len(input) >= 1 && isNum(input[0]) {
			found = append(found, input[:1])
			input = input[1:]
		} else {
			break
		}
		input, consumed = consumeNumSep(input)
		if consumed <= 0 && input != "" {
			break
		}
		if input == "" {
			return found
		}
	}
	return []string{s}
}

var (
	locPrefixes = []string{
		nfdString("proche de"),
		nfdString("proche"),
		nfdString("dpts"),
		nfdString("dpt"),
		nfdString("départem."),
		nfdString("départements"),
		nfdString("agglo."),
		nfdString("agglo"),
		nfdString("agence de"),
		nfdString("agence"),
		nfdString("basé"),
		nfdString("régions"),
		nfdString("région"),
	}
)

func stripPrefixes(s string) []string {
	orig := nfdString(s)
	stripped := orig
	for _, p := range locPrefixes {
		if strings.HasPrefix(stripped, p) {
			stripped = strings.TrimSpace(stripped[len(p):])
		}
	}
	if stripped != orig {
		s = stripped
	}
	return []string{s}
}

func splitAlternatives(s string) []string {
	return strings.Split(s, " ou ")
}

var (
	wellKnown = map[string]string{
		"st quentin en yvel": "saint-quentin-en-yvelines",
		"montigny le breton": "montigny-le-bretonneux",
		"75000":              "paris",
		"boulogne-billancou": "boulogne-billancourt",
	}
)

func fixWellKnown(s string) []string {
	if r, ok := wellKnown[s]; ok {
		return []string{r}
	}
	if s == "idf" {
		return []string{"ile-de-france"}
	}
	if strings.Contains(s, "boulogne b") {
		return []string{"boulogne billancourt"}
	}
	if strings.Contains(s, "velizy") || strings.Contains(s, "vélizy") {
		return []string{"velizy"}
	}
	return []string{s}
}

func apply(input []string, fn func(string) []string) []string {
	output := []string{}
	for _, s := range input {
		res := fn(s)
		for _, r := range res {
			r = strings.TrimSpace(r)
			if r != "" {
				output = append(output, r)
			}
		}
	}
	return output
}

func nfcString(s string) []string {
	result, _, _ := transform.String(norm.NFC, s)
	return []string{result}
}

func fixLocation(s string) []string {
	result := []string{strings.TrimSpace(strings.ToLower(s))}
	result = apply(result, splitAlternatives)
	result = apply(result, stripPrefixes)
	result = apply(result, fixWellKnown)
	result = apply(result, fixCountryNums)
	result = apply(result, nfcString)
	return result
}

// getOfferLocation returns a cached or live geocoded location, an updated
// "offline" boolean signaling whether live calls could proceed or not, and an
// error on failure.
func geocodeOffer(geocoder *Geocoder, location string, offline bool,
	minQuota int) (*Location, bool, bool, error) {

	candidates := fixLocation(location)
	for _, c := range candidates {
		// Resolve from cache
		pos, ok, err := geocoder.GetCachedLocation(c, "fr")
		if err != nil {
			return nil, false, offline, err
		}
		if pos != nil || ok {
			return pos, false, offline, nil
		}
		if offline {
			// Tolerate a lower quality geocoding for now
			continue
		}
		loc, err := geocoder.Geocode(c, "fr", false)
		if err != nil {
			if err != QuotaError {
				return nil, false, offline, err
			}
			offline = true
			continue
		}
		if loc.Rate.Remaining <= minQuota {
			// Try to preserve quota for test purpose. This is not
			// perfect as it consumes one geocoding token per function
			// call. I do not know how to query quota directly yet.
			offline = true
		}
		p := buildLocation(loc)
		result := "no result"
		if p != nil {
			result = loc.Results[0].Component.String()
		}
		log.Printf("geocoding %s => %s => %s (quota: %d/%d)\n",
			location, c, result, loc.Rate.Remaining, loc.Rate.Limit)
		time.Sleep(1 * time.Second)
		if p != nil {
			return p, true, offline, nil
		}
	}
	return nil, false, offline, nil
}

func geocodeOffers(store *Store, geocoder *Geocoder, offers []*Offer,
	minQuota int) (int, error) {

	rejected := 0
	offline := false
	for _, offer := range offers {
		pos, _, off, err := geocodeOffer(geocoder, offer.Location,
			offline, minQuota)
		if err != nil {
			return rejected, err
		}
		offline = off
		if !offline {
			err = store.PutLocation(offer.Id, pos, offer.Date)
			if err != nil {
				return rejected, err
			}
		}
		if pos == nil {
			rejected++
		}
	}
	return rejected, nil
}
<|endoftext|>


<commit_before>package log

func SetV(v int) {
	if v < 0 {
		v = 0
	}
	logging.verbosity.setInt(v)
}

func SetVModule(vmod string) error {
	return logging.vmodule.Set(vmod)
}

func UseStderr(use bool) {
	logging.toStderr = true
}

func DevelDefaults() {
	UseStderr(true)
	SetV(5)
}

func ProdDefaults() {
	UseStderr(false)
	SetV(2)
}
<commit_msg>reduce prod logging<commit_after>package log

func SetV(v int) {
	if v < 0 {
		v = 0
	}
	logging.verbosity.setInt(v)
}

func SetVModule(vmod string) error {
	return logging.vmodule.Set(vmod)
}

func UseStderr(use bool) {
	logging.toStderr = true
}

func DevelDefaults() {
	UseStderr(true)
	SetV(5)
}

func ProdDefaults() {
	UseStderr(false)
	SetV(1)
}
<|endoftext|>


<commit_before>// Copyright 2012 The Oneslang Authors.  All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package log

import (
	"bytes"
	"os"
	"regexp"
	"testing"
)

const (
	Rdate         = `[0-9][0-9][0-9][0-9]/[0-9][0-9]/[0-9][0-9]`
	Rtime         = `[0-9][0-9]:[0-9][0-9]:[0-9][0-9]`
	Rmicroseconds = `\.[0-9][0-9][0-9][0-9][0-9][0-9]`
	Rline         = `(48|56):` // must update if the calls to l.Printf / l.Print below move
	Rlongfile     = `.*/[A-Za-z0-9_\-]+\.go:` + Rline
	Rshortfile    = `[A-Za-z0-9_\-]+\.go:` + Rline
	Rinfo         = `INFO `
)

type tester struct {
	flag    int
	prefix  string
	pattern string // regexp that log output must match; we add ^ and expected_text$ always
}

var tests = []tester{
	// individual pieces:
	{0, "", ""},
	{0, "XXX", "XXX"},
	{Ldate, "", Rdate + " "},
	{Ltime, "", Rtime + " "},
	{Ltime | Lmicroseconds, "", Rtime + Rmicroseconds + " "},
	{Lmicroseconds, "", Rtime + Rmicroseconds + " "}, // microsec implies time
	{Llongfile, "", Rlongfile + " "},
	{Lshortfile, "", Rshortfile + " "},
	{Llongfile | Lshortfile, "", Rshortfile + " "}, // shortfile overrides longfile
	{Lpriority, "", Rinfo},
	{Lpriority | Ltime, "", Rinfo + Rtime + " "},
	// everything at once:
	{Ldate | Ltime | Lmicroseconds | Llongfile, "XXX", "XXX" + Rdate + " " + Rtime + Rmicroseconds + " " + Rlongfile + " "},
	{Ldate | Ltime | Lmicroseconds | Lshortfile, "XXX", "XXX" + Rdate + " " + Rtime + Rmicroseconds + " " + Rshortfile + " "},
}

// Test using Println("hello", 23, "world") or using Printf("hello %d world", 23)
func testPrint(t *testing.T, flag int, prefix string, pattern string, useFormat bool) {
	buf := new(bytes.Buffer)
	SetOutput(buf)
	SetFlags(flag)
	SetPrefix(prefix)
	SetPriority(Pall)
	if useFormat {
		Infof("hello %d world", 23)
	} else {
		Infoln("hello", 23, "world")
	}
	line := buf.String()
	line = line[0 : len(line)-1]
	pattern = "^" + pattern + "hello 23 world$"
	matched, err4 := regexp.MatchString(pattern, line)
	if err4 != nil {
		t.Fatal("pattern did not compile:", err4)
	}
	if !matched {
		t.Errorf("log output should match %q but is %q", pattern, line)
	}
	SetOutput(os.Stderr)
}

func TestAll(t *testing.T) {
	for _, testcase := range tests {
		testPrint(t, testcase.flag, testcase.prefix, testcase.pattern, false)
		testPrint(t, testcase.flag, testcase.prefix, testcase.pattern, true)
	}
}

// XXX can't run this test because the program dies. How to test fatal?
//func TestFatal(t *testing.T) {
//	SetPriority(log.Pinfo)
//	SetLayouts(log.Lstd | log.Lpriority | log.Llongfile)
//	Fatal("-----death------")
//}
<commit_msg>change err4 to err<commit_after>// Copyright 2012 The Oneslang Authors.  All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package log

import (
	"bytes"
	"os"
	"regexp"
	"testing"
)

const (
	Rdate         = `[0-9][0-9][0-9][0-9]/[0-9][0-9]/[0-9][0-9]`
	Rtime         = `[0-9][0-9]:[0-9][0-9]:[0-9][0-9]`
	Rmicroseconds = `\.[0-9][0-9][0-9][0-9][0-9][0-9]`
	Rline         = `(48|56):` // must update if the calls to l.Printf / l.Print below move
	Rlongfile     = `.*/[A-Za-z0-9_\-]+\.go:` + Rline
	Rshortfile    = `[A-Za-z0-9_\-]+\.go:` + Rline
	Rinfo         = `INFO `
)

type tester struct {
	flag    int
	prefix  string
	pattern string // regexp that log output must match; we add ^ and expected_text$ always
}

var tests = []tester{
	// individual pieces:
	{0, "", ""},
	{0, "XXX", "XXX"},
	{Ldate, "", Rdate + " "},
	{Ltime, "", Rtime + " "},
	{Ltime | Lmicroseconds, "", Rtime + Rmicroseconds + " "},
	{Lmicroseconds, "", Rtime + Rmicroseconds + " "}, // microsec implies time
	{Llongfile, "", Rlongfile + " "},
	{Lshortfile, "", Rshortfile + " "},
	{Llongfile | Lshortfile, "", Rshortfile + " "}, // shortfile overrides longfile
	{Lpriority, "", Rinfo},
	{Lpriority | Ltime, "", Rinfo + Rtime + " "},
	// everything at once:
	{Ldate | Ltime | Lmicroseconds | Llongfile, "XXX", "XXX" + Rdate + " " + Rtime + Rmicroseconds + " " + Rlongfile + " "},
	{Ldate | Ltime | Lmicroseconds | Lshortfile, "XXX", "XXX" + Rdate + " " + Rtime + Rmicroseconds + " " + Rshortfile + " "},
}

// Test using Println("hello", 23, "world") or using Printf("hello %d world", 23)
func testPrint(t *testing.T, flag int, prefix string, pattern string, useFormat bool) {
	buf := new(bytes.Buffer)
	SetOutput(buf)
	SetFlags(flag)
	SetPrefix(prefix)
	SetPriority(Pall)
	if useFormat {
		Infof("hello %d world", 23)
	} else {
		Infoln("hello", 23, "world")
	}
	line := buf.String()
	line = line[0 : len(line)-1]
	pattern = "^" + pattern + "hello 23 world$"
	matched, err := regexp.MatchString(pattern, line)
	if err != nil {
		t.Fatal("pattern did not compile:", err)
	}
	if !matched {
		t.Errorf("log output should match %q but is %q", pattern, line)
	}
	SetOutput(os.Stderr)
}

func TestAll(t *testing.T) {
	for _, testcase := range tests {
		testPrint(t, testcase.flag, testcase.prefix, testcase.pattern, false)
		testPrint(t, testcase.flag, testcase.prefix, testcase.pattern, true)
	}
}

func TestPriority(t *testing.T) {
	buf := new(bytes.Buffer)
	SetOutput(buf)
	SetFlags(0)
	SetPriority(Pinfo)
	Warn("a")
	Debug("b")
	if buf.String() != "a\n" {
		t.Fatal("expected a\\n, got %#v", buf)
	}
}

// XXX can't run this test because the program dies. How to test fatal?
//func TestFatal(t *testing.T) {
//	SetPriority(log.Pinfo)
//	SetLayouts(log.Lstd | log.Lpriority | log.Llongfile)
//	Fatal("-----death------")
//}
<|endoftext|>


<commit_before>// A client implementation that includes concurrency and dropping.
package logplexc

import (
	"errors"
	"net/http"
	"net/url"
	"sync"
	"sync/atomic"
	"time"
)

type Stats struct {
	// Number of concurrent requests at the time of retrieval.
	Concurrency int32

	// Message-level statistics

	// Total messages submitted
	Total uint64

	// Incremented when a message is ignored outright because of
	// too much work being done already.
	Dropped uint64

	// Incremented when a log post request is not known to have
	// succeeded and one has given up waiting.
	Cancelled uint64

	// Incremented when a log post request is responded to,
	// affirming that the messages have been rejected.
	Rejected uint64

	// Incremented only when a positive response is received from
	// logplex.
	Successful uint64

	// Request-level statistics

	TotalRequests   uint64
	DroppedRequests uint64
	CancelRequests  uint64
	RejectRequests  uint64
	SuccessRequests uint64
}

type Client struct {
	Stats
	statLock sync.Mutex

	c *MiniClient

	// Concurrency control of POST workers: the current level of
	// concurrency, and a token bucket channel.
	concurrency int32
	bucket      chan bool

	// Threshold of logplex request size to trigger POST.
	RequestSizeTrigger int

	// For forcing periodic posting of low-activity logs.
	ticker *time.Ticker

	// Closed when cleaning up
	finalize chan bool
}

type Config struct {
	Logplex            url.URL
	Token              string
	HttpClient         http.Client
	RequestSizeTrigger int
	Concurrency        int
	TargetLogLatency   time.Duration
}

func NewClient(cfg *Config) (*Client, error) {
	c, err := NewMiniClient(
		&MiniConfig{
			Logplex:    cfg.Logplex,
			Token:      cfg.Token,
			HttpClient: cfg.HttpClient,
		})

	if err != nil {
		return nil, err
	}

	if cfg.TargetLogLatency < 0 {
		return nil, errors.New("logplexc.Client: negative target " +
			"latency not allowed")
	}

	m := Client{
		c:                  c,
		finalize:           make(chan bool),
		bucket:             make(chan bool),
		RequestSizeTrigger: cfg.RequestSizeTrigger,
	}

	// If duration is zero, don't bother starting the ticker; a
	// special code path for a nil ticker will send immediately.
	if cfg.TargetLogLatency > 0 {
		m.ticker = time.NewTicker(cfg.TargetLogLatency)
	}

	// Supply tokens to the buckets.
	//
	// This goroutine exits when it has supplied all of the
	// initial tokens: that's because worker goroutines are
	// responsible for re-inserting tokens.
	go func() {
		for i := 0; i < cfg.Concurrency; i += 1 {
			m.bucket <- true
		}
	}()

	// Periodic log-sending ticker for responsive low-volume
	// logging.
	if m.ticker != nil {
		go func() {
			for {
				// Wait for a while to do work, or to
				// exit
				select {
				case <-m.ticker.C:
				case _, _ = <-m.finalize:
					return
				}

				// Avoid sending empty requests
				s := m.c.Statistics()
				if s.NumberFramed > 0 {
					go m.syncWorker()
				}
			}
		}()
	}

	return &m, nil
}

func (m *Client) Close() {
	// Clean up otherwise immortal ticker goroutine
	m.ticker.Stop()
	close(m.finalize)
}

func (m *Client) BufferMessage(
	when time.Time, procId string, log []byte) error {

	select {
	case _, _ = <-m.finalize:
		return errors.New("Failed trying to buffer a message: " +
			"client already Closed")
	default:
		// no-op
	}

	m.statLock.Lock()
	defer m.statLock.Unlock()

	s := m.c.BufferMessage(when, procId, log)
	if s.Buffered >= m.RequestSizeTrigger || m.ticker == nil {
		go m.syncWorker()
	}

	return nil
}

func (m *Client) Statistics() (s Stats) {
	m.statLock.Lock()
	defer m.statLock.Unlock()

	s = m.Stats
	return s
}

func (m *Client) syncWorker() {
	atomic.AddInt32(&m.Stats.Concurrency, 1)
	defer atomic.AddInt32(&m.Stats.Concurrency, -1)

	b := m.c.SwapBundle()

	// Check if there are any worker tokens available. If not,
	// then just abort after recording drop statistics.
	select {
	case <-m.bucket:
		// When exiting, free up the token for use by another
		// worker.
		defer func() {
			m.bucket <- true
		}()
	default:
		m.statReqDrop(&b.MiniStats)
		return
	}

	// Post to logplex.
	resp, err := m.c.Post(&b)
	if err != nil {
		m.statReqErr(&b.MiniStats)
	}

	defer resp.Body.Close()

	// Check HTTP return code and accrue statistics accordingly.
	if resp.StatusCode != http.StatusNoContent {
		m.statReqRej(&b.MiniStats)
	} else {
		m.statReqSuccess(&b.MiniStats)
	}

	return
}

func (m *Client) statReqTotalUnsync(s *MiniStats) {
	m.Total += s.NumberFramed
	m.TotalRequests += 1
}

func (m *Client) statReqSuccess(s *MiniStats) {
	m.statLock.Lock()
	defer m.statLock.Unlock()
	m.statReqTotalUnsync(s)

	m.Successful += s.NumberFramed
	m.SuccessRequests += 1
}

func (m *Client) statReqErr(s *MiniStats) {
	m.statLock.Lock()
	defer m.statLock.Unlock()
	m.statReqTotalUnsync(s)

	m.Cancelled += s.NumberFramed
	m.CancelRequests += 1
}

func (m *Client) statReqRej(s *MiniStats) {
	m.statLock.Lock()
	defer m.statLock.Unlock()
	m.statReqTotalUnsync(s)

	m.Rejected += s.NumberFramed
	m.RejectRequests += 1
}

func (m *Client) statReqDrop(s *MiniStats) {
	m.statLock.Lock()
	defer m.statLock.Unlock()
	m.statReqTotalUnsync(s)

	m.Dropped += s.NumberFramed
	m.DroppedRequests += 1
}
<commit_msg>Move empty-request-prevention guard<commit_after>// A client implementation that includes concurrency and dropping.
package logplexc

import (
	"errors"
	"net/http"
	"net/url"
	"sync"
	"sync/atomic"
	"time"
)

type Stats struct {
	// Number of concurrent requests at the time of retrieval.
	Concurrency int32

	// Message-level statistics

	// Total messages submitted
	Total uint64

	// Incremented when a message is ignored outright because of
	// too much work being done already.
	Dropped uint64

	// Incremented when a log post request is not known to have
	// succeeded and one has given up waiting.
	Cancelled uint64

	// Incremented when a log post request is responded to,
	// affirming that the messages have been rejected.
	Rejected uint64

	// Incremented only when a positive response is received from
	// logplex.
	Successful uint64

	// Request-level statistics

	TotalRequests   uint64
	DroppedRequests uint64
	CancelRequests  uint64
	RejectRequests  uint64
	SuccessRequests uint64
}

type Client struct {
	Stats
	statLock sync.Mutex

	c *MiniClient

	// Concurrency control of POST workers: the current level of
	// concurrency, and a token bucket channel.
	concurrency int32
	bucket      chan bool

	// Threshold of logplex request size to trigger POST.
	RequestSizeTrigger int

	// For forcing periodic posting of low-activity logs.
	ticker *time.Ticker

	// Closed when cleaning up
	finalize chan bool
}

type Config struct {
	Logplex            url.URL
	Token              string
	HttpClient         http.Client
	RequestSizeTrigger int
	Concurrency        int
	TargetLogLatency   time.Duration
}

func NewClient(cfg *Config) (*Client, error) {
	c, err := NewMiniClient(
		&MiniConfig{
			Logplex:    cfg.Logplex,
			Token:      cfg.Token,
			HttpClient: cfg.HttpClient,
		})

	if err != nil {
		return nil, err
	}

	if cfg.TargetLogLatency < 0 {
		return nil, errors.New("logplexc.Client: negative target " +
			"latency not allowed")
	}

	m := Client{
		c:                  c,
		finalize:           make(chan bool),
		bucket:             make(chan bool),
		RequestSizeTrigger: cfg.RequestSizeTrigger,
	}

	// If duration is zero, don't bother starting the ticker; a
	// special code path for a nil ticker will send immediately.
	if cfg.TargetLogLatency > 0 {
		m.ticker = time.NewTicker(cfg.TargetLogLatency)
	}

	// Supply tokens to the buckets.
	//
	// This goroutine exits when it has supplied all of the
	// initial tokens: that's because worker goroutines are
	// responsible for re-inserting tokens.
	go func() {
		for i := 0; i < cfg.Concurrency; i += 1 {
			m.bucket <- true
		}
	}()

	// Periodic log-sending ticker for responsive low-volume
	// logging.
	if m.ticker != nil {
		go func() {
			for {
				// Wait for a while to do work, or to
				// exit
				select {
				case <-m.ticker.C:
				case _, _ = <-m.finalize:
					return
				}

				go m.syncWorker()
			}
		}()
	}

	return &m, nil
}

func (m *Client) Close() {
	// Clean up otherwise immortal ticker goroutine
	m.ticker.Stop()
	close(m.finalize)
}

func (m *Client) BufferMessage(
	when time.Time, procId string, log []byte) error {

	select {
	case _, _ = <-m.finalize:
		return errors.New("Failed trying to buffer a message: " +
			"client already Closed")
	default:
		// no-op
	}

	m.statLock.Lock()
	defer m.statLock.Unlock()

	s := m.c.BufferMessage(when, procId, log)
	if s.Buffered >= m.RequestSizeTrigger || m.ticker == nil {
		go m.syncWorker()
	}

	return nil
}

func (m *Client) Statistics() (s Stats) {
	m.statLock.Lock()
	defer m.statLock.Unlock()

	s = m.Stats
	return s
}

func (m *Client) syncWorker() {
	atomic.AddInt32(&m.Stats.Concurrency, 1)
	defer atomic.AddInt32(&m.Stats.Concurrency, -1)

	b := m.c.SwapBundle()

	// Avoid sending empty requests
	if b.NumberFramed <= 0 {
		return
	}

	// Check if there are any worker tokens available. If not,
	// then just abort after recording drop statistics.
	select {
	case <-m.bucket:
		// When exiting, free up the token for use by another
		// worker.
		defer func() {
			m.bucket <- true
		}()
	default:
		m.statReqDrop(&b.MiniStats)
		return
	}

	// Post to logplex.
	resp, err := m.c.Post(&b)
	if err != nil {
		m.statReqErr(&b.MiniStats)
	}

	defer resp.Body.Close()

	// Check HTTP return code and accrue statistics accordingly.
	if resp.StatusCode != http.StatusNoContent {
		m.statReqRej(&b.MiniStats)
	} else {
		m.statReqSuccess(&b.MiniStats)
	}

	return
}

func (m *Client) statReqTotalUnsync(s *MiniStats) {
	m.Total += s.NumberFramed
	m.TotalRequests += 1
}

func (m *Client) statReqSuccess(s *MiniStats) {
	m.statLock.Lock()
	defer m.statLock.Unlock()
	m.statReqTotalUnsync(s)

	m.Successful += s.NumberFramed
	m.SuccessRequests += 1
}

func (m *Client) statReqErr(s *MiniStats) {
	m.statLock.Lock()
	defer m.statLock.Unlock()
	m.statReqTotalUnsync(s)

	m.Cancelled += s.NumberFramed
	m.CancelRequests += 1
}

func (m *Client) statReqRej(s *MiniStats) {
	m.statLock.Lock()
	defer m.statLock.Unlock()
	m.statReqTotalUnsync(s)

	m.Rejected += s.NumberFramed
	m.RejectRequests += 1
}

func (m *Client) statReqDrop(s *MiniStats) {
	m.statLock.Lock()
	defer m.statLock.Unlock()
	m.statReqTotalUnsync(s)

	m.Dropped += s.NumberFramed
	m.DroppedRequests += 1
}
<|endoftext|>


<commit_before>package git

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

type ApplyOptions struct {
	Stat, NumStat, Summary bool

	Check, Index, Cached bool

	ThreeWay bool

	BuildFakeAncestor string

	Reject bool

	NullTerminate bool

	Strip, Context int

	UnidiffZero bool

	ForceApply bool

	Reverse bool

	NoAdd bool

	ExcludePattern, IncludePattern string

	InaccurateEof bool

	Verbose bool

	Recount bool

	Directory string

	UnsafePaths bool

	Whitespace string
}

func Apply(c *Client, opts ApplyOptions, patches []File) error {
	// 1. Make a temporary directory to patch files in to ensure atomicity
	// 2. Copy files to tempdir
	// 3. Run an external patch tool
	// 4. If successful, copy the files back over WorkDir
	patchdir, err := ioutil.TempDir("", "gitapply")
	if err != nil {
		return err
	}
	if !opts.Cached {
		defer os.RemoveAll(patchdir)
	}

	// First pass, parse the patches to figure out which files are involved
	files := make(map[IndexPath]bool)
	for _, patch := range patches {
		patch, err := ioutil.ReadFile(patch.String())
		if err != nil {
			return err
		}
		hunks, err := splitPatch(string(patch), true)
		if err != nil {
			return err
		}
		for _, hunk := range hunks {
			files[hunk.File] = true
		}
	}

	// Copy all of the files. We do this in a second pass to void
	// needlessly recopying the same files multiple times.
	var idx *Index
	if opts.Cached {
		idx2, err := c.GitDir.ReadIndex()
		if err != nil {
			return err
		}
		idx = idx2
	}
	for file := range files {
		f, err := file.FilePath(c)
		if err != nil {
			return err
		}

		dst := patchdir + "/" + file.String()
		if opts.Cached {
			if err := copyFromIndex(c, idx, file, dst); err != nil {
				return err
			}
		} else {
			if err := copyFile(f.String(), dst); err != nil {
				return err
			}
		}
	}

	var patchDirection string
	if opts.Reverse {
		patchDirection = "-R"
	} else {
		patchDirection = "-N"
	}
	for _, patch := range patches {
		patchcmd := exec.Command(posixPatch, "--directory", patchdir, "-i", patch.String(), patchDirection, "-p1")
		patchcmd.Stderr = os.Stderr
		_, err := patchcmd.Output()
		if err != nil {
			return err
		}
	}
	if opts.Cached {
		return updateApplyIndex(c, idx, patchdir)
	}
	return copyApplyDir(c, patchdir)

}

// RestoreDir takes the directory dir, which is the directory that apply did
// its work in, and copies it back into the workdir.
func copyApplyDir(c *Client, dir string) error {
	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}
		relpath := strings.TrimPrefix(path, dir+"/")
		dstpath := c.WorkDir.String() + "/" + relpath
		return copyFile(path, dstpath)
	})
	return nil
}

// RestoreDir takes the directory dir, which is the directory that apply did
// its work in, and copies it back into the workdir.
func updateApplyIndex(c *Client, idx *Index, dir string) error {
	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}
		relpath := strings.TrimPrefix(path, dir+"/")
		fmt.Printf("%v\n", relpath)
		contents, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		sha1, err := c.WriteObject("blob", contents)
		if err != nil {
			return err
		}

		ipath := IndexPath(relpath)
		// Avoid using AddStage so that mtime doesn't get modified,
		// and cause false negatives for file status
		for _, entry := range idx.Objects {
			if entry.PathName != ipath {
				continue
			}
			entry.Sha1 = sha1
			return nil
		}
		return nil
	})
	// Write the index that the callback modified
	f, err := c.GitDir.Create(File("index"))
	if err != nil {
		return err
	}
	defer f.Close()
	return idx.WriteIndex(f)
}

func copyFile(src, dst string) error {
	s, err := os.Open(src)
	if err != nil {
		return err
	}
	defer s.Close()

	// Ensure that the directory exists for dst.
	dir := filepath.Dir(dst)
	if dir != "." {
		if err := os.MkdirAll(dir, 0755); err != nil {
			return err
		}
	}

	// Create/truncate the file and copy
	d, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer d.Close()

	_, err = io.Copy(d, s)
	return err
}

func copyFromIndex(c *Client, idx *Index, file IndexPath, dst string) error {
	sha1 := idx.GetSha1(file)
	obj, err := c.GetObject(sha1)
	if err != nil {
		return err
	}

	// Ensure that the directory exists for dst.
	dir := filepath.Dir(dst)
	if dir != "." {
		if err := os.MkdirAll(dir, 0755); err != nil {
			return err
		}
	}

	return ioutil.WriteFile(dst, obj.GetContent(), 0644)
}
<commit_msg>Remove accidentally left behind debugging statements<commit_after>package git

import (
	//	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

type ApplyOptions struct {
	Stat, NumStat, Summary bool

	Check, Index, Cached bool

	ThreeWay bool

	BuildFakeAncestor string

	Reject bool

	NullTerminate bool

	Strip, Context int

	UnidiffZero bool

	ForceApply bool

	Reverse bool

	NoAdd bool

	ExcludePattern, IncludePattern string

	InaccurateEof bool

	Verbose bool

	Recount bool

	Directory string

	UnsafePaths bool

	Whitespace string
}

func Apply(c *Client, opts ApplyOptions, patches []File) error {
	// 1. Make a temporary directory to patch files in to ensure atomicity
	// 2. Copy files to tempdir
	// 3. Run an external patch tool
	// 4. If successful, copy the files back over WorkDir
	patchdir, err := ioutil.TempDir("", "gitapply")
	if err != nil {
		return err
	}
	defer os.RemoveAll(patchdir)

	// First pass, parse the patches to figure out which files are involved
	files := make(map[IndexPath]bool)
	for _, patch := range patches {
		patch, err := ioutil.ReadFile(patch.String())
		if err != nil {
			return err
		}
		hunks, err := splitPatch(string(patch), true)
		if err != nil {
			return err
		}
		for _, hunk := range hunks {
			files[hunk.File] = true
		}
	}

	// Copy all of the files. We do this in a second pass to void
	// needlessly recopying the same files multiple times.
	var idx *Index
	if opts.Cached {
		idx2, err := c.GitDir.ReadIndex()
		if err != nil {
			return err
		}
		idx = idx2
	}
	for file := range files {
		f, err := file.FilePath(c)
		if err != nil {
			return err
		}

		dst := patchdir + "/" + file.String()
		if opts.Cached {
			if err := copyFromIndex(c, idx, file, dst); err != nil {
				return err
			}
		} else {
			if err := copyFile(f.String(), dst); err != nil {
				return err
			}
		}
	}

	var patchDirection string
	if opts.Reverse {
		patchDirection = "-R"
	} else {
		patchDirection = "-N"
	}
	for _, patch := range patches {
		patchcmd := exec.Command(posixPatch, "--directory", patchdir, "-i", patch.String(), patchDirection, "-p1")
		patchcmd.Stderr = os.Stderr
		_, err := patchcmd.Output()
		if err != nil {
			return err
		}
	}
	if opts.Cached {
		return updateApplyIndex(c, idx, patchdir)
	}
	return copyApplyDir(c, patchdir)

}

// RestoreDir takes the directory dir, which is the directory that apply did
// its work in, and copies it back into the workdir.
func copyApplyDir(c *Client, dir string) error {
	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}
		relpath := strings.TrimPrefix(path, dir+"/")
		dstpath := c.WorkDir.String() + "/" + relpath
		return copyFile(path, dstpath)
	})
	return nil
}

// RestoreDir takes the directory dir, which is the directory that apply did
// its work in, and copies it back into the workdir.
func updateApplyIndex(c *Client, idx *Index, dir string) error {
	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}
		relpath := strings.TrimPrefix(path, dir+"/")
		contents, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		sha1, err := c.WriteObject("blob", contents)
		if err != nil {
			return err
		}

		ipath := IndexPath(relpath)
		// Avoid using AddStage so that mtime doesn't get modified,
		// and cause false negatives for file status
		for _, entry := range idx.Objects {
			if entry.PathName != ipath {
				continue
			}
			entry.Sha1 = sha1
			return nil
		}
		return nil
	})
	// Write the index that the callback modified
	f, err := c.GitDir.Create(File("index"))
	if err != nil {
		return err
	}
	defer f.Close()
	return idx.WriteIndex(f)
}

func copyFile(src, dst string) error {
	s, err := os.Open(src)
	if err != nil {
		return err
	}
	defer s.Close()

	// Ensure that the directory exists for dst.
	dir := filepath.Dir(dst)
	if dir != "." {
		if err := os.MkdirAll(dir, 0755); err != nil {
			return err
		}
	}

	// Create/truncate the file and copy
	d, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer d.Close()

	_, err = io.Copy(d, s)
	return err
}

func copyFromIndex(c *Client, idx *Index, file IndexPath, dst string) error {
	sha1 := idx.GetSha1(file)
	obj, err := c.GetObject(sha1)
	if err != nil {
		return err
	}

	// Ensure that the directory exists for dst.
	dir := filepath.Dir(dst)
	if dir != "." {
		if err := os.MkdirAll(dir, 0755); err != nil {
			return err
		}
	}

	return ioutil.WriteFile(dst, obj.GetContent(), 0644)
}
<|endoftext|>


<commit_before>package app

import (
	"sync"
	"time"

	"github.com/moncho/dry/ui"
	"github.com/nsf/termbox-go"
	log "github.com/sirupsen/logrus"
)

var refreshScreen func() error
var refreshIfView func(v viewMode) error
var widgets *widgetRegistry

type nextHandler func(eh eventHandler)

//RenderLoop runs dry
// nolint: gocyclo
func RenderLoop(dry *Dry) {
	if ok, err := dry.Ok(); !ok {
		log.Error(err.Error())
		return
	}
	screen := dry.screen
	termuiEvents, done := ui.EventChannel()

	//use to signal rendering
	renderChan := make(chan struct{})

	var closingLock sync.RWMutex
	refreshScreen = func() error {
		closingLock.RLock()
		defer closingLock.RUnlock()

		renderChan <- struct{}{}
		return nil
	}

	refreshIfView = func(v viewMode) error {
		if v == dry.viewMode() {
			return refreshScreen()
		}
		return nil
	}

	dryOutputChan := dry.OuputChannel()

	var wg sync.WaitGroup
	go func() {
		wg.Add(1)
		defer wg.Done()

		for range renderChan {
			if !screen.Closing() {
				screen.Clear()
				render(dry, screen)
			}
		}
	}()

	refreshScreen()

	go func() {
		statusBar := widgets.MessageBar
		for {
			select {
			case dryMessage, ok := <-dryOutputChan:
				if ok {
					statusBar.Message(dryMessage, 10*time.Second)
					statusBar.Render()
				} else {
					return
				}
			}
		}
	}()

	handler := viewsToHandlers[dry.viewMode()]
	//main loop that handles termui events
loop:
	for event := range termuiEvents {
		switch event.Type {
		case termbox.EventInterrupt:
			break loop
		case termbox.EventKey:
			//Ctrl+C breaks the loop (and exits dry) no matter what
			if event.Key == termbox.KeyCtrlC || event.Ch == 'Q' {
				break loop
			}
			handler.handle(event, func(eh eventHandler) {
				handler = eh
			})

		case termbox.EventResize:
			ui.Resize()
			//Reload dry ui elements
			widgets = initRegistry(dry.dockerDaemon)
		}
	}

	log.Debug("something broke the loop. Time to die")

	//Close terminal event channel
	close(done)
	//make the global refreshScreen func a noop before closing
	closingLock.Lock()
	refreshScreen = func() error {
		return nil
	}
	closingLock.Unlock()

	//Close the channel used to notify the rendering goroutine
	close(renderChan)
	//Wait for the rendering goroutine to exit
	wg.Wait()
}
<commit_msg>Add to waitgroup before starting goroutine<commit_after>package app

import (
	"sync"
	"time"

	"github.com/moncho/dry/ui"
	"github.com/nsf/termbox-go"
	log "github.com/sirupsen/logrus"
)

var refreshScreen func() error
var refreshIfView func(v viewMode) error
var widgets *widgetRegistry

type nextHandler func(eh eventHandler)

//RenderLoop runs dry
// nolint: gocyclo
func RenderLoop(dry *Dry) {
	if ok, err := dry.Ok(); !ok {
		log.Error(err.Error())
		return
	}
	screen := dry.screen
	termuiEvents, done := ui.EventChannel()

	//use to signal rendering
	renderChan := make(chan struct{})

	var closingLock sync.RWMutex
	refreshScreen = func() error {
		closingLock.RLock()
		defer closingLock.RUnlock()

		renderChan <- struct{}{}
		return nil
	}

	refreshIfView = func(v viewMode) error {
		if v == dry.viewMode() {
			return refreshScreen()
		}
		return nil
	}

	dryOutputChan := dry.OuputChannel()

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()

		for range renderChan {
			if !screen.Closing() {
				screen.Clear()
				render(dry, screen)
			}
		}
	}()

	refreshScreen()

	go func() {
		statusBar := widgets.MessageBar
		for {
			select {
			case dryMessage, ok := <-dryOutputChan:
				if ok {
					statusBar.Message(dryMessage, 10*time.Second)
					statusBar.Render()
				} else {
					return
				}
			}
		}
	}()

	handler := viewsToHandlers[dry.viewMode()]
	//main loop that handles termui events
loop:
	for event := range termuiEvents {
		switch event.Type {
		case termbox.EventInterrupt:
			break loop
		case termbox.EventKey:
			//Ctrl+C breaks the loop (and exits dry) no matter what
			if event.Key == termbox.KeyCtrlC || event.Ch == 'Q' {
				break loop
			}
			handler.handle(event, func(eh eventHandler) {
				handler = eh
			})

		case termbox.EventResize:
			ui.Resize()
			//Reload dry ui elements
			widgets = initRegistry(dry.dockerDaemon)
		}
	}

	log.Debug("something broke the loop. Time to die")

	//Close terminal event channel
	close(done)
	//make the global refreshScreen func a noop before closing
	closingLock.Lock()
	refreshScreen = func() error {
		return nil
	}
	closingLock.Unlock()

	//Close the channel used to notify the rendering goroutine
	close(renderChan)
	//Wait for the rendering goroutine to exit
	wg.Wait()
}
<|endoftext|>