<commit_before>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"sync"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/log"
)

const (
	namespace = "nginx" // For Prometheus metrics.
)

var (
	listeningAddress = flag.String("telemetry.address", ":9113", "Address on which to expose metrics.")
	metricsEndpoint  = flag.String("telemetry.endpoint", "/metrics", "Path under which to expose metrics.")
	nginxScrapeURI   = flag.String("nginx.scrape_uri", "http://localhost/nginx_status", "URI to nginx stub status page")
	insecure         = flag.Bool("insecure", true, "Ignore server certificate if using https")
)

// Exporter collects nginx stats from the given URI and exports them using
// the prometheus metrics package.
type Exporter struct {
	URI    string
	mutex  sync.RWMutex
	client *http.Client

	scrapeFailures       prometheus.Counter
	processedConnections *prometheus.CounterVec
	currentConnections   *prometheus.GaugeVec
}

// NewExporter returns an initialized Exporter.
func NewExporter(uri string) *Exporter {
	return &Exporter{
		URI: uri,
		scrapeFailures: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Name:      "exporter_scrape_failures_total",
			Help:      "Number of errors while scraping nginx.",
		}),
		processedConnections: prometheus.NewCounterVec(prometheus.CounterOpts{
			Namespace: namespace,
			Name:      "connections_processed_total",
			Help:      "Number of connections processed by nginx",
		},
			[]string{"stage"},
		),
		currentConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "connections_current",
			Help:      "Number of connections currently processed by nginx",
		},
			[]string{"state"},
		),
		client: &http.Client{
			Transport: &http.Transport{
				TLSClientConfig: &tls.Config{InsecureSkipVerify: *insecure},
			},
		},
	}
}

// Describe describes all the metrics ever exported by the nginx exporter. It
// implements prometheus.Collector.
func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	e.processedConnections.Describe(ch)
	e.currentConnections.Describe(ch)
	e.scrapeFailures.Describe(ch)
}

func (e *Exporter) collect(ch chan<- prometheus.Metric) error {
	resp, err := e.client.Get(e.URI)
	if err != nil {
		return fmt.Errorf("Error scraping nginx: %v", err)
	}

	data, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 400 {
		if err != nil {
			data = []byte(err.Error())
		}
		return fmt.Errorf("Status %s (%d): %s", resp.Status, resp.StatusCode, data)
	}

	// Parsing results
	lines := strings.Split(string(data), "\n")
	if len(lines) != 5 {
		return fmt.Errorf("Unexpected number of lines in status: %v", lines)
	}

	// active connections
	parts := strings.Split(lines[0], ":")
	if len(parts) != 2 {
		return fmt.Errorf("Unexpected first line: %s", lines[0])
	}
	v, err := strconv.Atoi(strings.TrimSpace(parts[1]))
	if err != nil {
		return err
	}
	e.currentConnections.WithLabelValues("active").Set(float64(v))

	// processed connections
	parts = strings.Fields(lines[2])
	if len(parts) != 3 {
		return fmt.Errorf("Unexpected third line: %s", lines[2])
	}
	v, err = strconv.Atoi(strings.TrimSpace(parts[0]))
	if err != nil {
		return err
	}
	e.processedConnections.WithLabelValues("accepted").Set(float64(v))
	v, err = strconv.Atoi(strings.TrimSpace(parts[1]))
	if err != nil {
		return err
	}
	e.processedConnections.WithLabelValues("handled").Set(float64(v))
	v, err = strconv.Atoi(strings.TrimSpace(parts[2]))
	if err != nil {
		return err
	}
	e.processedConnections.WithLabelValues("any").Set(float64(v))

	// current connections
	parts = strings.Fields(lines[3])
	if len(parts) != 6 {
		return fmt.Errorf("Unexpected fourth line: %s", lines[3])
	}
	v, err = strconv.Atoi(strings.TrimSpace(parts[1]))
	if err != nil {
		return err
	}
	e.currentConnections.WithLabelValues("reading").Set(float64(v))
	v, err = strconv.Atoi(strings.TrimSpace(parts[3]))
	if err != nil {
		return err
	}

	e.currentConnections.WithLabelValues("writing").Set(float64(v))
	v, err = strconv.Atoi(strings.TrimSpace(parts[5]))
	if err != nil {
		return err
	}
	e.currentConnections.WithLabelValues("waiting").Set(float64(v))
	return nil
}

// Collect fetches the stats from configured nginx location and delivers them
// as Prometheus metrics. It implements prometheus.Collector.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	e.mutex.Lock() // To protect metrics from concurrent collects.
	defer e.mutex.Unlock()
	if err := e.collect(ch); err != nil {
		log.Printf("Error scraping nginx: %s", err)
		e.scrapeFailures.Inc()
		e.scrapeFailures.Collect(ch)
	}
	e.processedConnections.Collect(ch)
	e.currentConnections.Collect(ch)
	return
}

func main() {
	flag.Parse()

	exporter := NewExporter(*nginxScrapeURI)
	prometheus.MustRegister(exporter)

	log.Printf("Starting Server: %s", *listeningAddress)
	http.Handle(*metricsEndpoint, prometheus.Handler())
	log.Fatal(http.ListenAndServe(*listeningAddress, nil))
}
<commit_msg>serve index page instead of 404<commit_after>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"sync"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/log"
)

const (
	namespace = "nginx" // For Prometheus metrics.
)

var (
	listeningAddress = flag.String("telemetry.address", ":9113", "Address on which to expose metrics.")
	metricsEndpoint  = flag.String("telemetry.endpoint", "/metrics", "Path under which to expose metrics.")
	nginxScrapeURI   = flag.String("nginx.scrape_uri", "http://localhost/nginx_status", "URI to nginx stub status page")
	insecure         = flag.Bool("insecure", true, "Ignore server certificate if using https")
)

// Exporter collects nginx stats from the given URI and exports them using
// the prometheus metrics package.
type Exporter struct {
	URI    string
	mutex  sync.RWMutex
	client *http.Client

	scrapeFailures       prometheus.Counter
	processedConnections *prometheus.CounterVec
	currentConnections   *prometheus.GaugeVec
}

// NewExporter returns an initialized Exporter.
func NewExporter(uri string) *Exporter {
	return &Exporter{
		URI: uri,
		scrapeFailures: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Name:      "exporter_scrape_failures_total",
			Help:      "Number of errors while scraping nginx.",
		}),
		processedConnections: prometheus.NewCounterVec(prometheus.CounterOpts{
			Namespace: namespace,
			Name:      "connections_processed_total",
			Help:      "Number of connections processed by nginx",
		},
			[]string{"stage"},
		),
		currentConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "connections_current",
			Help:      "Number of connections currently processed by nginx",
		},
			[]string{"state"},
		),
		client: &http.Client{
			Transport: &http.Transport{
				TLSClientConfig: &tls.Config{InsecureSkipVerify: *insecure},
			},
		},
	}
}

// Describe describes all the metrics ever exported by the nginx exporter. It
// implements prometheus.Collector.
func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	e.processedConnections.Describe(ch)
	e.currentConnections.Describe(ch)
	e.scrapeFailures.Describe(ch)
}

func (e *Exporter) collect(ch chan<- prometheus.Metric) error {
	resp, err := e.client.Get(e.URI)
	if err != nil {
		return fmt.Errorf("Error scraping nginx: %v", err)
	}

	data, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 400 {
		if err != nil {
			data = []byte(err.Error())
		}
		return fmt.Errorf("Status %s (%d): %s", resp.Status, resp.StatusCode, data)
	}

	// Parsing results
	lines := strings.Split(string(data), "\n")
	if len(lines) != 5 {
		return fmt.Errorf("Unexpected number of lines in status: %v", lines)
	}

	// active connections
	parts := strings.Split(lines[0], ":")
	if len(parts) != 2 {
		return fmt.Errorf("Unexpected first line: %s", lines[0])
	}
	v, err := strconv.Atoi(strings.TrimSpace(parts[1]))
	if err != nil {
		return err
	}
	e.currentConnections.WithLabelValues("active").Set(float64(v))

	// processed connections
	parts = strings.Fields(lines[2])
	if len(parts) != 3 {
		return fmt.Errorf("Unexpected third line: %s", lines[2])
	}
	v, err = strconv.Atoi(strings.TrimSpace(parts[0]))
	if err != nil {
		return err
	}
	e.processedConnections.WithLabelValues("accepted").Set(float64(v))
	v, err = strconv.Atoi(strings.TrimSpace(parts[1]))
	if err != nil {
		return err
	}
	e.processedConnections.WithLabelValues("handled").Set(float64(v))
	v, err = strconv.Atoi(strings.TrimSpace(parts[2]))
	if err != nil {
		return err
	}
	e.processedConnections.WithLabelValues("any").Set(float64(v))

	// current connections
	parts = strings.Fields(lines[3])
	if len(parts) != 6 {
		return fmt.Errorf("Unexpected fourth line: %s", lines[3])
	}
	v, err = strconv.Atoi(strings.TrimSpace(parts[1]))
	if err != nil {
		return err
	}
	e.currentConnections.WithLabelValues("reading").Set(float64(v))
	v, err = strconv.Atoi(strings.TrimSpace(parts[3]))
	if err != nil {
		return err
	}

	e.currentConnections.WithLabelValues("writing").Set(float64(v))
	v, err = strconv.Atoi(strings.TrimSpace(parts[5]))
	if err != nil {
		return err
	}
	e.currentConnections.WithLabelValues("waiting").Set(float64(v))
	return nil
}

// Collect fetches the stats from configured nginx location and delivers them
// as Prometheus metrics. It implements prometheus.Collector.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	e.mutex.Lock() // To protect metrics from concurrent collects.
	defer e.mutex.Unlock()
	if err := e.collect(ch); err != nil {
		log.Printf("Error scraping nginx: %s", err)
		e.scrapeFailures.Inc()
		e.scrapeFailures.Collect(ch)
	}
	e.processedConnections.Collect(ch)
	e.currentConnections.Collect(ch)
	return
}

func main() {
	flag.Parse()

	exporter := NewExporter(*nginxScrapeURI)
	prometheus.MustRegister(exporter)

	log.Printf("Starting Server: %s", *listeningAddress)
	http.Handle(*metricsEndpoint, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
			<title>Nginx Exporter</title>

Nginx Exporter

<a href="` + *metricsEndpoint + `">Metrics

			`))
	})

	log.Fatal(http.ListenAndServe(*listeningAddress, nil))
}
<|endoftext|>


<commit_before>// Copyright 2015 CloudAwan LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package identity

import (
	"github.com/astaxie/beego/context"
	"github.com/cloudawan/cloudone_gui/controllers/utility/guimessagedisplay"
	"github.com/cloudawan/cloudone_utility/rbac"
)

const (
	loginPageURL = "/gui/login"
)

func FilterUser(ctx *context.Context) {
	if (ctx.Input.IsGet() || ctx.Input.IsPost()) && ctx.Input.URL() == loginPageURL {
		// Don't redirect itself to prevent the circle
	} else {
		user, ok := ctx.Input.Session("user").(*rbac.User)

		if ok == false {
			if guiMessage := guimessagedisplay.GetGUIMessageFromContext(ctx); guiMessage != nil {
				guiMessage.AddDanger("Username or password is incorrect")
			}
			ctx.Redirect(302, loginPageURL)
		} else {
			// Authorize
			if user.HasPermission(componentName, ctx.Input.Method(), ctx.Input.URL()) == false {
				if guiMessage := guimessagedisplay.GetGUIMessageFromContext(ctx); guiMessage != nil {
					guiMessage.AddDanger("User is not authorized to this page. Please use another user with priviledge.")
				}
				ctx.Redirect(302, loginPageURL)
			}

			// Resource check is in another place since GUI doesn't place the resource name in url
		}
	}
}
<commit_msg>Add audit<commit_after>// Copyright 2015 CloudAwan LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package identity

import (
	"github.com/astaxie/beego"
	"github.com/astaxie/beego/context"
	"github.com/cloudawan/cloudone_gui/controllers/utility/guimessagedisplay"
	"github.com/cloudawan/cloudone_utility/audit"
	"github.com/cloudawan/cloudone_utility/rbac"
	"github.com/cloudawan/cloudone_utility/restclient"
)

const (
	loginPageURL = "/gui/login"
)

func FilterUser(ctx *context.Context) {
	if (ctx.Input.IsGet() || ctx.Input.IsPost()) && ctx.Input.URL() == loginPageURL {
		// Don't redirect itself to prevent the circle
	} else {
		user, ok := ctx.Input.Session("user").(*rbac.User)

		if ok == false {
			if guiMessage := guimessagedisplay.GetGUIMessageFromContext(ctx); guiMessage != nil {
				guiMessage.AddDanger("Username or password is incorrect")
			}
			ctx.Redirect(302, loginPageURL)
		} else {
			// Authorize
			if user.HasPermission(componentName, ctx.Input.Method(), ctx.Input.URL()) == false {
				if guiMessage := guimessagedisplay.GetGUIMessageFromContext(ctx); guiMessage != nil {
					guiMessage.AddDanger("User is not authorized to this page. Please use another user with priviledge.")
				}
				ctx.Redirect(302, loginPageURL)
			}

			// Resource check is in another place since GUI doesn't place the resource name in url

			// Audit log
			go func() {
				sendAuditLog(ctx, user.Name)
			}()
		}
	}
}

func sendAuditLog(ctx *context.Context, userName string) {
	cloudoneAnalysisProtocol := beego.AppConfig.String("cloudoneAnalysisProtocol")
	cloudoneAnalysisHost := beego.AppConfig.String("cloudoneAnalysisHost")
	cloudoneAnalysisPort := beego.AppConfig.String("cloudoneAnalysisPort")

	tokenHeaderMap, _ := ctx.Input.Session("tokenHeaderMap").(map[string]string)
	requestURI := ctx.Input.URI()
	method := ctx.Input.Method()
	path := ctx.Input.URL()

	// Header is not used since the header has no useful information for now
	// Body is not used since the backend component will record again.
	// Path is not used since the backend component will record again.
	// Query is not used since the backend component will record again.
	auditLog := audit.CreateAuditLog(componentName, path, userName, nil, nil, method, requestURI, "", nil)

	url := cloudoneAnalysisProtocol + "://" + cloudoneAnalysisHost + ":" + cloudoneAnalysisPort + "/api/v1/auditlogs"

	_, err := restclient.RequestPost(url, auditLog, tokenHeaderMap, false)
	if err != nil {
		if guiMessage := guimessagedisplay.GetGUIMessageFromContext(ctx); guiMessage != nil {
			guiMessage.AddDanger("Fail to send audit log with error " + err.Error())
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2012-2015 Gonéri Le Bouder. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import "notmuch"
import "log"
import "encoding/json"
import "os"
import "io"
import "fmt"
import "regexp"
import "net/mail"
import "path"

type Filter struct {
	Field   string
	Pattern string
	Re      *regexp.Regexp
	Tags    string
}

type Result struct {
	MessageID string
	Tags      string
	Die       bool
        Filename  string
}

const NCPU = 1 // number of CPU cores

func getMaildirLoc() string {
	// honor NOTMUCH_CONFIG
	home := os.Getenv("NOTMUCH_CONFIG")
        notmuch_maildir := os.Getenv("NOTMUCH_MAILDIR")

        if notmuch_maildir != "" {
            return notmuch_maildir
	}

	if home == "" {
		home = os.Getenv("HOME")
	}

	return path.Join(home, "Maildir")
}

func RefreshFlags(nmdb *notmuch.Database) {

	query := nmdb.CreateQuery("tag:inbox and tag:delete")
	msgs := query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:archive")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and not tag:list")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.AddTag("archive")
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and tag:list")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and tag:bug")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		threadId := msg.GetThreadId()
		filter := fmt.Sprintf("thread:%s", threadId)

		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()
			msg.AddTag("inbox")
		}
	}

	query = nmdb.CreateQuery("tag:inbox and tag:killed")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		threadId := msg.GetThreadId()
		filter := fmt.Sprintf("thread:%s", threadId)

		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()
			msg.RemoveTag("inbox")
		}
	}

	nmdb.Close()
	fmt.Print("Ok\n")

}

func studyMsg(filter []Filter, filenameIn chan string, resultOut chan Result, quit chan bool) {
	for {
		filename := <-filenameIn

		if filename == "" {
			var result Result
			result.Die = true
			resultOut <- result

			return
		}
		// We can use Notmuch for this directly because Xappian will
		// fails as soon as we have 2 concurrent goroutine
		file, err := os.Open(filename) // For read access.
		if err != nil {
			log.Fatal(err)
		}
		var msg *mail.Message
		msg, err = mail.ReadMessage(file)

		if err != nil {
			log.Fatal(err)
		}

		var result Result
		result.MessageID = msg.Header.Get("Message-Id")
                if (result.MessageID == "") {
                    fmt.Printf("No message ID for %s\n", filename)
                    continue;
                }
		result.Filename = filename
		for _, f := range filter {
			if f.Re.MatchString(msg.Header.Get(f.Field)) {
				result.Tags += " "
				result.Tags += f.Tags
			}

		}
		file.Close()

		resultOut <- result
	}
}

func loadFilter() (filter []Filter) {

	file, err := os.Open(fmt.Sprintf("/%s/notmuch-filter.json", getMaildirLoc())) // For read access.
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	dec := json.NewDecoder(file)
	for {
		var f Filter
		if err := dec.Decode(&f); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}

		var err error = nil
		if f.Re, err = regexp.Compile(f.Pattern); err != nil {
			log.Printf("error: %v\n", err)
		}

		filter = append(filter, f)
	}

	return filter
}

func studyMsgs(resultOut chan Result, quit chan bool, filenames []string) {

	filter := loadFilter()

	filenameIn := make(chan string)
	for i := 0; i < NCPU+1; i++ {
		go studyMsg(filter, filenameIn, resultOut, quit)
	}
	for _, filename := range filenames {
		filenameIn <- filename
	}

	for i := 0; i < NCPU+1; i++ {
		filenameIn <- ""
	}

	quit <- true
}

func main() {
	var query *notmuch.Query
	var nmdb *notmuch.Database

	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		notmuch.DATABASE_MODE_READ_ONLY); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}

	quit := make(chan bool)
	resultOut := make(chan Result)

	query = nmdb.CreateQuery("tag:new")

	println(">", query.CountMessages(), "<")
	msgs := query.SearchMessages()

	var filenames []string
	if query.CountMessages() > 0 {
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()

			filenames = append(filenames, msg.GetFileName())
		}
	}

	go studyMsgs(resultOut, quit, filenames)

	//	var query *notmuch.Query
	var msgIDRegexp = regexp.MustCompile("^<(.*)>$")
	var tagRegexp = regexp.MustCompile("([\\+-])(\\S+)")

	// open the database
	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		1); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}
	defer nmdb.Close()

	var running int = NCPU + 1
	for {
		result := <-resultOut

		if result.Die {

			running--

			if running > 0 {
				continue
			} else {
                            break
			}
		}

		// Message-ID without the <>
                fmt.Printf("MessageID: %s\n", result.MessageID)
		reResult := msgIDRegexp.FindStringSubmatch(result.MessageID)
		if (reResult == nil) {
			fmt.Printf("Can't parse MessageID for mail %s\n", result.Filename)
			continue
		}

		msgID := reResult[1]
		filter := "id:"
		filter += msgID
		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		msg := msgs.Get()
		if msg == nil {
                    fmt.Printf("Can't find MessageID %s for mail %s\n", msgID, result.Filename)
                    continue
                }

		fmt.Printf("%s, tags: %s\n", msgID, result.Tags)
		msg.Freeze()
		for _, v := range tagRegexp.FindAllStringSubmatch(result.Tags, -1) {
			if v[1] == "+" {
				msg.AddTag(v[2])
			} else if v[1] == "-" {
				msg.RemoveTag(v[2])
			}
		}
		msg.Thaw()

	}
        RefreshFlags(nmdb)
        fmt.Printf("exit\n")
        os.Exit(0)

}
<commit_msg>relax the regex use to find the MsgID<commit_after>// Copyright 2012-2015 Gonéri Le Bouder. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import "notmuch"
import "log"
import "encoding/json"
import "os"
import "io"
import "fmt"
import "regexp"
import "net/mail"
import "path"

type Filter struct {
	Field   string
	Pattern string
	Re      *regexp.Regexp
	Tags    string
}

type Result struct {
	MessageID string
	Tags      string
	Die       bool
        Filename  string
}

const NCPU = 1 // number of CPU cores

func getMaildirLoc() string {
	// honor NOTMUCH_CONFIG
	home := os.Getenv("NOTMUCH_CONFIG")
        notmuch_maildir := os.Getenv("NOTMUCH_MAILDIR")

        if notmuch_maildir != "" {
            return notmuch_maildir
	}

	if home == "" {
		home = os.Getenv("HOME")
	}

	return path.Join(home, "Maildir")
}

func RefreshFlags(nmdb *notmuch.Database) {

	query := nmdb.CreateQuery("tag:inbox and tag:delete")
	msgs := query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:archive")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and not tag:list")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.AddTag("archive")
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and tag:list")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and tag:bug")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		threadId := msg.GetThreadId()
		filter := fmt.Sprintf("thread:%s", threadId)

		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()
			msg.AddTag("inbox")
		}
	}

	query = nmdb.CreateQuery("tag:inbox and tag:killed")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		threadId := msg.GetThreadId()
		filter := fmt.Sprintf("thread:%s", threadId)

		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()
			msg.RemoveTag("inbox")
		}
	}

	nmdb.Close()
	fmt.Print("Ok\n")

}

func studyMsg(filter []Filter, filenameIn chan string, resultOut chan Result, quit chan bool) {
	for {
		filename := <-filenameIn

		if filename == "" {
			var result Result
			result.Die = true
			resultOut <- result

			return
		}
		// We can use Notmuch for this directly because Xappian will
		// fails as soon as we have 2 concurrent goroutine
		file, err := os.Open(filename) // For read access.
		if err != nil {
			log.Fatal(err)
		}
		var msg *mail.Message
		msg, err = mail.ReadMessage(file)

		if err != nil {
			log.Fatal(err)
		}

		var result Result
		result.MessageID = msg.Header.Get("Message-Id")
                if (result.MessageID == "") {
                    fmt.Printf("No message ID for %s\n", filename)
                    continue;
                }
		result.Filename = filename
		for _, f := range filter {
			if f.Re.MatchString(msg.Header.Get(f.Field)) {
				result.Tags += " "
				result.Tags += f.Tags
			}

		}
		file.Close()

		resultOut <- result
	}
}

func loadFilter() (filter []Filter) {

	file, err := os.Open(fmt.Sprintf("/%s/notmuch-filter.json", getMaildirLoc())) // For read access.
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	dec := json.NewDecoder(file)
	for {
		var f Filter
		if err := dec.Decode(&f); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}

		var err error = nil
		if f.Re, err = regexp.Compile(f.Pattern); err != nil {
			log.Printf("error: %v\n", err)
		}

		filter = append(filter, f)
	}

	return filter
}

func studyMsgs(resultOut chan Result, quit chan bool, filenames []string) {

	filter := loadFilter()

	filenameIn := make(chan string)
	for i := 0; i < NCPU+1; i++ {
		go studyMsg(filter, filenameIn, resultOut, quit)
	}
	for _, filename := range filenames {
		filenameIn <- filename
	}

	for i := 0; i < NCPU+1; i++ {
		filenameIn <- ""
	}

	quit <- true
}

func main() {
	var query *notmuch.Query
	var nmdb *notmuch.Database

	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		notmuch.DATABASE_MODE_READ_ONLY); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}

	quit := make(chan bool)
	resultOut := make(chan Result)

	query = nmdb.CreateQuery("tag:new")

	println(">", query.CountMessages(), "<")
	msgs := query.SearchMessages()

	var filenames []string
	if query.CountMessages() > 0 {
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()

			filenames = append(filenames, msg.GetFileName())
		}
	}

	go studyMsgs(resultOut, quit, filenames)

	//	var query *notmuch.Query
	var msgIDRegexp = regexp.MustCompile("^<(.*?)>")
	var tagRegexp = regexp.MustCompile("([\\+-])(\\S+)")

	// open the database
	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		1); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}
	defer nmdb.Close()

	var running int = NCPU + 1
	for {
		result := <-resultOut

		if result.Die {

			running--

			if running > 0 {
				continue
			} else {
                            break
			}
		}

		// Message-ID without the <>
                fmt.Printf("MessageID: %s\n", result.MessageID)
		reResult := msgIDRegexp.FindStringSubmatch(result.MessageID)
		if (reResult == nil) {
			fmt.Printf("Can't parse MessageID for mail %s\n", result.Filename)
			continue
		}

		msgID := reResult[1]
		filter := "id:"
		filter += msgID
		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		msg := msgs.Get()
		if msg == nil {
                    fmt.Printf("Can't find MessageID %s for mail %s\n", msgID, result.Filename)
                    continue
                }

		fmt.Printf("%s, tags: %s\n", msgID, result.Tags)
		msg.Freeze()
		for _, v := range tagRegexp.FindAllStringSubmatch(result.Tags, -1) {
			if v[1] == "+" {
				msg.AddTag(v[2])
			} else if v[1] == "-" {
				msg.RemoveTag(v[2])
			}
		}
		msg.Thaw()

	}
        RefreshFlags(nmdb)
        fmt.Printf("exit\n")
        os.Exit(0)

}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// This file should be consistent with pkg/api/annotation_key_constants.go.

package v1

const (
	// ImagePolicyFailedOpenKey is added to pods created by failing open when the image policy
	// webhook backend fails.
	ImagePolicyFailedOpenKey string = "alpha.image-policy.k8s.io/failed-open"

	// PodPresetOptOutAnnotationKey represents the annotation key for a pod to exempt itself from pod preset manipulation
	PodPresetOptOutAnnotationKey string = "podpreset.admission.kubernetes.io/exclude"

	// MirrorAnnotationKey represents the annotation key set by kubelets when creating mirror pods
	MirrorPodAnnotationKey string = "kubernetes.io/config.mirror"

	// TolerationsAnnotationKey represents the key of tolerations data (json serialized)
	// in the Annotations of a Pod.
	TolerationsAnnotationKey string = "scheduler.alpha.kubernetes.io/tolerations"

	// TaintsAnnotationKey represents the key of taints data (json serialized)
	// in the Annotations of a Node.
	TaintsAnnotationKey string = "scheduler.alpha.kubernetes.io/taints"

	// SeccompPodAnnotationKey represents the key of a seccomp profile applied
	// to all containers of a pod.
	SeccompPodAnnotationKey string = "seccomp.security.alpha.kubernetes.io/pod"

	// SeccompContainerAnnotationKeyPrefix represents the key of a seccomp profile applied
	// to one container of a pod.
	SeccompContainerAnnotationKeyPrefix string = "container.seccomp.security.alpha.kubernetes.io/"

	// SeccompProfileRuntimeDefault represents the default seccomp profile used by container runtime.
	SeccompProfileRuntimeDefault string = "runtime/default"

	// DeprecatedSeccompProfileDockerDefault represents the default seccomp profile used by docker.
	// This is now deprecated and should be replaced by SeccompProfileRuntimeDefault.
	DeprecatedSeccompProfileDockerDefault string = "docker/default"

	// PreferAvoidPodsAnnotationKey represents the key of preferAvoidPods data (json serialized)
	// in the Annotations of a Node.
	PreferAvoidPodsAnnotationKey string = "scheduler.alpha.kubernetes.io/preferAvoidPods"

	// ObjectTTLAnnotations represents a suggestion for kubelet for how long it can cache
	// an object (e.g. secret, config map) before fetching it again from apiserver.
	// This annotation can be attached to node.
	ObjectTTLAnnotationKey string = "node.alpha.kubernetes.io/ttl"

	// annotation key prefix used to identify non-convertible json paths.
	NonConvertibleAnnotationPrefix = "non-convertible.kubernetes.io"

	kubectlPrefix = "kubectl.kubernetes.io/"

	// LastAppliedConfigAnnotation is the annotation used to store the previous
	// configuration of a resource for use in a three way diff by UpdateApplyAnnotation.
	LastAppliedConfigAnnotation = kubectlPrefix + "last-applied-configuration"

	// AnnotationLoadBalancerSourceRangesKey is the key of the annotation on a service to set allowed ingress ranges on their LoadBalancers
	//
	// It should be a comma-separated list of CIDRs, e.g. `0.0.0.0/0` to
	// allow full access (the default) or `18.0.0.0/8,56.0.0.0/8` to allow
	// access only from the CIDRs currently allocated to MIT & the USPS.
	//
	// Not all cloud providers support this annotation, though AWS & GCE do.
	AnnotationLoadBalancerSourceRangesKey = "service.beta.kubernetes.io/load-balancer-source-ranges"

	// EndpointsLastChangeTriggerTime is the annotation key, set for endpoints objects, that
	// represents the timestamp (stored as RFC 3339 date-time string, e.g. '2018-10-22T19:32:52.1Z')
	// of the last change, of some Pod or Service object, that triggered the endpoints object change.
	// In other words, if a Pod / Service changed at time T0, that change was observed by endpoints
	// controller at T1, and the Endpoints object was changed at T2, the
	// EndpointsLastChangeTriggerTime would be set to T0.
	//
	// The "endpoints change trigger" here means any Pod or Service change that resulted in the
	// Endpoints object change.
	//
	// Given the definition of the "endpoints change trigger", please note that this annotation will
	// be set ONLY for endpoints object changes triggered by either Pod or Service change. If the
	// Endpoints object changes due to other reasons, this annotation won't be set (or updated if it's
	// already set).
	//
	// This annotation will be used to compute the in-cluster network programming latency SLI, see
	// https://github.com/kubernetes/community/blob/master/sig-scalability/slos/network_programming_latency.md
	EndpointsLastChangeTriggerTime = "endpoints.kubernetes.io/last-change-trigger-time"

	// TODO(dyzz) Comment
	MigratedPluginsAnnotationKey = "storage.alpha.kubernetes.io/migrated-plugins"
)
<commit_msg>Address review comments<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// This file should be consistent with pkg/api/annotation_key_constants.go.

package v1

const (
	// ImagePolicyFailedOpenKey is added to pods created by failing open when the image policy
	// webhook backend fails.
	ImagePolicyFailedOpenKey string = "alpha.image-policy.k8s.io/failed-open"

	// PodPresetOptOutAnnotationKey represents the annotation key for a pod to exempt itself from pod preset manipulation
	PodPresetOptOutAnnotationKey string = "podpreset.admission.kubernetes.io/exclude"

	// MirrorAnnotationKey represents the annotation key set by kubelets when creating mirror pods
	MirrorPodAnnotationKey string = "kubernetes.io/config.mirror"

	// TolerationsAnnotationKey represents the key of tolerations data (json serialized)
	// in the Annotations of a Pod.
	TolerationsAnnotationKey string = "scheduler.alpha.kubernetes.io/tolerations"

	// TaintsAnnotationKey represents the key of taints data (json serialized)
	// in the Annotations of a Node.
	TaintsAnnotationKey string = "scheduler.alpha.kubernetes.io/taints"

	// SeccompPodAnnotationKey represents the key of a seccomp profile applied
	// to all containers of a pod.
	SeccompPodAnnotationKey string = "seccomp.security.alpha.kubernetes.io/pod"

	// SeccompContainerAnnotationKeyPrefix represents the key of a seccomp profile applied
	// to one container of a pod.
	SeccompContainerAnnotationKeyPrefix string = "container.seccomp.security.alpha.kubernetes.io/"

	// SeccompProfileRuntimeDefault represents the default seccomp profile used by container runtime.
	SeccompProfileRuntimeDefault string = "runtime/default"

	// DeprecatedSeccompProfileDockerDefault represents the default seccomp profile used by docker.
	// This is now deprecated and should be replaced by SeccompProfileRuntimeDefault.
	DeprecatedSeccompProfileDockerDefault string = "docker/default"

	// PreferAvoidPodsAnnotationKey represents the key of preferAvoidPods data (json serialized)
	// in the Annotations of a Node.
	PreferAvoidPodsAnnotationKey string = "scheduler.alpha.kubernetes.io/preferAvoidPods"

	// ObjectTTLAnnotations represents a suggestion for kubelet for how long it can cache
	// an object (e.g. secret, config map) before fetching it again from apiserver.
	// This annotation can be attached to node.
	ObjectTTLAnnotationKey string = "node.alpha.kubernetes.io/ttl"

	// annotation key prefix used to identify non-convertible json paths.
	NonConvertibleAnnotationPrefix = "non-convertible.kubernetes.io"

	kubectlPrefix = "kubectl.kubernetes.io/"

	// LastAppliedConfigAnnotation is the annotation used to store the previous
	// configuration of a resource for use in a three way diff by UpdateApplyAnnotation.
	LastAppliedConfigAnnotation = kubectlPrefix + "last-applied-configuration"

	// AnnotationLoadBalancerSourceRangesKey is the key of the annotation on a service to set allowed ingress ranges on their LoadBalancers
	//
	// It should be a comma-separated list of CIDRs, e.g. `0.0.0.0/0` to
	// allow full access (the default) or `18.0.0.0/8,56.0.0.0/8` to allow
	// access only from the CIDRs currently allocated to MIT & the USPS.
	//
	// Not all cloud providers support this annotation, though AWS & GCE do.
	AnnotationLoadBalancerSourceRangesKey = "service.beta.kubernetes.io/load-balancer-source-ranges"

	// EndpointsLastChangeTriggerTime is the annotation key, set for endpoints objects, that
	// represents the timestamp (stored as RFC 3339 date-time string, e.g. '2018-10-22T19:32:52.1Z')
	// of the last change, of some Pod or Service object, that triggered the endpoints object change.
	// In other words, if a Pod / Service changed at time T0, that change was observed by endpoints
	// controller at T1, and the Endpoints object was changed at T2, the
	// EndpointsLastChangeTriggerTime would be set to T0.
	//
	// The "endpoints change trigger" here means any Pod or Service change that resulted in the
	// Endpoints object change.
	//
	// Given the definition of the "endpoints change trigger", please note that this annotation will
	// be set ONLY for endpoints object changes triggered by either Pod or Service change. If the
	// Endpoints object changes due to other reasons, this annotation won't be set (or updated if it's
	// already set).
	//
	// This annotation will be used to compute the in-cluster network programming latency SLI, see
	// https://github.com/kubernetes/community/blob/master/sig-scalability/slos/network_programming_latency.md
	EndpointsLastChangeTriggerTime = "endpoints.kubernetes.io/last-change-trigger-time"

	// MigratedPluginsAnnotationKey is the annotation key, set for CSINode objects, that is a comma-separated
	// list of in-tree plugins that will be serviced by the CSI backend on the Node represented by CSINode.
	// This annotation is used by the Attach Detach Controller to determine whether to use the in-tree or
	// CSI Backend for a volume plugin on a specific node.
	MigratedPluginsAnnotationKey = "storage.alpha.kubernetes.io/migrated-plugins"
)
<|endoftext|>


<commit_before>package main

import (
	"../nsq"
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"sync"
)

type DiskQueue struct {
	readMutex         sync.Mutex
	writeMutex        sync.Mutex
	name              string
	dataPath          string
	maxBytesPerFile   int64
	readPos           int64
	nextReadPos       int64
	writePos          int64
	readFileNum       int64
	writeFileNum      int64
	depth             int64
	readFile          *os.File
	writeFile         *os.File
	readChan          chan []byte
	exitChan          chan int
	writeContinueChan chan int
}

func NewDiskQueue(name string, dataPath string, maxBytesPerFile int64) nsq.BackendQueue {
	diskQueue := DiskQueue{
		name:              name,
		dataPath:          dataPath,
		maxBytesPerFile:   maxBytesPerFile,
		readChan:          make(chan []byte),
		exitChan:          make(chan int),
		writeContinueChan: make(chan int),
	}

	err := diskQueue.retrieveMetaData()
	if err != nil {
		log.Printf("WARNING: failed to retrieveMetaData() - %s", err.Error())
	}

	go diskQueue.readAheadPump()

	return &diskQueue
}

func (d *DiskQueue) Depth() int64 {
	return d.depth
}

func (d *DiskQueue) ReadChan() chan []byte {
	return d.readChan
}

func (d *DiskQueue) Put(p []byte) error {
	err := d.writeOne(p)
	if err == nil {
		d.depth += 1
	}
	d.writeContinueChan <- 1
	return err
}

func (d *DiskQueue) Close() error {
	d.exitChan <- 1

	if d.readFile != nil {
		d.readFile.Close()
	}
	if d.writeFile != nil {
		d.writeFile.Close()
	}

	err := d.persistMetaData()
	if err != nil {
		return err
	}

	return nil
}

func (d *DiskQueue) readOne() ([]byte, error) {
	var err error
	var msgSize int32

	d.readMutex.Lock()
	defer d.readMutex.Unlock()

	if d.readPos > d.maxBytesPerFile {
		d.readFileNum++
		d.readPos = 0

		if d.readFile != nil {
			d.readFile.Close()
			d.readFile = nil
		}

		err = d.persistMetaData()
		if err != nil {
			return nil, err
		}
	}

	if d.readFile == nil {
		curFileName := d.fileName(d.readFileNum)
		d.readFile, err = os.OpenFile(curFileName, os.O_RDONLY, 0600)
		if err != nil {
			return nil, err
		}

		if d.readPos > 0 {
			_, err = d.readFile.Seek(d.readPos, 0)
			if err != nil {
				return nil, err
			}
		}
	}

	err = binary.Read(d.readFile, binary.BigEndian, &msgSize)
	if err != nil {
		d.readFile.Close()
		d.readFile = nil
		return nil, err
	}

	totalBytes := 4 + msgSize

	readBuf := make([]byte, msgSize)
	_, err = io.ReadFull(d.readFile, readBuf)
	if err != nil {
		d.readFile.Close()
		d.readFile = nil
		return nil, err
	}

	d.nextReadPos = d.readPos + int64(totalBytes)

	// log.Printf("DISK: read %d bytes - readFileNum=%d writeFileNum=%d readPos=%d writePos=%d\n",
	// 	totalBytes, d.readFileNum, d.writeFileNum, d.readPos, d.writePos)

	return readBuf, nil
}

func (d *DiskQueue) writeOne(data []byte) error {
	var err error
	var buf bytes.Buffer

	d.writeMutex.Lock()
	defer d.writeMutex.Unlock()

	if d.writePos > d.maxBytesPerFile {
		d.writeFileNum++
		d.writePos = 0

		if d.writeFile != nil {
			d.writeFile.Close()
			d.writeFile = nil
		}

		err = d.persistMetaData()
		if err != nil {
			return err
		}
	}

	if d.writeFile == nil {
		curFileName := d.fileName(d.writeFileNum)
		d.writeFile, err = os.OpenFile(curFileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
		if err != nil {
			return err
		}

		if d.writePos > 0 {
			_, err = d.writeFile.Seek(d.writePos, 0)
			if err != nil {
				d.writeFile.Close()
				d.writeFile = nil
				return err
			}
		}
	}

	dataLen := len(data)
	totalBytes := 4 + dataLen

	err = binary.Write(&buf, binary.BigEndian, int32(dataLen))
	if err != nil {
		return err
	}

	_, err = buf.Write(data)
	if err != nil {
		return err
	}

	_, err = d.writeFile.Write(buf.Bytes())
	if err != nil {
		d.writeFile.Close()
		d.writeFile = nil
		return err
	}

	err = d.writeFile.Sync()
	if err != nil {
		d.writeFile.Close()
		d.writeFile = nil
		return err
	}

	d.writePos += int64(totalBytes)

	// log.Printf("DISK: wrote %d bytes - readFileNum=%d writeFileNum=%d readPos=%d writePos=%d\n",
	// 	totalBytes, d.readFileNum, d.writeFileNum, d.readPos, d.writePos)

	return nil
}

func (d *DiskQueue) retrieveMetaData() error {
	var f *os.File
	var err error

	fileName := d.metaDataFileName()
	f, err = os.OpenFile(fileName, os.O_RDONLY, 0600)
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = fmt.Fscanf(f, "%d,%d\n%d,%d\n", &d.readFileNum, &d.readPos, &d.writeFileNum, &d.writePos)
	if err != nil {
		return err
	}
	d.nextReadPos = d.readPos

	log.Printf("DISK: retrieved meta data for (%s) - readFileNum=%d writeFileNum=%d readPos=%d writePos=%d",
		d.name, d.readFileNum, d.writeFileNum, d.readPos, d.writePos)

	return nil
}

func (d *DiskQueue) persistMetaData() error {
	var f *os.File
	var err error

	fileName := d.metaDataFileName()
	tmpFileName := fileName + ".tmp"

	// write to tmp file
	f, err = os.OpenFile(tmpFileName, os.O_RDWR|os.O_CREATE, 0600)
	if err != nil {
		return err
	}

	_, err = fmt.Fprintf(f, "%d,%d\n%d,%d\n", d.readFileNum, d.readPos, d.writeFileNum, d.writePos)
	if err != nil {
		f.Close()
		return err
	}
	f.Sync()
	f.Close()

	log.Printf("DISK: persisted meta data for (%s) - readFileNum=%d writeFileNum=%d readPos=%d writePos=%d",
		d.name, d.readFileNum, d.writeFileNum, d.readPos, d.writePos)

	// atomically rename
	return os.Rename(tmpFileName, fileName)
}

func (d *DiskQueue) metaDataFileName() string {
	return fmt.Sprintf(path.Join(d.dataPath, "%s.diskqueue.meta.dat"), d.name)
}

func (d *DiskQueue) fileName(fileNum int64) string {
	return fmt.Sprintf(path.Join(d.dataPath, "%s.diskqueue.%06d.dat"), d.name, fileNum)
}

func (d *DiskQueue) hasDataToRead() bool {
	return (d.readFileNum < d.writeFileNum) || (d.readPos < d.writePos)
}

// read the next message off disk to prime ReadChan
func (d *DiskQueue) readAheadPump() {
	var data []byte
	var err error
	for {
		if d.hasDataToRead() {
			if d.nextReadPos == d.readPos {
				data, err = d.readOne()
				if err != nil {
					// TODO: should this be fatal?
					log.Printf("ERROR: reading from diskqueue(%s) at %d of %s - %s", d.name, d.readPos, d.fileName(d.readFileNum), err.Error())
					continue
				}
			}
			select {
			case d.readChan <- data:
				d.readPos = d.nextReadPos
			case <-d.writeContinueChan:
			case <-d.exitChan:
				return
			}
		} else {
			select {
			case <-d.writeContinueChan:
			case <-d.exitChan:
				return
			}
		}
	}
}
<commit_msg>diskqueue: separate read/write metadata so that synchronization is easier/correct<commit_after>package main

import (
	"../nsq"
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"sync"
)

type DiskQueue struct {
	readMutex         sync.Mutex
	writeMutex        sync.Mutex
	name              string
	dataPath          string
	maxBytesPerFile   int64
	readPos           int64
	nextReadPos       int64
	writePos          int64
	readFileNum       int64
	writeFileNum      int64
	depth             int64
	readFile          *os.File
	writeFile         *os.File
	readChan          chan []byte
	exitChan          chan int
	writeContinueChan chan int
}

func NewDiskQueue(name string, dataPath string, maxBytesPerFile int64) nsq.BackendQueue {
	d := DiskQueue{
		name:              name,
		dataPath:          dataPath,
		maxBytesPerFile:   maxBytesPerFile,
		readChan:          make(chan []byte),
		exitChan:          make(chan int),
		writeContinueChan: make(chan int),
	}

	err := d.retrieveMetaData("read", &d.readFileNum, &d.readPos)
	if err != nil {
		log.Printf("WARNING: failed to retrieveMetaData('read') - %s", err.Error())
	}
	d.nextReadPos = d.readPos

	err = d.retrieveMetaData("write", &d.writeFileNum, &d.writePos)
	if err != nil {
		log.Printf("WARNING: failed to retrieveMetaData('write') - %s", err.Error())
	}

	go d.readAheadPump()

	return &d
}

func (d *DiskQueue) Depth() int64 {
	return d.depth
}

func (d *DiskQueue) ReadChan() chan []byte {
	return d.readChan
}

func (d *DiskQueue) Put(p []byte) error {
	err := d.writeOne(p)
	if err == nil {
		d.depth += 1
	}
	d.writeContinueChan <- 1
	return err
}

func (d *DiskQueue) Close() error {
	d.exitChan <- 1

	d.readMutex.Lock()
	defer d.readMutex.Unlock()

	d.writeMutex.Lock()
	defer d.writeMutex.Unlock()

	if d.readFile != nil {
		d.readFile.Close()
	}

	if d.writeFile != nil {
		d.writeFile.Close()
	}

	err := d.persistMetaData("read", &d.readFileNum, &d.readPos)
	if err != nil {
		return err
	}

	err = d.persistMetaData("write", &d.writeFileNum, &d.writePos)
	if err != nil {
		return err
	}

	return nil
}

func (d *DiskQueue) readOne() ([]byte, error) {
	var err error
	var msgSize int32

	d.readMutex.Lock()
	defer d.readMutex.Unlock()

	if d.readPos > d.maxBytesPerFile {
		d.readFileNum++
		d.readPos = 0

		if d.readFile != nil {
			d.readFile.Close()
			d.readFile = nil
		}

		err = d.persistMetaData("read", &d.readFileNum, &d.readPos)
		if err != nil {
			return nil, err
		}
	}

	if d.readFile == nil {
		curFileName := d.fileName(d.readFileNum)
		d.readFile, err = os.OpenFile(curFileName, os.O_RDONLY, 0600)
		if err != nil {
			return nil, err
		}

		if d.readPos > 0 {
			_, err = d.readFile.Seek(d.readPos, 0)
			if err != nil {
				return nil, err
			}
		}
	}

	err = binary.Read(d.readFile, binary.BigEndian, &msgSize)
	if err != nil {
		d.readFile.Close()
		d.readFile = nil
		return nil, err
	}

	totalBytes := 4 + msgSize

	readBuf := make([]byte, msgSize)
	_, err = io.ReadFull(d.readFile, readBuf)
	if err != nil {
		d.readFile.Close()
		d.readFile = nil
		return nil, err
	}

	d.nextReadPos = d.readPos + int64(totalBytes)

	// log.Printf("DISK: read %d bytes - readFileNum=%d writeFileNum=%d readPos=%d writePos=%d\n",
	// 	totalBytes, d.readFileNum, d.writeFileNum, d.readPos, d.writePos)

	return readBuf, nil
}

func (d *DiskQueue) writeOne(data []byte) error {
	var err error
	var buf bytes.Buffer

	d.writeMutex.Lock()
	defer d.writeMutex.Unlock()

	if d.writePos > d.maxBytesPerFile {
		d.writeFileNum++
		d.writePos = 0

		if d.writeFile != nil {
			d.writeFile.Close()
			d.writeFile = nil
		}

		err = d.persistMetaData("write", &d.writeFileNum, &d.writePos)
		if err != nil {
			return err
		}
	}

	if d.writeFile == nil {
		curFileName := d.fileName(d.writeFileNum)
		d.writeFile, err = os.OpenFile(curFileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
		if err != nil {
			return err
		}

		if d.writePos > 0 {
			_, err = d.writeFile.Seek(d.writePos, 0)
			if err != nil {
				d.writeFile.Close()
				d.writeFile = nil
				return err
			}
		}
	}

	dataLen := len(data)
	totalBytes := 4 + dataLen

	err = binary.Write(&buf, binary.BigEndian, int32(dataLen))
	if err != nil {
		return err
	}

	_, err = buf.Write(data)
	if err != nil {
		return err
	}

	_, err = d.writeFile.Write(buf.Bytes())
	if err != nil {
		d.writeFile.Close()
		d.writeFile = nil
		return err
	}

	err = d.writeFile.Sync()
	if err != nil {
		d.writeFile.Close()
		d.writeFile = nil
		return err
	}

	d.writePos += int64(totalBytes)

	// log.Printf("DISK: wrote %d bytes - readFileNum=%d writeFileNum=%d readPos=%d writePos=%d\n",
	// 	totalBytes, d.readFileNum, d.writeFileNum, d.readPos, d.writePos)

	return nil
}

func (d *DiskQueue) retrieveMetaData(typ string, fileNum *int64, pos *int64) error {
	var f *os.File
	var err error

	fileName := d.metaDataFileName(typ)
	f, err = os.OpenFile(fileName, os.O_RDONLY, 0600)
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = fmt.Fscanf(f, "%d,%d\n", fileNum, pos)
	if err != nil {
		return err
	}

	return nil
}

func (d *DiskQueue) persistMetaData(typ string, fileNum *int64, pos *int64) error {
	var f *os.File
	var err error

	fileName := d.metaDataFileName(typ)
	tmpFileName := fileName + ".tmp"

	// write to tmp file
	f, err = os.OpenFile(tmpFileName, os.O_RDWR|os.O_CREATE, 0600)
	if err != nil {
		return err
	}

	_, err = fmt.Fprintf(f, "%d,%d\n", fileNum, pos)
	if err != nil {
		f.Close()
		return err
	}
	f.Sync()
	f.Close()

	// atomically rename
	return os.Rename(tmpFileName, fileName)
}

func (d *DiskQueue) metaDataFileName(typ string) string {
	return fmt.Sprintf(path.Join(d.dataPath, "%s.diskqueue.%smeta.dat"), d.name, typ)
}

func (d *DiskQueue) fileName(fileNum int64) string {
	return fmt.Sprintf(path.Join(d.dataPath, "%s.diskqueue.%06d.dat"), d.name, fileNum)
}

func (d *DiskQueue) hasDataToRead() bool {
	return (d.readFileNum < d.writeFileNum) || (d.readPos < d.writePos)
}

// read the next message off disk to prime ReadChan
func (d *DiskQueue) readAheadPump() {
	var data []byte
	var err error
	for {
		if d.hasDataToRead() {
			if d.nextReadPos == d.readPos {
				data, err = d.readOne()
				if err != nil {
					// TODO: should this be fatal?
					log.Printf("ERROR: reading from diskqueue(%s) at %d of %s - %s", d.name, d.readPos, d.fileName(d.readFileNum), err.Error())
					continue
				}
			}
			select {
			case d.readChan <- data:
				d.readPos = d.nextReadPos
			case <-d.writeContinueChan:
			case <-d.exitChan:
				return
			}
		} else {
			select {
			case <-d.writeContinueChan:
			case <-d.exitChan:
				return
			}
		}
	}
}
<|endoftext|>


<commit_before>package cucumberexpressions

import "fmt"

type Argument struct {
	group         *Group
	parameterType *ParameterType
}

func BuildArguments(treeRegexp *TreeRegexp, text string, parameterTypes []*ParameterType) []*Argument {
	group := treeRegexp.Match(text)
	if group == nil {
		return nil
	}
	argGroups := group.Children()
	if len(argGroups) != len(parameterTypes) {
		panic(fmt.Errorf("%s has %d capture groups (%v), but there were %d parameter types (%v)", treeRegexp.Regexp().String(), len(argGroups), argGroups, len(parameterTypes), parameterTypes))
	}
	arguments := make([]*Argument, len(parameterTypes))
	for i, parameterType := range parameterTypes {
		arguments[i] = NewArgument(argGroups[i], parameterType)
	}
	return arguments
}

func NewArgument(group *Group, parameterType *ParameterType) *Argument {
	return &Argument{
		group:         group,
		parameterType: parameterType,
	}
}

func (a *Argument) Group() *Group {
	return a.group
}

func (a *Argument) GetValue() interface{} {
	return a.parameterType.Transform(a.group.Values())
}
<commit_msg>update<commit_after>package cucumberexpressions

import "fmt"

type Argument struct {
	group         *Group
	parameterType *ParameterType
}

func BuildArguments(treeRegexp *TreeRegexp, text string, parameterTypes []*ParameterType) []*Argument {
	group := treeRegexp.Match(text)
	if group == nil {
		return nil
	}
	argGroups := group.Children()
	if len(argGroups) != len(parameterTypes) {
		panic(fmt.Errorf("%s has %d capture groups (%v), but there were %d parameter types (%v)", treeRegexp.Regexp().String(), len(argGroups), argGroups, len(parameterTypes), parameterTypes))
	}
	arguments := make([]*Argument, len(parameterTypes))
	for i, parameterType := range parameterTypes {
		arguments[i] = NewArgument(argGroups[i], parameterType)
	}
	return arguments
}

func NewArgument(group *Group, parameterType *ParameterType) *Argument {
	return &Argument{
		group:         group,
		parameterType: parameterType,
	}
}

func (a *Argument) Group() *Group {
	return a.group
}

func (a *Argument) GetValue() interface{} {
	return a.parameterType.Transform(a.group.Values())
}

func (a *Argument) ParameterType() *ParameterType {
	return a.parameterType
}
<|endoftext|>


<commit_before>package nyb

import (
	"strings"
	"testing"
)

func TestChangeNick(t *testing.T) {
	ln := "h123456789123456"
	sn := changeNick(ln)
	if len(sn) != 13 {
		t.Errorf("Expecting 13 chars got %d chars", len(sn))
	}
	if !strings.HasSuffix(sn, "_") {
		t.Errorf("%s dosen't have _ suffix", sn)
	}
	sn = changeNick(sn)
	if !strings.HasSuffix(sn, "__") {
		t.Errorf("didn't get __ suffix")
	}
	sn = changeNick(sn)
	if !strings.HasSuffix(sn, "___") {
		t.Errorf("didn't get ___ suffix")
	}
	sn = changeNick(sn)
	if !strings.HasSuffix(sn, "____") {
		t.Errorf("didn't get ____ suffix")
	}
	if len(sn) != 16 {
		t.Error("sn not 16 chars")
	}
	sn = changeNick(sn)
	if sn != ln[:12] {
		t.Errorf("didn't get %s, got %s", ln[:12], sn)
	}
	n := "a"
	for i := 1; i <= 15; i++ {
		n = changeNick(n)
		c := strings.Count(n, "_")
		if c != i {
			t.Errorf("expecting %d _'s got %d: string: %s", i, c, n)
		}
	}

}
<commit_msg>proper nick change test<commit_after>package nyb

import (
	"strings"
	"testing"
)

func TestChangeNick(t *testing.T) {
	ln := "aaaaaaaaaaaaaaaa"
	for i := 1; i <= 4; i++ {
		ln = changeNick(ln)
		if got := strings.Count(ln, "_"); got != i {
			t.Errorf("expecting %d _'s got %d ", i, got)
		}
		if len(ln) != 12+i {
			t.Errorf("expecting lenght %d got %d", 12+i, len(ln))
		}
	}
	if ln := changeNick(ln); len(ln) != 12 {
		t.Errorf("expecting lenght 12 got %d", len(ln))
	}
	n := "a"
	for i := 1; i <= 15; i++ {
		n = changeNick(n)
		c := strings.Count(n, "_")
		if c != i {
			t.Errorf("expecting %d _'s got %d: string: %s", i, c, n)
		}
	}
	if n := changeNick(n); n != "a" {
		t.Error("n wasn't 'a'")
	}
}
<|endoftext|>


<commit_before>package ibclient

import (
	"encoding/json"
	"fmt"
	"regexp"
)

type IBObjectManager interface {
	CreateNetworkView(name string) (*NetworkView, error)
	CreateDefaultNetviews(globalNetview string, localNetview string) (globalNetviewRef string, localNetviewRef string, err error)
	CreateNetwork(netview string, cidr string, name string) (*Network, error)
	CreateNetworkContainer(netview string, cidr string) (*NetworkContainer, error)
	GetNetworkView(name string) (*NetworkView, error)
	GetNetwork(netview string, cidr string, ea EA) (*Network, error)
	GetNetworkContainer(netview string, cidr string) (*NetworkContainer, error)
	AllocateIP(netview string, cidr string, ipAddr string, macAddress string, vmID string) (*FixedAddress, error)
	AllocateNetwork(netview string, cidr string, prefixLen uint, name string) (network *Network, err error)
	GetFixedAddress(netview string, cidr string, ipAddr string, macAddr string) (*FixedAddress, error)
	ReleaseIP(netview string, cidr string, ipAddr string, macAddr string) (string, error)
	DeleteNetwork(ref string, netview string) (string, error)
	GetEADefinition(name string) (*EADefinition, error)
	CreateEADefinition(eadef EADefinition) (*EADefinition, error)
}

type ObjectManager struct {
	connector IBConnector
	cmpType   string
	tenantID  string
}

func NewObjectManager(connector IBConnector, cmpType string, tenantID string) *ObjectManager {
	objMgr := new(ObjectManager)

	objMgr.connector = connector
	objMgr.cmpType = cmpType
	objMgr.tenantID = tenantID

	return objMgr
}

func (objMgr *ObjectManager) getBasicEA(cloudApiOwned Bool) EA {
	ea := make(EA)
	ea["Cloud API Owned"] = cloudApiOwned
	ea["CMP Type"] = objMgr.cmpType
	ea["Tenant ID"] = objMgr.tenantID
	return ea
}

func (objMgr *ObjectManager) CreateNetworkView(name string) (*NetworkView, error) {
	networkView := NewNetworkView(NetworkView{
		Name: name,
		Ea:   objMgr.getBasicEA(false)})

	ref, err := objMgr.connector.CreateObject(networkView)
	networkView.Ref = ref

	return networkView, err
}

func (objMgr *ObjectManager) makeNetworkView(netviewName string) (netviewRef string, err error) {
	var netviewObj *NetworkView
	if netviewObj, err = objMgr.GetNetworkView(netviewName); err != nil {
		return
	}
	if netviewObj == nil {
		if netviewObj, err = objMgr.CreateNetworkView(netviewName); err != nil {
			return
		}
	}

	netviewRef = netviewObj.Ref

	return
}

func (objMgr *ObjectManager) CreateDefaultNetviews(globalNetview string, localNetview string) (globalNetviewRef string, localNetviewRef string, err error) {
	if globalNetviewRef, err = objMgr.makeNetworkView(globalNetview); err != nil {
		return
	}

	if localNetviewRef, err = objMgr.makeNetworkView(localNetview); err != nil {
		return
	}

	return
}

func (objMgr *ObjectManager) CreateNetwork(netview string, cidr string, name string) (*Network, error) {
	network := NewNetwork(Network{
		NetviewName: netview,
		Cidr:        cidr,
		Ea:          objMgr.getBasicEA(true)})

	if name != "" {
		network.Ea["Network Name"] = name
	}
	ref, err := objMgr.connector.CreateObject(network)
	if err != nil {
		return nil, err
	}
	network.Ref = ref

	return network, err
}

func (objMgr *ObjectManager) CreateNetworkContainer(netview string, cidr string) (*NetworkContainer, error) {
	container := NewNetworkContainer(NetworkContainer{
		NetviewName: netview,
		Cidr:        cidr,
		Ea:          objMgr.getBasicEA(true)})

	ref, err := objMgr.connector.CreateObject(container)
	container.Ref = ref

	return container, err
}

func (objMgr *ObjectManager) GetNetworkView(name string) (*NetworkView, error) {
	var res []NetworkView

	netview := NewNetworkView(NetworkView{Name: name})

	err := objMgr.connector.GetObject(netview, "", &res)

	if err != nil || res == nil || len(res) == 0 {
		return nil, err
	}

	return &res[0], nil
}

func (objMgr *ObjectManager) UpdateNetworkViewEA(ref string, addEA EA, removeEA EA) error {
	var res NetworkView

	nv := NetworkView{}
	nv.returnFields = []string{"extattrs"}
	err := objMgr.connector.GetObject(&nv, ref, &res)

	if err != nil {
		return err
	}

	for k, v := range addEA {
		res.Ea[k] = v
	}

	for k, _ := range removeEA {
		_, ok := res.Ea[k]
		if ok {
			delete(res.Ea, k)
		}
	}

	_, err = objMgr.connector.UpdateObject(&res, ref)
	return err
}

func BuildNetworkViewFromRef(ref string) *NetworkView {
	// networkview/ZG5zLm5ldHdvcmtfdmlldyQyMw:global_view/false
	r := regexp.MustCompile(`networkview/\w+:([^/]+)/\w+`)
	m := r.FindStringSubmatch(ref)

	if m == nil {
		return nil
	}

	return &NetworkView{
		Ref:  ref,
		Name: m[1],
	}
}

func BuildNetworkFromRef(ref string) *Network {
	// network/ZG5zLm5ldHdvcmskODkuMC4wLjAvMjQvMjU:89.0.0.0/24/global_view
	r := regexp.MustCompile(`network/\w+:(\d+\.\d+\.\d+\.\d+/\d+)/(.+)`)
	m := r.FindStringSubmatch(ref)

	if m == nil {
		return nil
	}

	return &Network{
		Ref:         ref,
		NetviewName: m[2],
		Cidr:        m[1],
	}
}

func (objMgr *ObjectManager) GetNetwork(netview string, cidr string, ea EA) (*Network, error) {
	var res []Network

	network := NewNetwork(Network{
		NetviewName: netview})

	if cidr != "" {
		network.Cidr = cidr
	}

	if ea != nil && len(ea) > 0 {
		network.eaSearch = EASearch(ea)
	}

	err := objMgr.connector.GetObject(network, "", &res)

	if err != nil || res == nil || len(res) == 0 {
		return nil, err
	}

	return &res[0], nil
}

func (objMgr *ObjectManager) GetNetworkContainer(netview string, cidr string) (*NetworkContainer, error) {
	var res []NetworkContainer

	nwcontainer := NewNetworkContainer(NetworkContainer{
		NetviewName: netview,
		Cidr:        cidr})

	err := objMgr.connector.GetObject(nwcontainer, "", &res)

	if err != nil || res == nil || len(res) == 0 {
		return nil, err
	}

	return &res[0], nil
}

func GetIPAddressFromRef(ref string) string {
	// fixedaddress/ZG5zLmJpbmRfY25h:12.0.10.1/external
	r := regexp.MustCompile(`fixedaddress/\w+:(\d+\.\d+\.\d+\.\d+)/.+`)
	m := r.FindStringSubmatch(ref)

	if m != nil {
		return m[1]
	}
	return ""
}

func (objMgr *ObjectManager) AllocateIP(netview string, cidr string, ipAddr string, macAddress string, vmID string) (*FixedAddress, error) {
	if len(macAddress) == 0 {
		macAddress = MACADDR_ZERO
	}

	ea := objMgr.getBasicEA(true)
	ea["VM ID"] = "N/A"
	if vmID != "" {
		ea["VM ID"] = vmID
	}

	fixedAddr := NewFixedAddress(FixedAddress{
		NetviewName: netview,
		Cidr:        cidr,
		Mac:         macAddress,
		Ea:          ea})

	if ipAddr == "" {
		fixedAddr.IPAddress = fmt.Sprintf("func:nextavailableip:%s,%s", cidr, netview)
	} else {
		fixedAddr.IPAddress = ipAddr
	}

	ref, err := objMgr.connector.CreateObject(fixedAddr)
	fixedAddr.Ref = ref
	fixedAddr.IPAddress = GetIPAddressFromRef(ref)

	return fixedAddr, err
}

func (objMgr *ObjectManager) AllocateNetwork(netview string, cidr string, prefixLen uint, name string) (network *Network, err error) {
	network = nil

	networkReq := NewNetwork(Network{
		NetviewName: netview,
		Cidr:        fmt.Sprintf("func:nextavailablenetwork:%s,%s,%d", cidr, netview, prefixLen),
		Ea:          objMgr.getBasicEA(true)})
	if name != "" {
		networkReq.Ea["Network Name"] = name
	}

	ref, err := objMgr.connector.CreateObject(networkReq)
	if err == nil && len(ref) > 0 {
		network = BuildNetworkFromRef(ref)
	}

	return
}

func (objMgr *ObjectManager) GetFixedAddress(netview string, cidr string, ipAddr string, macAddr string) (*FixedAddress, error) {
	var res []FixedAddress

	fixedAddr := NewFixedAddress(FixedAddress{
		NetviewName: netview,
		Cidr:        cidr,
		IPAddress:   ipAddr,
		Mac:         macAddr})

	err := objMgr.connector.GetObject(fixedAddr, "", &res)

	if err != nil || res == nil || len(res) == 0 {
		return nil, err
	}

	return &res[0], nil
}

func (objMgr *ObjectManager) ReleaseIP(netview string, cidr string, ipAddr string, macAddr string) (string, error) {
	fixAddress, _ := objMgr.GetFixedAddress(netview, cidr, ipAddr, macAddr)
	if fixAddress == nil {
		return "", nil
	}
	return objMgr.connector.DeleteObject(fixAddress.Ref)
}

func (objMgr *ObjectManager) DeleteNetwork(ref string, netview string) (string, error) {
	network := BuildNetworkFromRef(ref)
	if network != nil && network.NetviewName == netview {
		return objMgr.connector.DeleteObject(ref)
	}

	return "", nil
}

func (objMgr *ObjectManager) GetEADefinition(name string) (*EADefinition, error) {
	var res []EADefinition

	eadef := NewEADefinition(EADefinition{Name: name})

	err := objMgr.connector.GetObject(eadef, "", &res)

	if err != nil || res == nil || len(res) == 0 {
		return nil, err
	}

	return &res[0], nil
}

func (objMgr *ObjectManager) CreateEADefinition(eadef EADefinition) (*EADefinition, error) {
	newEadef := NewEADefinition(eadef)

	ref, err := objMgr.connector.CreateObject(newEadef)
	newEadef.Ref = ref

	return newEadef, err
}

func (objMgr *ObjectManager) CreateMultiObject(req *MultiRequest) ([]map[string]interface{}, error) {

	conn := objMgr.connector.(*Connector)

	res, err := conn.makeRequest(CREATE, req, "")

	if err != nil {
		return nil, err
	}

	var result []map[string]interface{}
	err = json.Unmarshal(res, &result)

	if err != nil {
		return nil, err
	}

	return result, nil
}
<commit_msg>Added support for UpdateFixedAddress in client<commit_after>package ibclient

import (
	"encoding/json"
	"fmt"
	"regexp"
)

type IBObjectManager interface {
	CreateNetworkView(name string) (*NetworkView, error)
	CreateDefaultNetviews(globalNetview string, localNetview string) (globalNetviewRef string, localNetviewRef string, err error)
	CreateNetwork(netview string, cidr string, name string) (*Network, error)
	CreateNetworkContainer(netview string, cidr string) (*NetworkContainer, error)
	GetNetworkView(name string) (*NetworkView, error)
	GetNetwork(netview string, cidr string, ea EA) (*Network, error)
	GetNetworkContainer(netview string, cidr string) (*NetworkContainer, error)
	AllocateIP(netview string, cidr string, ipAddr string, macAddress string, vmID string) (*FixedAddress, error)
	AllocateNetwork(netview string, cidr string, prefixLen uint, name string) (network *Network, err error)
	UpdateFixedAddress(netview string, cidr string, ipAddr string, macAddress string, vmID string) (*FixedAddress, error)
	GetFixedAddress(netview string, cidr string, ipAddr string, macAddr string) (*FixedAddress, error)
	ReleaseIP(netview string, cidr string, ipAddr string, macAddr string) (string, error)
	DeleteNetwork(ref string, netview string) (string, error)
	GetEADefinition(name string) (*EADefinition, error)
	CreateEADefinition(eadef EADefinition) (*EADefinition, error)
}

type ObjectManager struct {
	connector IBConnector
	cmpType   string
	tenantID  string
}

func NewObjectManager(connector IBConnector, cmpType string, tenantID string) *ObjectManager {
	objMgr := new(ObjectManager)

	objMgr.connector = connector
	objMgr.cmpType = cmpType
	objMgr.tenantID = tenantID

	return objMgr
}

func (objMgr *ObjectManager) getBasicEA(cloudApiOwned Bool) EA {
	ea := make(EA)
	ea["Cloud API Owned"] = cloudApiOwned
	ea["CMP Type"] = objMgr.cmpType
	ea["Tenant ID"] = objMgr.tenantID
	return ea
}

func (objMgr *ObjectManager) CreateNetworkView(name string) (*NetworkView, error) {
	networkView := NewNetworkView(NetworkView{
		Name: name,
		Ea:   objMgr.getBasicEA(false)})

	ref, err := objMgr.connector.CreateObject(networkView)
	networkView.Ref = ref

	return networkView, err
}

func (objMgr *ObjectManager) makeNetworkView(netviewName string) (netviewRef string, err error) {
	var netviewObj *NetworkView
	if netviewObj, err = objMgr.GetNetworkView(netviewName); err != nil {
		return
	}
	if netviewObj == nil {
		if netviewObj, err = objMgr.CreateNetworkView(netviewName); err != nil {
			return
		}
	}

	netviewRef = netviewObj.Ref

	return
}

func (objMgr *ObjectManager) CreateDefaultNetviews(globalNetview string, localNetview string) (globalNetviewRef string, localNetviewRef string, err error) {
	if globalNetviewRef, err = objMgr.makeNetworkView(globalNetview); err != nil {
		return
	}

	if localNetviewRef, err = objMgr.makeNetworkView(localNetview); err != nil {
		return
	}

	return
}

func (objMgr *ObjectManager) CreateNetwork(netview string, cidr string, name string) (*Network, error) {
	network := NewNetwork(Network{
		NetviewName: netview,
		Cidr:        cidr,
		Ea:          objMgr.getBasicEA(true)})

	if name != "" {
		network.Ea["Network Name"] = name
	}
	ref, err := objMgr.connector.CreateObject(network)
	if err != nil {
		return nil, err
	}
	network.Ref = ref

	return network, err
}

func (objMgr *ObjectManager) CreateNetworkContainer(netview string, cidr string) (*NetworkContainer, error) {
	container := NewNetworkContainer(NetworkContainer{
		NetviewName: netview,
		Cidr:        cidr,
		Ea:          objMgr.getBasicEA(true)})

	ref, err := objMgr.connector.CreateObject(container)
	container.Ref = ref

	return container, err
}

func (objMgr *ObjectManager) GetNetworkView(name string) (*NetworkView, error) {
	var res []NetworkView

	netview := NewNetworkView(NetworkView{Name: name})

	err := objMgr.connector.GetObject(netview, "", &res)

	if err != nil || res == nil || len(res) == 0 {
		return nil, err
	}

	return &res[0], nil
}

func (objMgr *ObjectManager) UpdateNetworkViewEA(ref string, addEA EA, removeEA EA) error {
	var res NetworkView

	nv := NetworkView{}
	nv.returnFields = []string{"extattrs"}
	err := objMgr.connector.GetObject(&nv, ref, &res)

	if err != nil {
		return err
	}

	for k, v := range addEA {
		res.Ea[k] = v
	}

	for k, _ := range removeEA {
		_, ok := res.Ea[k]
		if ok {
			delete(res.Ea, k)
		}
	}

	_, err = objMgr.connector.UpdateObject(&res, ref)
	return err
}

func BuildNetworkViewFromRef(ref string) *NetworkView {
	// networkview/ZG5zLm5ldHdvcmtfdmlldyQyMw:global_view/false
	r := regexp.MustCompile(`networkview/\w+:([^/]+)/\w+`)
	m := r.FindStringSubmatch(ref)

	if m == nil {
		return nil
	}

	return &NetworkView{
		Ref:  ref,
		Name: m[1],
	}
}

func BuildNetworkFromRef(ref string) *Network {
	// network/ZG5zLm5ldHdvcmskODkuMC4wLjAvMjQvMjU:89.0.0.0/24/global_view
	r := regexp.MustCompile(`network/\w+:(\d+\.\d+\.\d+\.\d+/\d+)/(.+)`)
	m := r.FindStringSubmatch(ref)

	if m == nil {
		return nil
	}

	return &Network{
		Ref:         ref,
		NetviewName: m[2],
		Cidr:        m[1],
	}
}

func (objMgr *ObjectManager) GetNetwork(netview string, cidr string, ea EA) (*Network, error) {
	var res []Network

	network := NewNetwork(Network{
		NetviewName: netview})

	if cidr != "" {
		network.Cidr = cidr
	}

	if ea != nil && len(ea) > 0 {
		network.eaSearch = EASearch(ea)
	}

	err := objMgr.connector.GetObject(network, "", &res)

	if err != nil || res == nil || len(res) == 0 {
		return nil, err
	}

	return &res[0], nil
}

func (objMgr *ObjectManager) GetNetworkContainer(netview string, cidr string) (*NetworkContainer, error) {
	var res []NetworkContainer

	nwcontainer := NewNetworkContainer(NetworkContainer{
		NetviewName: netview,
		Cidr:        cidr})

	err := objMgr.connector.GetObject(nwcontainer, "", &res)

	if err != nil || res == nil || len(res) == 0 {
		return nil, err
	}

	return &res[0], nil
}

func GetIPAddressFromRef(ref string) string {
	// fixedaddress/ZG5zLmJpbmRfY25h:12.0.10.1/external
	r := regexp.MustCompile(`fixedaddress/\w+:(\d+\.\d+\.\d+\.\d+)/.+`)
	m := r.FindStringSubmatch(ref)

	if m != nil {
		return m[1]
	}
	return ""
}

func (objMgr *ObjectManager) AllocateIP(netview string, cidr string, ipAddr string, macAddress string, vmID string) (*FixedAddress, error) {
	if len(macAddress) == 0 {
		macAddress = MACADDR_ZERO
	}

	ea := objMgr.getBasicEA(true)
	ea["VM ID"] = "N/A"
	if vmID != "" {
		ea["VM ID"] = vmID
	}

	fixedAddr := NewFixedAddress(FixedAddress{
		NetviewName: netview,
		Cidr:        cidr,
		Mac:         macAddress,
		Ea:          ea})

	if ipAddr == "" {
		fixedAddr.IPAddress = fmt.Sprintf("func:nextavailableip:%s,%s", cidr, netview)
	} else {
		fixedAddr.IPAddress = ipAddr
	}

	ref, err := objMgr.connector.CreateObject(fixedAddr)
	fixedAddr.Ref = ref
	fixedAddr.IPAddress = GetIPAddressFromRef(ref)

	return fixedAddr, err
}

func (objMgr *ObjectManager) AllocateNetwork(netview string, cidr string, prefixLen uint, name string) (network *Network, err error) {
	network = nil

	networkReq := NewNetwork(Network{
		NetviewName: netview,
		Cidr:        fmt.Sprintf("func:nextavailablenetwork:%s,%s,%d", cidr, netview, prefixLen),
		Ea:          objMgr.getBasicEA(true)})
	if name != "" {
		networkReq.Ea["Network Name"] = name
	}

	ref, err := objMgr.connector.CreateObject(networkReq)
	if err == nil && len(ref) > 0 {
		network = BuildNetworkFromRef(ref)
	}

	return
}

func (objMgr *ObjectManager) GetFixedAddress(netview string, cidr string, ipAddr string, macAddr string) (*FixedAddress, error) {
	var res []FixedAddress

	fixedAddr := NewFixedAddress(FixedAddress{
		NetviewName: netview,
		Cidr:        cidr,
		IPAddress:   ipAddr,
		Mac:         macAddr})

	err := objMgr.connector.GetObject(fixedAddr, "", &res)

	if err != nil || res == nil || len(res) == 0 {
		return nil, err
	}

	return &res[0], nil
}

func (objMgr *ObjectManager) UpdateFixedAddress(netview string, cidr string, ipAddr string, macAddress string, vmID string) (*FixedAddress, error) {

        var res []FixedAddress
        // Update is based on IP, CIDR and NETWORK VIEW
        fixedAddr := NewFixedAddress(FixedAddress{
                NetviewName: netview,
                Cidr:        cidr,
                IPAddress:   ipAddr})

        err := objMgr.connector.GetObject(fixedAddr, "", &res)
        if err != nil || res == nil || len(res) == 0 {
                return nil, err
        }
        _fixedAddr := &res[0]

	ea := objMgr.getBasicEA(true)
	ea["VM ID"] = "N/A"
	if vmID != "" {
		ea["VM ID"] = vmID
                _fixedAddr.Ea = ea
	}

	if len(macAddress) == 0 {
		_fixedAddr.Mac = MACADDR_ZERO
	} else {
                _fixedAddr.Mac = macAddress
        }

	refResp, err := objMgr.connector.UpdateObject(_fixedAddr, _fixedAddr.Ref)
        _fixedAddr.Ref = refResp
	return _fixedAddr, err
}

func (objMgr *ObjectManager) ReleaseIP(netview string, cidr string, ipAddr string, macAddr string) (string, error) {
	fixAddress, _ := objMgr.GetFixedAddress(netview, cidr, ipAddr, macAddr)
	if fixAddress == nil {
		return "", nil
	}
	return objMgr.connector.DeleteObject(fixAddress.Ref)
}

func (objMgr *ObjectManager) DeleteNetwork(ref string, netview string) (string, error) {
	network := BuildNetworkFromRef(ref)
	if network != nil && network.NetviewName == netview {
		return objMgr.connector.DeleteObject(ref)
	}

	return "", nil
}

func (objMgr *ObjectManager) GetEADefinition(name string) (*EADefinition, error) {
	var res []EADefinition

	eadef := NewEADefinition(EADefinition{Name: name})

	err := objMgr.connector.GetObject(eadef, "", &res)

	if err != nil || res == nil || len(res) == 0 {
		return nil, err
	}

	return &res[0], nil
}

func (objMgr *ObjectManager) CreateEADefinition(eadef EADefinition) (*EADefinition, error) {
	newEadef := NewEADefinition(eadef)

	ref, err := objMgr.connector.CreateObject(newEadef)
	newEadef.Ref = ref

	return newEadef, err
}

func (objMgr *ObjectManager) CreateMultiObject(req *MultiRequest) ([]map[string]interface{}, error) {

	conn := objMgr.connector.(*Connector)

	res, err := conn.makeRequest(CREATE, req, "")

	if err != nil {
		return nil, err
	}

	var result []map[string]interface{}
	err = json.Unmarshal(res, &result)

	if err != nil {
		return nil, err
	}

	return result, nil
}
<|endoftext|>


<commit_before>// Package ocsp implements an OCSP responder based on a generic storage backend.
// It provides a couple of sample implementations.
// Because OCSP responders handle high query volumes, we have to be careful
// about how much logging we do. Error-level logs are reserved for problems
// internal to the server, that can be fixed by an administrator. Any type of
// incorrect input from a user should be logged and Info or below. For things
// that are logged on every request, Debug is the appropriate level.
package ocsp

import (
	"crypto/sha256"
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"time"

	"github.com/cloudflare/cfssl/log"
	"github.com/jmhodges/clock"
	"golang.org/x/crypto/ocsp"
)

var (
	malformedRequestErrorResponse = []byte{0x30, 0x03, 0x0A, 0x01, 0x01}
	internalErrorErrorResponse    = []byte{0x30, 0x03, 0x0A, 0x01, 0x02}
	tryLaterErrorResponse         = []byte{0x30, 0x03, 0x0A, 0x01, 0x03}
	sigRequredErrorResponse       = []byte{0x30, 0x03, 0x0A, 0x01, 0x05}
	unauthorizedErrorResponse     = []byte{0x30, 0x03, 0x0A, 0x01, 0x06}
)

// Source represents the logical source of OCSP responses, i.e.,
// the logic that actually chooses a response based on a request.  In
// order to create an actual responder, wrap one of these in a Responder
// object and pass it to http.Handle.
type Source interface {
	Response(*ocsp.Request) ([]byte, bool)
}

// An InMemorySource is a map from serialNumber -> der(response)
type InMemorySource map[string][]byte

// Response looks up an OCSP response to provide for a given request.
// InMemorySource looks up a response purely based on serial number,
// without regard to what issuer the request is asking for.
func (src InMemorySource) Response(request *ocsp.Request) (response []byte, present bool) {
	response, present = src[request.SerialNumber.String()]
	return
}

// NewSourceFromFile reads the named file into an InMemorySource.
// The file read by this function must contain whitespace-separated OCSP
// responses. Each OCSP response must be in base64-encoded DER form (i.e.,
// PEM without headers or whitespace).  Invalid responses are ignored.
// This function pulls the entire file into an InMemorySource.
func NewSourceFromFile(responseFile string) (Source, error) {
	fileContents, err := ioutil.ReadFile(responseFile)
	if err != nil {
		return nil, err
	}

	responsesB64 := regexp.MustCompile("\\s").Split(string(fileContents), -1)
	src := InMemorySource{}
	for _, b64 := range responsesB64 {
		// if the line/space is empty just skip
		if b64 == "" {
			continue
		}
		der, tmpErr := base64.StdEncoding.DecodeString(b64)
		if tmpErr != nil {
			log.Errorf("Base64 decode error %s on: %s", tmpErr, b64)
			continue
		}

		response, tmpErr := ocsp.ParseResponse(der, nil)
		if tmpErr != nil {
			log.Errorf("OCSP decode error %s on: %s", tmpErr, b64)
			continue
		}

		src[response.SerialNumber.String()] = der
	}

	log.Infof("Read %d OCSP responses", len(src))
	return src, nil
}

// A Responder object provides the HTTP logic to expose a
// Source of OCSP responses.
type Responder struct {
	Source Source
	clk    clock.Clock
}

// NewResponder instantiates a Responder with the give Source.
func NewResponder(source Source) *Responder {
	return &Responder{
		Source: source,
		clk:    clock.Default(),
	}
}

// A Responder can process both GET and POST requests.  The mapping
// from an OCSP request to an OCSP response is done by the Source;
// the Responder simply decodes the request, and passes back whatever
// response is provided by the source.
// Note: The caller must use http.StripPrefix to strip any path components
// (including '/') on GET requests.
// Do not use this responder in conjunction with http.NewServeMux, because the
// default handler will try to canonicalize path components by changing any
// strings of repeated '/' into a single '/', which will break the base64
// encoding.
func (rs Responder) ServeHTTP(response http.ResponseWriter, request *http.Request) {
	// By default we set a 'max-age=0, no-cache' Cache-Control header, this
	// is only returned to the client if a valid authorized OCSP response
	// is not found or an error is returned. If a response if found the header
	// will be altered to contain the proper max-age and modifiers.
	response.Header().Add("Cache-Control", "max-age=0, no-cache")
	// Read response from request
	var requestBody []byte
	var err error
	switch request.Method {
	case "GET":
		base64Request, err := url.QueryUnescape(request.URL.Path)
		if err != nil {
			log.Infof("Error decoding URL: %s", request.URL.Path)
			response.WriteHeader(http.StatusBadRequest)
			return
		}
		// url.QueryUnescape not only unescapes %2B escaping, but it additionally
		// turns the resulting '+' into a space, which makes base64 decoding fail.
		// So we go back afterwards and turn ' ' back into '+'. This means we
		// accept some malformed input that includes ' ' or %20, but that's fine.
		base64RequestBytes := []byte(base64Request)
		for i := range base64RequestBytes {
			if base64RequestBytes[i] == ' ' {
				base64RequestBytes[i] = '+'
			}
		}
		requestBody, err = base64.StdEncoding.DecodeString(string(base64RequestBytes))
		if err != nil {
			log.Infof("Error decoding base64 from URL: %s", base64Request)
			response.WriteHeader(http.StatusBadRequest)
			return
		}
	case "POST":
		requestBody, err = ioutil.ReadAll(request.Body)
		if err != nil {
			log.Errorf("Problem reading body of POST: %s", err)
			response.WriteHeader(http.StatusBadRequest)
			return
		}
	default:
		response.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	b64Body := base64.StdEncoding.EncodeToString(requestBody)
	log.Debugf("Received OCSP request: %s", b64Body)

	// All responses after this point will be OCSP.
	// We could check for the content type of the request, but that
	// seems unnecessariliy restrictive.
	response.Header().Add("Content-Type", "application/ocsp-response")

	// Parse response as an OCSP request
	// XXX: This fails if the request contains the nonce extension.
	//      We don't intend to support nonces anyway, but maybe we
	//      should return unauthorizedRequest instead of malformed.
	ocspRequest, err := ocsp.ParseRequest(requestBody)
	if err != nil {
		log.Infof("Error decoding request body: %s", b64Body)
		response.WriteHeader(http.StatusBadRequest)
		response.Write(malformedRequestErrorResponse)
		return
	}

	// Look up OCSP response from source
	ocspResponse, found := rs.Source.Response(ocspRequest)
	if !found {
		log.Infof("No response found for request: %s", b64Body)
		response.Write(unauthorizedErrorResponse)
		return
	}

	parsedResponse, err := ocsp.ParseResponse(ocspResponse, nil)
	if err != nil {
		log.Errorf("Error parsing response: %s", err)
		response.Write(unauthorizedErrorResponse)
		return
	}

	// Write OCSP response to response
	response.Header().Add("Last-Modified", parsedResponse.ThisUpdate.Format(time.RFC1123))
	response.Header().Add("Expires", parsedResponse.NextUpdate.Format(time.RFC1123))
	now := rs.clk.Now()
	maxAge := 0
	if now.Before(parsedResponse.NextUpdate) {
		maxAge = int(parsedResponse.NextUpdate.Sub(now) / time.Second)
	} else {
		// TODO(#530): we want max-age=0 but this is technically an authorized OCSP response
		//             (despite being stale) and 5019 forbids attaching no-cache
		maxAge = 0
	}
	response.Header().Set(
		"Cache-Control",
		fmt.Sprintf(
			"max-age=%d, public, no-transform, must-revalidate",
			maxAge,
		),
	)
	responseHash := sha256.Sum256(ocspResponse)
	response.Header().Add("ETag", fmt.Sprintf("\"%X\"", responseHash))

	// RFC 7232 says that a 304 response must contain the above
	// headers if they would also be sent for a 200 for the same
	// request, so we have to wait until here to do this
	if etag := request.Header.Get("If-None-Match"); etag != "" {
		if etag == fmt.Sprintf("\"%X\"", responseHash) {
			response.WriteHeader(http.StatusNotModified)
			return
		}
	}
	response.WriteHeader(http.StatusOK)
	response.Write(ocspResponse)
}
<commit_msg>Add logging of request serial to OCSP responder. (#662)<commit_after>// Package ocsp implements an OCSP responder based on a generic storage backend.
// It provides a couple of sample implementations.
// Because OCSP responders handle high query volumes, we have to be careful
// about how much logging we do. Error-level logs are reserved for problems
// internal to the server, that can be fixed by an administrator. Any type of
// incorrect input from a user should be logged and Info or below. For things
// that are logged on every request, Debug is the appropriate level.
package ocsp

import (
	"crypto/sha256"
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"time"

	"github.com/cloudflare/cfssl/log"
	"github.com/jmhodges/clock"
	"golang.org/x/crypto/ocsp"
)

var (
	malformedRequestErrorResponse = []byte{0x30, 0x03, 0x0A, 0x01, 0x01}
	internalErrorErrorResponse    = []byte{0x30, 0x03, 0x0A, 0x01, 0x02}
	tryLaterErrorResponse         = []byte{0x30, 0x03, 0x0A, 0x01, 0x03}
	sigRequredErrorResponse       = []byte{0x30, 0x03, 0x0A, 0x01, 0x05}
	unauthorizedErrorResponse     = []byte{0x30, 0x03, 0x0A, 0x01, 0x06}
)

// Source represents the logical source of OCSP responses, i.e.,
// the logic that actually chooses a response based on a request.  In
// order to create an actual responder, wrap one of these in a Responder
// object and pass it to http.Handle.
type Source interface {
	Response(*ocsp.Request) ([]byte, bool)
}

// An InMemorySource is a map from serialNumber -> der(response)
type InMemorySource map[string][]byte

// Response looks up an OCSP response to provide for a given request.
// InMemorySource looks up a response purely based on serial number,
// without regard to what issuer the request is asking for.
func (src InMemorySource) Response(request *ocsp.Request) (response []byte, present bool) {
	response, present = src[request.SerialNumber.String()]
	return
}

// NewSourceFromFile reads the named file into an InMemorySource.
// The file read by this function must contain whitespace-separated OCSP
// responses. Each OCSP response must be in base64-encoded DER form (i.e.,
// PEM without headers or whitespace).  Invalid responses are ignored.
// This function pulls the entire file into an InMemorySource.
func NewSourceFromFile(responseFile string) (Source, error) {
	fileContents, err := ioutil.ReadFile(responseFile)
	if err != nil {
		return nil, err
	}

	responsesB64 := regexp.MustCompile("\\s").Split(string(fileContents), -1)
	src := InMemorySource{}
	for _, b64 := range responsesB64 {
		// if the line/space is empty just skip
		if b64 == "" {
			continue
		}
		der, tmpErr := base64.StdEncoding.DecodeString(b64)
		if tmpErr != nil {
			log.Errorf("Base64 decode error %s on: %s", tmpErr, b64)
			continue
		}

		response, tmpErr := ocsp.ParseResponse(der, nil)
		if tmpErr != nil {
			log.Errorf("OCSP decode error %s on: %s", tmpErr, b64)
			continue
		}

		src[response.SerialNumber.String()] = der
	}

	log.Infof("Read %d OCSP responses", len(src))
	return src, nil
}

// A Responder object provides the HTTP logic to expose a
// Source of OCSP responses.
type Responder struct {
	Source Source
	clk    clock.Clock
}

// NewResponder instantiates a Responder with the give Source.
func NewResponder(source Source) *Responder {
	return &Responder{
		Source: source,
		clk:    clock.Default(),
	}
}

// A Responder can process both GET and POST requests.  The mapping
// from an OCSP request to an OCSP response is done by the Source;
// the Responder simply decodes the request, and passes back whatever
// response is provided by the source.
// Note: The caller must use http.StripPrefix to strip any path components
// (including '/') on GET requests.
// Do not use this responder in conjunction with http.NewServeMux, because the
// default handler will try to canonicalize path components by changing any
// strings of repeated '/' into a single '/', which will break the base64
// encoding.
func (rs Responder) ServeHTTP(response http.ResponseWriter, request *http.Request) {
	// By default we set a 'max-age=0, no-cache' Cache-Control header, this
	// is only returned to the client if a valid authorized OCSP response
	// is not found or an error is returned. If a response if found the header
	// will be altered to contain the proper max-age and modifiers.
	response.Header().Add("Cache-Control", "max-age=0, no-cache")
	// Read response from request
	var requestBody []byte
	var err error
	switch request.Method {
	case "GET":
		base64Request, err := url.QueryUnescape(request.URL.Path)
		if err != nil {
			log.Infof("Error decoding URL: %s", request.URL.Path)
			response.WriteHeader(http.StatusBadRequest)
			return
		}
		// url.QueryUnescape not only unescapes %2B escaping, but it additionally
		// turns the resulting '+' into a space, which makes base64 decoding fail.
		// So we go back afterwards and turn ' ' back into '+'. This means we
		// accept some malformed input that includes ' ' or %20, but that's fine.
		base64RequestBytes := []byte(base64Request)
		for i := range base64RequestBytes {
			if base64RequestBytes[i] == ' ' {
				base64RequestBytes[i] = '+'
			}
		}
		requestBody, err = base64.StdEncoding.DecodeString(string(base64RequestBytes))
		if err != nil {
			log.Infof("Error decoding base64 from URL: %s", base64Request)
			response.WriteHeader(http.StatusBadRequest)
			return
		}
	case "POST":
		requestBody, err = ioutil.ReadAll(request.Body)
		if err != nil {
			log.Errorf("Problem reading body of POST: %s", err)
			response.WriteHeader(http.StatusBadRequest)
			return
		}
	default:
		response.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	b64Body := base64.StdEncoding.EncodeToString(requestBody)
	log.Debugf("Received OCSP request: %s", b64Body)

	// All responses after this point will be OCSP.
	// We could check for the content type of the request, but that
	// seems unnecessariliy restrictive.
	response.Header().Add("Content-Type", "application/ocsp-response")

	// Parse response as an OCSP request
	// XXX: This fails if the request contains the nonce extension.
	//      We don't intend to support nonces anyway, but maybe we
	//      should return unauthorizedRequest instead of malformed.
	ocspRequest, err := ocsp.ParseRequest(requestBody)
	if err != nil {
		log.Infof("Error decoding request body: %s", b64Body)
		response.WriteHeader(http.StatusBadRequest)
		response.Write(malformedRequestErrorResponse)
		return
	}

	// Look up OCSP response from source
	ocspResponse, found := rs.Source.Response(ocspRequest)
	if !found {
		log.Infof("No response found for request: serial %x, request body %s",
			ocspRequest.SerialNumber, b64Body)
		response.Write(unauthorizedErrorResponse)
		return
	}

	parsedResponse, err := ocsp.ParseResponse(ocspResponse, nil)
	if err != nil {
		log.Errorf("Error parsing response for serial %x: %s",
			ocspRequest.SerialNumber, err)
		response.Write(unauthorizedErrorResponse)
		return
	}

	// Write OCSP response to response
	response.Header().Add("Last-Modified", parsedResponse.ThisUpdate.Format(time.RFC1123))
	response.Header().Add("Expires", parsedResponse.NextUpdate.Format(time.RFC1123))
	now := rs.clk.Now()
	maxAge := 0
	if now.Before(parsedResponse.NextUpdate) {
		maxAge = int(parsedResponse.NextUpdate.Sub(now) / time.Second)
	} else {
		// TODO(#530): we want max-age=0 but this is technically an authorized OCSP response
		//             (despite being stale) and 5019 forbids attaching no-cache
		maxAge = 0
	}
	response.Header().Set(
		"Cache-Control",
		fmt.Sprintf(
			"max-age=%d, public, no-transform, must-revalidate",
			maxAge,
		),
	)
	responseHash := sha256.Sum256(ocspResponse)
	response.Header().Add("ETag", fmt.Sprintf("\"%X\"", responseHash))

	// RFC 7232 says that a 304 response must contain the above
	// headers if they would also be sent for a 200 for the same
	// request, so we have to wait until here to do this
	if etag := request.Header.Get("If-None-Match"); etag != "" {
		if etag == fmt.Sprintf("\"%X\"", responseHash) {
			response.WriteHeader(http.StatusNotModified)
			return
		}
	}
	response.WriteHeader(http.StatusOK)
	response.Write(ocspResponse)
}
<|endoftext|>


<commit_before>package prpcx

import (
	"github.com/smallnest/rpcx"
	"github.com/smallnest/rpcx/codec"
	"github.com/smallnest/rpcx/plugin"
)

var server *rpcx.Server

func InitRpcx(name string, class interface{}) {
	server = rpcx.NewServer()
	server.PluginContainer.Add(plugin.NewMetricsPlugin())
	server.ServerCodecFunc = codec.NewJSONRPC2ServerCodec
	server.RegisterName(name, class)
}

func Run(addr string) error{
	err := server.Serve("tcp", addr)
	return err
}

<commit_msg>modify metric<commit_after>package prpcx

import (
	"github.com/smallnest/rpcx"
	"github.com/smallnest/rpcx/codec"
)

var server *rpcx.Server

func InitRpcx(name string, class interface{}) {
	server = rpcx.NewServer()
	//server.PluginContainer.Add(plugin.NewMetricsPlugin())
	server.ServerCodecFunc = codec.NewJSONRPC2ServerCodec
	server.RegisterName(name, class)
}

func Run(addr string) error{
	err := server.Serve("tcp", addr)
	return err
}

<|endoftext|>