<commit_before>/*
 A Naive Bayesian Classifier
 Jake Brukhman <jbrukh@gmail.com>

 BAYESIAN CLASSIFICATION REFRESHER: suppose you have a set
 of classes (e.g. categories) `C := {C_1, ..., C_n}`, and a
 document D consisting of words D := {W_1, ..., W_k}.
 We wish to ascertain the probability that the document
 belongs to some class C_j given some set of training data
 associating documents and classes.

 By Bayes Theorem, we have that

    P(C_j|D) = P(D|C_j)*P(C_j)/P(D).

 The LHS is the probability that the document belongs to class
 C_j given the document itself (by which is meant, in practice,
 the word frequencies occurring in this document), and our program
 will calculate this probability for each j and spit out the
 most likely class for this document.

 P(C_j) is referred to as the "prior" probability, or the
 probability that a document belongs to C_j in general, without
 seeing the document first. P(D|C_j) is the probability of seeing
 such a document, given that it belongs to C_j. Here, by assuming
 that words appear independently in documents (this being the
 "naive" assumption), we can estimate

    P(D|C_j) ~= P(W_1|C_j)*...*P(W_k|C_j)

 where P(W_i|C_j) is the probability of seeing the given word
 in a document of the given class. Finally, P(D) can be seen as
 merely a scaling factor and is not strictly relevant to
 classificiation, unless you want to normalize the resulting
 scores and actually see probabilities. In this case, note that

    P(D) = SUM_j(P(D|C_j)*P(C_j))

 One practical issue with performing these calculations is the
 possibility of float64 underflow when calculating P(D|C_j), as
 individual word probabilities can be arbitrarily small, and
 a document can have an arbitrarily large number of them. A
 typical method for dealing with this case is to transform the
 probability to the log domain and perform additions instead
 of multiplications:

   log P(C_j|D) ~ log(P(C_j)) + SUM_i(log P(W_i|C_j))

 where i = 1, ..., k. Note that by doing this, we are discarding
 the scaling factor P(D) and our scores are no longer
 probabilities.
*/
package bayesian

import (
    "math"
    "gob"
    "os"
)

// defaultProb is the tiny non-zero probability that a word
// we have not seen before appears in the class.
const defaultProb = 0.00000000001

// This type defines a set of classes that the classifier will
// filter: C = {C_1, ..., C_n}. You should define your classes
// as a set of constants, for example as follows:
//
//    const (
//        Good Class = "Good"
//        Bad Class = "Bad
//    )
//
type Class string

// Classifier implements the Naive Bayesian Classifier.
type Classifier struct {
    Classes []Class
    datas map[Class]*classData
}

// serializableClassifier represents an container for
// classifier objects whose fields are modifiable by
// reflection and are therefore writeable.
type serializableClassifier struct {
    Classes []Class
    Datas map[Class]*classData
}

// classData holds the frequency data for words in a
// particular class. In the future, we may replace this
// structure with a trie-like structure for more
// efficient storage.
type classData struct {
    Freqs map[string]int
    Total int
}

// newClassData creates a new empty class data node.
func newClassData() *classData {
    return &classData{
        Freqs: make(map[string]int),
    }
}

// P(W|Cj) -- the probability of seeing a particular word
// in a document of this class.
func (d *classData) getWordProb(word string) float64 {
    value, ok := d.Freqs[word]
    if !ok {
        return defaultProb
    }
    return float64(value)/float64(d.Total)
}

// P(D|C_j) -- the probability of seeing this set of words
// in a document of this class.
//
// Note that words should not be empty, and this method of
// calulation is prone to underflow if there are many words
// and their individual probabilties are small.
func (d *classData) getWordsProb(words []string) (prob float64) {
    prob = 1
    for _, word := range words {
        prob *= d.getWordProb(word)
    }
    return
}

// New creates a new Classifier. The classes the provided
// should be at least 2 in number and unique from each other.
func NewClassifier(classes ...Class) (inst *Classifier) {
    if len(classes) < 2 {
        panic("provide at least two classes")
    }
    inst = &Classifier{
            classes,
            make(map[Class]*classData),
    }
    for _, class := range classes {
        inst.datas[class] = newClassData()
    }
    return
}

// NewClassifierFromFile loads an existing classifier from
// disk. The classifier was previously saved with a call
// to WriteToFile().
func NewClassifierFromFile(name string) (c *Classifier, err os.Error) {
    file, err := os.Open(name)
    if err != nil {
        return nil, err
    }
    dec := gob.NewDecoder(file)
    w := new(serializableClassifier)
    err = dec.Decode(w)

    return &Classifier{w.Classes, w.Datas}, err
}

// getPriors returns the prior probabilities for the
// classes provided -- P(C_i). There is a way to
// smooth priors, currently not implemented here.
func (c *Classifier) getPriors() (priors []float64) {
    n := len(c.Classes)
    priors = make([]float64, n, n)
    sum := 0
    for index, class := range c.Classes {
        total := c.datas[class].Total;
        priors[index] = float64(total)
        sum += total
    }
    if sum != 0 {
        for i := 0; i < n; i++ {
            priors[i] /= float64(sum)
        }
    }
    return
}

// Learn will train the classifier on the provided data.
func (c *Classifier) Learn(words []string, which Class) {
    data := c.datas[which]
    for _, word := range words {
        data.Freqs[word]++
        data.Total++
    }
}

// Scores will produce an array of scores that correspond
// to its opinion on the document in question, and whether it
// belongs to the given class. The order of the scores
// in the return values follows the order of the inital array
// of Class objects parameterized to the NewClassifier() function.
// If no training data has been provided, c will return
// a 0 array.
//
// The value of the score is proportional to the likelihood,
// even if the score is negative, so that the score with the
// greatest value corresponds to the most likely class.
//
// Additionally, c function will return the index of the
// maximum probability. The value of c number is given by
// scores[inx]. The class of that corresponds to c number
// is classifier.Classes[inx]. If more than one of the
// returned probabilities has the maximum values, then
// strict is false.
func (c *Classifier) LogScores(words []string) (scores []float64, inx int, strict bool) {
    n := len(c.Classes)
    scores = make([]float64, n, n)
    priors := c.getPriors()

    // calculate the score for each class
    for index, class := range c.Classes {
        data := c.datas[class]
        // c is the sum of the logarithms
        // as outlined in the refresher
        score := math.Log(priors[index])
        for _, word := range words {
            score += math.Log(data.getWordProb(word))
        }
        scores[index] = score
    }
    inx, strict = findMax(scores)
    return scores, inx, strict
}

// Probabilities works the same as Score, but delivers
// actual probabilities as discussed above. Note that float64
// underflow is possible if the word list contains too
// many doc that have probabilities very close to 0.
func (c *Classifier) Probabilities(doc []string) (scores []float64, inx int, strict bool) {
    n := len(c.Classes)
    scores = make([]float64, n, n)
    priors := c.getPriors()
    sum := float64(0)
    // calculate the score for each class
    for index, class := range c.Classes {
        data := c.datas[class]
        // c is the sum of the logarithms
        // as outlined in the refresher
        score := priors[index]
        for _, word := range doc {
            score *= data.getWordProb(word)
        }
        scores[index] = score
        sum += score
    }
    for i := 0; i < n; i++ {
        scores[i] /= sum
    }
    inx, strict = findMax(scores)
    return scores, inx, strict
}

// WordFrequencies returns a matrix of word frequencies that currently
// exist in the classifier for each class state for the given input
// words. In other words, if you obtain the frequencies
//
//    freqs := c.WordFrequencies(/* ... array of j words ... */)
//
// then the expression freq[i][j] represents the frequency of the j-th
// word within the i-th class.
func (c *Classifier) WordFrequencies(words []string) (freqMatrix [][]float64) {
    n, l := len(c.Classes), len(words)
    freqMatrix = make([][]float64, n)
    for i, _ := range freqMatrix {
        arr := make([]float64, l)
        data := c.datas[c.Classes[i]]
        for j, _ := range arr {
            arr[j] = data.getWordProb(words[j])
        }
        freqMatrix[i] = arr
    }
    return
}

// Serialize this classifier to a file.
func (c *Classifier) WriteToFile(name string) (err os.Error) {
    file, err := os.OpenFile(name, os.O_WRONLY | os.O_CREATE, 0655)
    if err != nil {
        return err
    }
    enc := gob.NewEncoder(file)
    err = enc.Encode(&serializableClassifier{c.Classes, c.datas})
    return
}

// Seen returns the number of words seen for
// each class.
func (c *Classifier) Seen() (result []int) {
    result = make([]int, len(c.Classes))
    for inx, class := range c.Classes {
        data := c.datas[class]
        result[inx] = data.Total
    }
    return
}

// findMax finds the maximum of a set of scores; if the
// maximum is strict -- that is, it is the single unique
// maximum from the set -- then strict has return value
// true. Otherwise it is false.
func findMax(scores []float64) (inx int, strict bool) {
    inx = 0
    strict = true
    for i := 1; i < len(scores); i++ {
        if scores[inx] < scores[i] {
            inx = i
            strict = true
        } else if scores[inx] == scores[i] {
            strict = false
        }
    }
    return
}
<commit_msg>Reworded documentation.<commit_after>/*
 A Naive Bayesian Classifier
 Jake Brukhman <jbrukh@gmail.com>

 BAYESIAN CLASSIFICATION REFRESHER: suppose you have a set
 of classes (e.g. categories) C := {C_1, ..., C_n}, and a
 document D consisting of words D := {W_1, ..., W_k}.
 We wish to ascertain the probability that the document
 belongs to some class C_j given some set of training data
 associating documents and classes.

 By Bayes' Theorem, we have that

    P(C_j|D) = P(D|C_j)*P(C_j)/P(D).

 The LHS is the probability that the document belongs to class
 C_j given the document itself (by which is meant, in practice,
 the word frequencies occurring in this document), and our program
 will calculate this probability for each j and spit out the
 most likely class for this document.

 P(C_j) is referred to as the "prior" probability, or the
 probability that a document belongs to C_j in general, without
 seeing the document first. P(D|C_j) is the probability of seeing
 such a document, given that it belongs to C_j. Here, by assuming
 that words appear independently in documents (this being the
 "naive" assumption), we can estimate

    P(D|C_j) ~= P(W_1|C_j)*...*P(W_k|C_j)

 where P(W_i|C_j) is the probability of seeing the given word
 in a document of the given class. Finally, P(D) can be seen as
 merely a scaling factor and is not strictly relevant to
 classificiation, unless you want to normalize the resulting
 scores and actually see probabilities. In this case, note that

    P(D) = SUM_j(P(D|C_j)*P(C_j))

 One practical issue with performing these calculations is the
 possibility of float64 underflow when calculating P(D|C_j), as
 individual word probabilities can be arbitrarily small, and
 a document can have an arbitrarily large number of them. A
 typical method for dealing with this case is to transform the
 probability to the log domain and perform additions instead
 of multiplications:

   log P(C_j|D) ~ log(P(C_j)) + SUM_i(log P(W_i|C_j))

 where i = 1, ..., k. Note that by doing this, we are discarding
 the scaling factor P(D) and our scores are no longer
 probabilities.
*/
package bayesian

import (
    "math"
    "gob"
    "os"
)

// defaultProb is the tiny non-zero probability that a word
// we have not seen before appears in the class.
const defaultProb = 0.00000000001

// Class defines a set of classes that the classifier will
// filter: C = {C_1, ..., C_n}. You should define your classes
// as a set of constants, for example as follows:
//
//    const (
//        Good Class = "Good"
//        Bad Class = "Bad
//    )
//
// Class values should be unique.
type Class string

// Classifier implements the Naive Bayesian Classifier.
type Classifier struct {
    Classes []Class
    datas map[Class]*classData
}

// serializableClassifier represents a container for
// Classifier objects whose fields are modifiable by
// reflection and are therefore writeable by gob.
type serializableClassifier struct {
    Classes []Class
    Datas map[Class]*classData
}

// classData holds the frequency data for words in a
// particular class. In the future, we may replace this
// structure with a trie-like structure for more
// efficient storage.
type classData struct {
    Freqs map[string]int
    Total int
}

// newClassData creates a new empty classData node.
func newClassData() *classData {
    return &classData{
        Freqs: make(map[string]int),
    }
}

// getWordProb returns P(W|C_j) -- the probability of seeing
// a particular word W in a document of this class.
func (d *classData) getWordProb(word string) float64 {
    value, ok := d.Freqs[word]
    if !ok {
        return defaultProb
    }
    return float64(value)/float64(d.Total)
}

// getWordsProb returns P(D|C_j) -- the probability of seeing
// this set of words in a document of this class.
//
// Note that words should not be empty, and this method of
// calulation is prone to underflow if there are many words
// and their individual probabilties are small.
func (d *classData) getWordsProb(words []string) (prob float64) {
    prob = 1
    for _, word := range words {
        prob *= d.getWordProb(word)
    }
    return
}

// NewClassifier returns a new classifier. The classes the provided
// should be at least 2 in number and unique, or this method will
// panic.
func NewClassifier(classes ...Class) (c *Classifier) {
    n := len(classes)

    // check size
    if n < 2 {
        panic("provide at least two classes")
    }

    // check uniqueness
    check := make(map[Class]bool, n)
    for _, class := range classes {
        check[class] = true
    }
    if len(check) != n {
        panic("classes must be unique")
    }
    // create the classifier
    c = &Classifier{
            classes,
            make(map[Class]*classData, n),
    }
    for _, class := range classes {
        c.datas[class] = newClassData()
    }
    return
}

// NewClassifierFromFile loads an existing classifier from
// file. The classifier was previously saved with a call
// to c.WriteToFile(string).
func NewClassifierFromFile(name string) (c *Classifier, err os.Error) {
    file, err := os.Open(name)
    if err != nil {
        return nil, err
    }
    dec := gob.NewDecoder(file)
    w := new(serializableClassifier)
    err = dec.Decode(w)

    return &Classifier{w.Classes, w.Datas}, err
}

// getPriors returns the prior probabilities for the
// classes provided -- P(C_j).
//
// TODO: There is a way to smooth priors, currently
// not implemented here.
func (c *Classifier) getPriors() (priors []float64) {
    n := len(c.Classes)
    priors = make([]float64, n, n)
    sum := 0
    for index, class := range c.Classes {
        total := c.datas[class].Total;
        priors[index] = float64(total)
        sum += total
    }
    if sum != 0 {
        for i := 0; i < n; i++ {
            priors[i] /= float64(sum)
        }
    }
    return
}

// Learn will accept new training documents for
// supervised learning.
func (c *Classifier) Learn(document []string, which Class) {
    data := c.datas[which]
    for _, word := range document {
        data.Freqs[word]++
        data.Total++
    }
}

// LogScores produces "log-likelihood"-like scores that can
// be used to classify documents into classes.
//
// The value of the score is proportional to the likelihood,
// as determined by the classifier, that the given document
// belongs to the given class. This is true even when scores
// returned are negative, which they will be (since we are
// taking logs of probabilities).
//
// The index j of the score corresponds to the class given
// by c.Classes[j].
//
// Additionally returned are "inx" and "strict" values. The
// inx corresponds to the maximum score in the array. If more
// than one of the scores holds the maximum values, then
// strict is false.
//
// Unlike c.Probabilities(), this function is not prone to
// floating point underflow and is relatively safe to use.
func (c *Classifier) LogScores(document []string) (scores []float64, inx int, strict bool) {
    n := len(c.Classes)
    scores = make([]float64, n, n)
    priors := c.getPriors()

    // calculate the score for each class
    for index, class := range c.Classes {
        data := c.datas[class]
        // c is the sum of the logarithms
        // as outlined in the refresher
        score := math.Log(priors[index])
        for _, word := range document {
            score += math.Log(data.getWordProb(word))
        }
        scores[index] = score
    }
    inx, strict = findMax(scores)
    return scores, inx, strict
}

// Probabilities works the same as Score, but delivers
// actual probabilities as discussed above. Note that float64
// underflow is possible if the word list contains too
// many words that have probabilities very close to 0.
func (c *Classifier) Probabilities(doc []string) (scores []float64, inx int, strict bool) {
    n := len(c.Classes)
    scores = make([]float64, n, n)
    priors := c.getPriors()
    sum := float64(0)
    // calculate the score for each class
    for index, class := range c.Classes {
        data := c.datas[class]
        // c is the sum of the logarithms
        // as outlined in the refresher
        score := priors[index]
        for _, word := range doc {
            score *= data.getWordProb(word)
        }
        scores[index] = score
        sum += score
    }
    for i := 0; i < n; i++ {
        scores[i] /= sum
    }
    inx, strict = findMax(scores)
    return scores, inx, strict
}

// WordFrequencies returns a matrix of word frequencies that currently
// exist in the classifier for each class state for the given input
// words. In other words, if you obtain the frequencies
//
//    freqs := c.WordFrequencies(/* [j]string */)
//
// then the expression freq[i][j] represents the frequency of the j-th
// word within the i-th class.
func (c *Classifier) WordFrequencies(words []string) (freqMatrix [][]float64) {
    n, l := len(c.Classes), len(words)
    freqMatrix = make([][]float64, n)
    for i, _ := range freqMatrix {
        arr := make([]float64, l)
        data := c.datas[c.Classes[i]]
        for j, _ := range arr {
            arr[j] = data.getWordProb(words[j])
        }
        freqMatrix[i] = arr
    }
    return
}

// Serialize this classifier to a file.
func (c *Classifier) WriteToFile(name string) (err os.Error) {
    file, err := os.OpenFile(name, os.O_WRONLY | os.O_CREATE, 0655)
    if err != nil {
        return err
    }
    enc := gob.NewEncoder(file)
    err = enc.Encode(&serializableClassifier{c.Classes, c.datas})
    return
}

// Seen returns the number of words seen for
// each class, in the lifetime of the classifier.
func (c *Classifier) Seen() (result []int) {
    result = make([]int, len(c.Classes))
    for inx, class := range c.Classes {
        data := c.datas[class]
        result[inx] = data.Total
    }
    return
}

// findMax finds the maximum of a set of scores; if the
// maximum is strict -- that is, it is the single unique
// maximum from the set -- then strict has return value
// true. Otherwise it is false.
func findMax(scores []float64) (inx int, strict bool) {
    inx = 0
    strict = true
    for i := 1; i < len(scores); i++ {
        if scores[inx] < scores[i] {
            inx = i
            strict = true
        } else if scores[inx] == scores[i] {
            strict = false
        }
    }
    return
}
<|endoftext|>


<commit_before>/*******************************************************************************
*
* Copyright 2018 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package audit

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/sapcc/go-bits/logg"
	"github.com/sapcc/limes/pkg/limes"
	"github.com/streadway/amqp"
)

//sendEvents sends audit events to a RabbitMQ server
func sendEvents(clusterID string, config limes.CADFConfiguration, events []CADFevent) error {
	labels := prometheus.Labels{
		"os_cluster": clusterID,
	}
	eventPublishSuccessCounter.With(labels).Add(0)
	eventPublishFailedCounter.With(labels).Add(0)

	// establish a connection with the RabbitMQ server
	conn, err := amqp.Dial(config.RabbitMQ.URL)
	if err != nil {
		eventPublishFailedCounter.With(labels).Inc()
		return fmt.Errorf("%s -- Failed to establish a connection with the server: %s", events[0].ID, err)
	}
	defer conn.Close()

	// open a unique, concurrent server channel to process the bulk of AMQP messages.
	ch, err := conn.Channel()
	if err != nil {
		eventPublishFailedCounter.With(labels).Inc()
		return fmt.Errorf("%s -- Failed to open a channel: %s", events[0].ID, err)
	}
	defer ch.Close()

	// declare a queue to hold and deliver messages to consumers.
	q, err := ch.QueueDeclare(
		config.RabbitMQ.QueueName, // name of the queue
		true,  // durable: queue should survive cluster reset (or broker restart)
		false, // autodelete when unused
		false, // exclusive: queue only accessible by connection that declares and deleted when the connection closes?
		false, // noWait: the queue will assume to be declared on the server
		nil,   // arguments for advanced config
	)
	if err != nil {
		eventPublishFailedCounter.With(labels).Inc()
		return fmt.Errorf("%s -- Failed to declare a queue: %s", events[0].ID, err)
	}

	// publish the events to an exchange on the server
	for _, event := range events {
		body, _ := json.Marshal(event)
		err = ch.Publish(
			"",     // exchange: publish to default
			q.Name, // routing key: same as queue name
			false,  // mandatory: don't publish if no queue is bound that matches the routing key
			false,  // immediate: don't publish if no consumer on the matched queue is ready to accept the delivery
			amqp.Publishing{
				ContentType: "text/plain",
				Body:        []byte(body),
			},
		)
		if err != nil {
			eventPublishFailedCounter.With(labels).Inc()
			return fmt.Errorf("%s -- Failed to publish the audit event: %s", event.ID, err)
		}
		eventPublishSuccessCounter.With(labels).Inc()
	}

	return err
}

//backoff creates a retry loop with an exponential backoff
func backoff(action func() error) {
	duration := time.Second
	for {
		err := action()
		if err != nil {
			logg.Error("RabbitMQ -- %s", err)
			if duration > 5*time.Minute {
				duration = 5 * time.Minute
			} else {
				duration *= 2
			}
			time.Sleep(duration)
			continue
		}
		break
	}
}
<commit_msg>fix backoff loop algorithm<commit_after>/*******************************************************************************
*
* Copyright 2018 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package audit

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/sapcc/go-bits/logg"
	"github.com/sapcc/limes/pkg/limes"
	"github.com/streadway/amqp"
)

//sendEvents sends audit events to a RabbitMQ server
func sendEvents(clusterID string, config limes.CADFConfiguration, events []CADFevent) error {
	labels := prometheus.Labels{
		"os_cluster": clusterID,
	}
	eventPublishSuccessCounter.With(labels).Add(0)
	eventPublishFailedCounter.With(labels).Add(0)

	// establish a connection with the RabbitMQ server
	conn, err := amqp.Dial(config.RabbitMQ.URL)
	if err != nil {
		eventPublishFailedCounter.With(labels).Inc()
		return fmt.Errorf("%s -- Failed to establish a connection with the server: %s", events[0].ID, err)
	}
	defer conn.Close()

	// open a unique, concurrent server channel to process the bulk of AMQP messages.
	ch, err := conn.Channel()
	if err != nil {
		eventPublishFailedCounter.With(labels).Inc()
		return fmt.Errorf("%s -- Failed to open a channel: %s", events[0].ID, err)
	}
	defer ch.Close()

	// declare a queue to hold and deliver messages to consumers.
	q, err := ch.QueueDeclare(
		config.RabbitMQ.QueueName, // name of the queue
		true,  // durable: queue should survive cluster reset (or broker restart)
		false, // autodelete when unused
		false, // exclusive: queue only accessible by connection that declares and deleted when the connection closes?
		false, // noWait: the queue will assume to be declared on the server
		nil,   // arguments for advanced config
	)
	if err != nil {
		eventPublishFailedCounter.With(labels).Inc()
		return fmt.Errorf("%s -- Failed to declare a queue: %s", events[0].ID, err)
	}

	// publish the events to an exchange on the server
	for _, event := range events {
		body, _ := json.Marshal(event)
		err = ch.Publish(
			"",     // exchange: publish to default
			q.Name, // routing key: same as queue name
			false,  // mandatory: don't publish if no queue is bound that matches the routing key
			false,  // immediate: don't publish if no consumer on the matched queue is ready to accept the delivery
			amqp.Publishing{
				ContentType: "text/plain",
				Body:        []byte(body),
			},
		)
		if err != nil {
			eventPublishFailedCounter.With(labels).Inc()
			return fmt.Errorf("%s -- Failed to publish the audit event: %s", event.ID, err)
		}
		eventPublishSuccessCounter.With(labels).Inc()
	}

	return err
}

//backoff creates a retry loop with an exponential backoff
func backoff(action func() error) {
	duration := time.Second
	for {
		err := action()
		if err != nil {
			logg.Error("RabbitMQ -- %s", err)
			duration *= 2
			if duration > 5*time.Minute {
				duration = 5 * time.Minute
			}
			time.Sleep(duration)
			continue
		}
		break
	}
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vulkan

import (
	"testing"

	"github.com/google/gapid/core/log"
	"github.com/google/gapid/core/os/device"
	"github.com/google/gapid/gapis/api"
	"github.com/google/gapid/gapis/memory"
)

func TestCallReflectedCommand(t *testing.T) {
	ctx := log.Testing(t)
	s := api.NewStateWithEmptyAllocator(device.Little32)
	cb := CommandBuilder{}
	cmd := cb.VkCreateBuffer(
		VkDevice(0),
		memory.Nullptr,
		memory.Nullptr,
		memory.Nullptr,
		VkResult_VK_SUCCESS,
	)
	CallReflectedCommand(ctx, cmd, 10, s, nil, subDovkCmdDispatch, MakeVkCmdDispatchArgsʳ())
}
<commit_msg>gapis/api/vulkan: Fix TestCallReflectedCommand<commit_after>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vulkan

import (
	"testing"

	"github.com/google/gapid/core/log"
	"github.com/google/gapid/core/os/device"
	"github.com/google/gapid/gapis/api"
	"github.com/google/gapid/gapis/memory"
)

func TestCallReflectedCommand(t *testing.T) {
	ctx := log.Testing(t)
	s := api.NewStateWithEmptyAllocator(device.Little32)
	cb := CommandBuilder{}
	cmd := cb.VkCreateBuffer(
		VkDevice(0),
		memory.Nullptr,
		memory.Nullptr,
		memory.Nullptr,
		VkResult_VK_SUCCESS,
	)
	CallReflectedCommand(ctx, cmd, 10, s, nil, subDovkCmdWriteTimestamp, NewVkCmdWriteTimestampArgsʳ(0, 0, 0))
}
<|endoftext|>


<commit_before>// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package build

import (
	"fmt"
	"path/filepath"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/osutil"
)

type gvisor struct{}

func (gvisor gvisor) build(targetArch, vmType, kernelDir, outputDir, compiler, userspaceDir,
	cmdlineFile, sysctlFile string, config []byte) error {
	args := []string{"build", "--verbose_failures"}
	if strings.Contains(" "+string(config)+" ", " -race ") {
		args = append(args, "--features=race")
	}
	args = append(args, "runsc")
	if _, err := osutil.RunCmd(20*time.Minute, kernelDir, compiler, args...); err != nil {
		return err
	}
	if err := gvisor.copyBinary(kernelDir, outputDir); err != nil {
		return err
	}
	if len(config) != 0 {
		if err := osutil.WriteFile(filepath.Join(outputDir, "kernel.config"), config); err != nil {
			return fmt.Errorf("failed to save kernel config: %v", err)
		}
	}
	osutil.RunCmd(10*time.Minute, kernelDir, compiler, "shutdown")
	return nil
}

func (gvisor) copyBinary(kernelDir, outputDir string) error {
	// Funny it's not possible to understand what bazel actually built...
	for _, typ := range []string{
		"linux_amd64_pure_stripped",
		"linux_amd64_static_stripped",
		"linux_amd64_static_race_stripped",
	} {
		runsc := filepath.Join(kernelDir, "bazel-bin", "runsc", typ, "runsc")
		if !osutil.IsExist(runsc) {
			continue
		}
		return osutil.CopyFile(runsc, filepath.Join(outputDir, "image"))
	}
	return fmt.Errorf("failed to locate bazel output")
}

func (gvisor) clean(kernelDir string) error {
	// Let's assume that bazel always properly handles build without cleaning (until proven otherwise).
	return nil
}
<commit_msg>pkg/build: update gvisor race build process<commit_after>// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package build

import (
	"path/filepath"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/osutil"
)

type gvisor struct{}

func (gvisor gvisor) build(targetArch, vmType, kernelDir, outputDir, compiler, userspaceDir,
	cmdlineFile, sysctlFile string, config []byte) error {
	outBinary := ""
	args := []string{"build", "--verbose_failures"}
	if strings.Contains(" "+string(config)+" ", " -race ") {
		args = append(args, "--features=race", "//runsc:runsc-race")
		outBinary = "bazel-bin/runsc/linux_amd64_static_race_stripped/runsc-race"
	} else {
		args = append(args, "//runsc:runsc")
		outBinary = "bazel-bin/runsc/linux_amd64_pure_stripped/runsc"
	}
	outBinary = filepath.Join(kernelDir, filepath.FromSlash(outBinary))
	if _, err := osutil.RunCmd(20*time.Minute, kernelDir, compiler, args...); err != nil {
		return err
	}
	if err := osutil.CopyFile(outBinary, filepath.Join(outputDir, "image")); err != nil {
		return err
	}
	osutil.RunCmd(10*time.Minute, kernelDir, compiler, "shutdown")
	return nil
}

func (gvisor) clean(kernelDir string) error {
	// Let's assume that bazel always properly handles build without cleaning (until proven otherwise).
	return nil
}
<|endoftext|>


<commit_before>// Package datastore TODO
package datastore

import (
	"context"
	"database/sql"
	"fmt"
	"os"

	_ "github.com/lib/pq" // pq driver calls for blank identifier
)

// Datastore struct stores common environment related items
type Datastore struct {
	MainDb *sql.DB
	LogDb  *sql.DB
}

// NewDatastore initializes the datastore struct
func NewDatastore() (*Datastore, error) {
	db, err := newMainDB()
	if err != nil {
		return nil, err
	}
	return &Datastore{MainDb: db, LogDb: nil}, nil
}

// NewMainDB returns an open database handle of 0 or more underlying connections
func newMainDB() (*sql.DB, error) {

	// Get Database connection credentials from environment variables
	dbName := os.Getenv("PG_DBNAME")
	dbUser := os.Getenv("PG_USERNAME")
	dbPassword := os.Getenv("PG_PASSWORD")

	// Craft string for database connection
	dbinfo := fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable", dbUser, dbPassword, dbName)

	// Open the postgres database using the postgres driver (pq)
	// func Open(driverName, dataSourceName string) (*DB, error)
	db, err := sql.Open("postgres", dbinfo)
	if err != nil {
		return nil, err
	}
	if err = db.Ping(); err != nil {
		return nil, err
	}
	return db, nil
}

// Tx opens a database connection and starts a database transaction using the
// BeginTx method which allows for rollback of the transaction if the context
// is cancelled
func (ds Datastore) Tx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) {

	// Calls the BeginTx method of the MainDb opened database
	tx, err := ds.MainDb.BeginTx(ctx, opts)
	if err != nil {
		return nil, err
	}

	return tx, nil

}

// // The key type is unexported to prevent collisions with context keys defined in
// // other packages.
// type key int

// // dbTxKey is the context key for the database txn.  Its value of zero is
// // arbitrary.  If this package defined other context keys, they would have
// // different integer values.
// const dbTxKey key = 0

// // Tx2Context returns Context carrying the database transaction (tx).
// func Tx2Context(ctx context.Context, opts *sql.TxOptions) context.Context {
// 	tx, _ := Tx(ctx, db, opts)
// 	return context.WithValue(ctx, dbTxKey, tx)
// }

// // TxFromContext extracts the database transaction from the context, if present.
// func TxFromContext(ctx context.Context) (*sql.Tx, bool) {
// 	// ctx.Value returns nil if ctx has no value for the key;
// 	// the *sql.Tx type assertion returns ok=false for nil.
// 	dbTx, ok := ctx.Value(dbTxKey).(*sql.Tx)
// 	return dbTx, ok
// }
<commit_msg>LogDB changes<commit_after>// Package datastore has all the functions needed to start any database
// connections as well as transactions
package datastore

import (
	"context"
	"database/sql"
	"fmt"
	"os"

	_ "github.com/lib/pq" // pq driver calls for blank identifier
)

// Datastore struct stores common environment related items
type Datastore struct {
	MainDb *sql.DB
	LogDb  *sql.DB
}

// NewDatastore initializes the datastore struct
// NOTE: I have chosen to use the same database for logging as
// my "main" app database. I'd recommend having a separate db and
// would have a separate method to start that connection pool up and
// pass it, but since this is just an example....
func NewDatastore() (*Datastore, error) {
	mdb, err := newMainDB()
	if err != nil {
		return nil, err
	}

	return &Datastore{MainDb: mdb, LogDb: mdb}, nil
}

// NewMainDB returns an open database handle of 0 or more underlying connections
func newMainDB() (*sql.DB, error) {

	// Get Database connection credentials from environment variables
	dbName := os.Getenv("PG_DBNAME")
	dbUser := os.Getenv("PG_USERNAME")
	dbPassword := os.Getenv("PG_PASSWORD")

	// Craft string for database connection
	dbinfo := fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable", dbUser, dbPassword, dbName)

	// Open the postgres database using the postgres driver (pq)
	// func Open(driverName, dataSourceName string) (*DB, error)
	db, err := sql.Open("postgres", dbinfo)
	if err != nil {
		return nil, err
	}
	if err = db.Ping(); err != nil {
		return nil, err
	}
	return db, nil
}

// Tx opens a database connection and starts a database transaction using the
// BeginTx method which allows for rollback of the transaction if the context
// is cancelled
func (ds Datastore) Tx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) {

	// Calls the BeginTx method of the MainDb opened database
	tx, err := ds.MainDb.BeginTx(ctx, opts)
	if err != nil {
		return nil, err
	}

	return tx, nil
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium

package ipcache

import (
	"context"
	"fmt"
	"net"
	"net/netip"
	"strings"

	"github.com/sirupsen/logrus"

	"github.com/cilium/cilium/pkg/identity"
	"github.com/cilium/cilium/pkg/ip"
	"github.com/cilium/cilium/pkg/labels"
	"github.com/cilium/cilium/pkg/labels/cidr"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/metrics"
	"github.com/cilium/cilium/pkg/option"
	"github.com/cilium/cilium/pkg/source"
)

// AllocateCIDRs attempts to allocate identities for a list of CIDRs. If any
// allocation fails, all allocations are rolled back and the error is returned.
// When an identity is freshly allocated for a CIDR, it is added to the
// ipcache if 'newlyAllocatedIdentities' is 'nil', otherwise the newly allocated
// identities are placed in 'newlyAllocatedIdentities' and it is the caller's
// responsibility to upsert them into ipcache by calling UpsertGeneratedIdentities().
//
// Previously used numeric identities for the given prefixes may be passed in as the
// 'oldNIDs' parameter; nil slice must be passed if no previous numeric identities exist.
// Previously used NID is allocated if still available. Non-availability is not an error.
//
// Upon success, the caller must also arrange for the resulting identities to
// be released via a subsequent call to ReleaseCIDRIdentitiesByCIDR().
func (ipc *IPCache) AllocateCIDRs(
	prefixes []netip.Prefix, oldNIDs []identity.NumericIdentity, newlyAllocatedIdentities map[string]*identity.Identity,
) ([]*identity.Identity, error) {
	// maintain list of used identities to undo on error
	usedIdentities := make([]*identity.Identity, 0, len(prefixes))

	// Maintain list of newly allocated identities to update ipcache,
	// but upsert them to ipcache only if no map was given by the caller.
	upsert := false
	if newlyAllocatedIdentities == nil {
		upsert = true
		newlyAllocatedIdentities = map[string]*identity.Identity{}
	}

	allocateCtx, cancel := context.WithTimeout(context.Background(), option.Config.IPAllocationTimeout)
	defer cancel()

	ipc.Lock()
	allocatedIdentities := make(map[netip.Prefix]*identity.Identity, len(prefixes))
	for i, prefix := range prefixes {
		lbls := cidr.GetCIDRLabels(prefix)
		lbls.MergeLabels(ipc.metadata.get(prefix).ToLabels())
		oldNID := identity.InvalidIdentity
		if oldNIDs != nil && len(oldNIDs) > i {
			oldNID = oldNIDs[i]
		}
		id, isNew, err := ipc.allocate(allocateCtx, prefix, lbls, oldNID)
		if err != nil {
			ipc.IdentityAllocator.ReleaseSlice(context.Background(), nil, usedIdentities)
			ipc.Unlock()
			return nil, err
		}

		usedIdentities = append(usedIdentities, id)
		allocatedIdentities[prefix] = id
		if isNew {
			newlyAllocatedIdentities[prefix.String()] = id
		}
	}
	ipc.Unlock()

	// Only upsert into ipcache if identity wasn't allocated
	// before and the caller does not care doing this
	if upsert {
		ipc.UpsertGeneratedIdentities(newlyAllocatedIdentities, nil)
	}

	identities := make([]*identity.Identity, 0, len(allocatedIdentities))
	for _, id := range allocatedIdentities {
		identities = append(identities, id)
	}
	return identities, nil
}

// AllocateCIDRsForIPs performs the same action as AllocateCIDRs but for IP
// addresses instead of CIDRs.
//
// Upon success, the caller must also arrange for the resulting identities to
// be released via a subsequent call to ReleaseCIDRIdentitiesByID().
func (ipc *IPCache) AllocateCIDRsForIPs(
	prefixes []net.IP, newlyAllocatedIdentities map[string]*identity.Identity,
) ([]*identity.Identity, error) {
	return ipc.AllocateCIDRs(ip.IPsToNetPrefixes(prefixes), nil, newlyAllocatedIdentities)
}

func cidrLabelToPrefix(label string) (string, bool) {
	if !strings.HasPrefix(label, labels.LabelSourceCIDR) {
		return "", false
	}
	return strings.TrimPrefix(label, labels.LabelSourceCIDR+":"), true
}

// UpsertGeneratedIdentities unconditionally upserts 'newlyAllocatedIdentities'
// into the ipcache, then also upserts any CIDR identities in 'usedIdentities'
// that were not already upserted. If any 'usedIdentities' are upserted, these
// are counted separately as they may provide an indication of another logic
// error elsewhere in the codebase that is causing premature ipcache deletions.
func (ipc *IPCache) UpsertGeneratedIdentities(newlyAllocatedIdentities map[string]*identity.Identity, usedIdentities []*identity.Identity) {
	for prefixString, id := range newlyAllocatedIdentities {
		ipc.Upsert(prefixString, nil, 0, nil, Identity{
			ID:     id.ID,
			Source: source.Generated,
		})
	}
	if len(usedIdentities) == 0 {
		return
	}

	toUpsert := make(map[string]*identity.Identity)
	ipc.mutex.RLock()
	for _, id := range usedIdentities {
		prefix, ok := cidrLabelToPrefix(id.CIDRLabel.String())
		if !ok {
			log.WithFields(logrus.Fields{
				logfields.Identity: id.ID,
			}).Warning("BUG: Attempting to upsert non-CIDR identity")
			continue
		}
		if _, ok := ipc.LookupByIPRLocked(prefix); ok {
			// Already there; continue
			continue
		}
		toUpsert[prefix] = id
	}
	ipc.mutex.RUnlock()
	for prefix, id := range toUpsert {
		metrics.IPCacheErrorsTotal.WithLabelValues(
			metricTypeRecover, metricErrorUnexpected,
		).Inc()
		ipc.Upsert(prefix, nil, 0, nil, Identity{
			ID:     id.ID,
			Source: source.Generated,
		})
	}
}

// allocate will allocate a new identity for the given prefix based on the
// given set of labels. This function performs both global and local (CIDR)
// identity allocation and the set of labels determine which identity
// allocation type is to occur.
//
// If the identity is a CIDR identity, then its corresponding Identity will
// have its CIDR labels set correctly.
//
// A possible previously used numeric identity for these labels can be passed
// in as the 'oldNID' parameter; identity.InvalidIdentity must be passed if no
// previous numeric identity exists.
//
// It is up to the caller to provide the full set of labels for identity
// allocation.
func (ipc *IPCache) allocate(ctx context.Context, prefix netip.Prefix, lbls labels.Labels, oldNID identity.NumericIdentity) (*identity.Identity, bool, error) {
	id, isNew, err := ipc.IdentityAllocator.AllocateIdentity(ctx, lbls, false, oldNID)
	if err != nil {
		return nil, isNew, fmt.Errorf("failed to allocate identity for cidr %s: %s", prefix, err)
	}

	if lbls.Has(labels.LabelWorld[labels.IDNameWorld]) {
		id.CIDRLabel = labels.NewLabelsFromModel([]string{labels.LabelSourceCIDR + ":" + prefix.String()})
	}

	return id, isNew, err
}

func (ipc *IPCache) releaseCIDRIdentities(ctx context.Context, identities map[netip.Prefix]*identity.Identity) {
	// Create a critical section for identity release + removal from ipcache.
	// Otherwise, it's possible to trigger the following race condition:
	//
	// Goroutine 1                | Goroutine 2
	// releaseCIDRIdentities()    | AllocateCIDRs()
	// -> Release(..., id, ...)   |
	//                            | -> allocate(...)
	//                            | -> ipc.UpsertGeneratedIdentities(...)
	// -> ipc.deleteLocked(...)   |
	//
	// In this case, the expectation from Goroutine 2 is that an identity
	// is allocated and that identity is in the ipcache, but the result
	// is that the identity is allocated but the ipcache entry is missing.
	ipc.Lock()
	defer ipc.Unlock()
	for prefix, id := range identities {
		released, err := ipc.IdentityAllocator.Release(ctx, id, false)
		if err != nil {
			log.WithFields(logrus.Fields{
				logfields.Identity: id,
				logfields.CIDR:     prefix,
			}).WithError(err).Warning("Unable to release CIDR identity. Ignoring error. Identity may be leaked")
		}

		if released {
			ipc.deleteLocked(prefix.String(), source.Generated)
		}
	}
}

// ReleaseCIDRIdentitiesByCIDR releases the identities of a list of CIDRs.
// When the last use of the identity is released, the ipcache entry is deleted.
func (ipc *IPCache) ReleaseCIDRIdentitiesByCIDR(prefixes []netip.Prefix) {
	// TODO: Structure the code to pass context down from the Daemon.
	releaseCtx, cancel := context.WithTimeout(context.TODO(), option.Config.KVstoreConnectivityTimeout)
	defer cancel()

	identities := make(map[netip.Prefix]*identity.Identity, len(prefixes))
	for _, p := range prefixes {
		if id := ipc.IdentityAllocator.LookupIdentity(releaseCtx, cidr.GetCIDRLabels(p)); id != nil {
			identities[p] = id
		} else {
			log.Errorf("Unable to find identity of previously used CIDR %s", p.String())
		}
	}

	ipc.releaseCIDRIdentities(releaseCtx, identities)
}

// ReleaseCIDRIdentitiesByID releases the specified identities.
// When the last use of the identity is released, the ipcache entry is deleted.
func (ipc *IPCache) ReleaseCIDRIdentitiesByID(ctx context.Context, identities []identity.NumericIdentity) {
	fullIdentities := make(map[netip.Prefix]*identity.Identity, len(identities))
	for _, nid := range identities {
		if id := ipc.IdentityAllocator.LookupIdentityByID(ctx, nid); id != nil {
			cidr, ok := cidrLabelToPrefix(id.CIDRLabel.String())
			if !ok {
				log.WithFields(logrus.Fields{
					logfields.Identity: nid,
					logfields.Labels:   id.Labels,
				}).Warn("Unexpected release of non-CIDR identity, will leak this identity. Please report this issue to the developers.")
				continue
			}
			prefix, err := netip.ParsePrefix(strings.TrimPrefix(cidr, labels.LabelSourceCIDR+":"))
			if err != nil {
				log.WithFields(logrus.Fields{
					logfields.Identity: nid,
					logfields.Labels:   id.Labels,
				}).Warn("BUG: Cannot parse prefix from CIDR label during CIDR identity release. Please report this issue to the developers.")
				continue
			}
			fullIdentities[prefix] = id
		} else {
			log.WithFields(logrus.Fields{
				logfields.Identity: nid,
			}).Warn("Unexpected release of numeric identity that is no longer allocated")
		}
	}

	ipc.releaseCIDRIdentities(ctx, fullIdentities)
}
<commit_msg>ipcache: Fix metadata access from CIDR allocation<commit_after>// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium

package ipcache

import (
	"context"
	"fmt"
	"net"
	"net/netip"
	"strings"

	"github.com/sirupsen/logrus"

	"github.com/cilium/cilium/pkg/identity"
	"github.com/cilium/cilium/pkg/ip"
	"github.com/cilium/cilium/pkg/labels"
	"github.com/cilium/cilium/pkg/labels/cidr"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/metrics"
	"github.com/cilium/cilium/pkg/option"
	"github.com/cilium/cilium/pkg/source"
)

// AllocateCIDRs attempts to allocate identities for a list of CIDRs. If any
// allocation fails, all allocations are rolled back and the error is returned.
// When an identity is freshly allocated for a CIDR, it is added to the
// ipcache if 'newlyAllocatedIdentities' is 'nil', otherwise the newly allocated
// identities are placed in 'newlyAllocatedIdentities' and it is the caller's
// responsibility to upsert them into ipcache by calling UpsertGeneratedIdentities().
//
// Previously used numeric identities for the given prefixes may be passed in as the
// 'oldNIDs' parameter; nil slice must be passed if no previous numeric identities exist.
// Previously used NID is allocated if still available. Non-availability is not an error.
//
// Upon success, the caller must also arrange for the resulting identities to
// be released via a subsequent call to ReleaseCIDRIdentitiesByCIDR().
func (ipc *IPCache) AllocateCIDRs(
	prefixes []netip.Prefix, oldNIDs []identity.NumericIdentity, newlyAllocatedIdentities map[string]*identity.Identity,
) ([]*identity.Identity, error) {
	// maintain list of used identities to undo on error
	usedIdentities := make([]*identity.Identity, 0, len(prefixes))

	// Maintain list of newly allocated identities to update ipcache,
	// but upsert them to ipcache only if no map was given by the caller.
	upsert := false
	if newlyAllocatedIdentities == nil {
		upsert = true
		newlyAllocatedIdentities = map[string]*identity.Identity{}
	}

	allocateCtx, cancel := context.WithTimeout(context.Background(), option.Config.IPAllocationTimeout)
	defer cancel()

	ipc.metadata.RLock()
	ipc.Lock()
	allocatedIdentities := make(map[netip.Prefix]*identity.Identity, len(prefixes))
	for i, prefix := range prefixes {
		lbls := cidr.GetCIDRLabels(prefix)
		lbls.MergeLabels(ipc.metadata.getLocked(prefix).ToLabels())
		oldNID := identity.InvalidIdentity
		if oldNIDs != nil && len(oldNIDs) > i {
			oldNID = oldNIDs[i]
		}
		id, isNew, err := ipc.allocate(allocateCtx, prefix, lbls, oldNID)
		if err != nil {
			ipc.IdentityAllocator.ReleaseSlice(context.Background(), nil, usedIdentities)
			ipc.Unlock()
			return nil, err
		}

		usedIdentities = append(usedIdentities, id)
		allocatedIdentities[prefix] = id
		if isNew {
			newlyAllocatedIdentities[prefix.String()] = id
		}
	}
	ipc.Unlock()
	ipc.metadata.RUnlock()

	// Only upsert into ipcache if identity wasn't allocated
	// before and the caller does not care doing this
	if upsert {
		ipc.UpsertGeneratedIdentities(newlyAllocatedIdentities, nil)
	}

	identities := make([]*identity.Identity, 0, len(allocatedIdentities))
	for _, id := range allocatedIdentities {
		identities = append(identities, id)
	}
	return identities, nil
}

// AllocateCIDRsForIPs performs the same action as AllocateCIDRs but for IP
// addresses instead of CIDRs.
//
// Upon success, the caller must also arrange for the resulting identities to
// be released via a subsequent call to ReleaseCIDRIdentitiesByID().
func (ipc *IPCache) AllocateCIDRsForIPs(
	prefixes []net.IP, newlyAllocatedIdentities map[string]*identity.Identity,
) ([]*identity.Identity, error) {
	return ipc.AllocateCIDRs(ip.IPsToNetPrefixes(prefixes), nil, newlyAllocatedIdentities)
}

func cidrLabelToPrefix(label string) (string, bool) {
	if !strings.HasPrefix(label, labels.LabelSourceCIDR) {
		return "", false
	}
	return strings.TrimPrefix(label, labels.LabelSourceCIDR+":"), true
}

// UpsertGeneratedIdentities unconditionally upserts 'newlyAllocatedIdentities'
// into the ipcache, then also upserts any CIDR identities in 'usedIdentities'
// that were not already upserted. If any 'usedIdentities' are upserted, these
// are counted separately as they may provide an indication of another logic
// error elsewhere in the codebase that is causing premature ipcache deletions.
func (ipc *IPCache) UpsertGeneratedIdentities(newlyAllocatedIdentities map[string]*identity.Identity, usedIdentities []*identity.Identity) {
	for prefixString, id := range newlyAllocatedIdentities {
		ipc.Upsert(prefixString, nil, 0, nil, Identity{
			ID:     id.ID,
			Source: source.Generated,
		})
	}
	if len(usedIdentities) == 0 {
		return
	}

	toUpsert := make(map[string]*identity.Identity)
	ipc.mutex.RLock()
	for _, id := range usedIdentities {
		prefix, ok := cidrLabelToPrefix(id.CIDRLabel.String())
		if !ok {
			log.WithFields(logrus.Fields{
				logfields.Identity: id.ID,
			}).Warning("BUG: Attempting to upsert non-CIDR identity")
			continue
		}
		if _, ok := ipc.LookupByIPRLocked(prefix); ok {
			// Already there; continue
			continue
		}
		toUpsert[prefix] = id
	}
	ipc.mutex.RUnlock()
	for prefix, id := range toUpsert {
		metrics.IPCacheErrorsTotal.WithLabelValues(
			metricTypeRecover, metricErrorUnexpected,
		).Inc()
		ipc.Upsert(prefix, nil, 0, nil, Identity{
			ID:     id.ID,
			Source: source.Generated,
		})
	}
}

// allocate will allocate a new identity for the given prefix based on the
// given set of labels. This function performs both global and local (CIDR)
// identity allocation and the set of labels determine which identity
// allocation type is to occur.
//
// If the identity is a CIDR identity, then its corresponding Identity will
// have its CIDR labels set correctly.
//
// A possible previously used numeric identity for these labels can be passed
// in as the 'oldNID' parameter; identity.InvalidIdentity must be passed if no
// previous numeric identity exists.
//
// It is up to the caller to provide the full set of labels for identity
// allocation.
func (ipc *IPCache) allocate(ctx context.Context, prefix netip.Prefix, lbls labels.Labels, oldNID identity.NumericIdentity) (*identity.Identity, bool, error) {
	id, isNew, err := ipc.IdentityAllocator.AllocateIdentity(ctx, lbls, false, oldNID)
	if err != nil {
		return nil, isNew, fmt.Errorf("failed to allocate identity for cidr %s: %s", prefix, err)
	}

	if lbls.Has(labels.LabelWorld[labels.IDNameWorld]) {
		id.CIDRLabel = labels.NewLabelsFromModel([]string{labels.LabelSourceCIDR + ":" + prefix.String()})
	}

	return id, isNew, err
}

func (ipc *IPCache) releaseCIDRIdentities(ctx context.Context, identities map[netip.Prefix]*identity.Identity) {
	// Create a critical section for identity release + removal from ipcache.
	// Otherwise, it's possible to trigger the following race condition:
	//
	// Goroutine 1                | Goroutine 2
	// releaseCIDRIdentities()    | AllocateCIDRs()
	// -> Release(..., id, ...)   |
	//                            | -> allocate(...)
	//                            | -> ipc.UpsertGeneratedIdentities(...)
	// -> ipc.deleteLocked(...)   |
	//
	// In this case, the expectation from Goroutine 2 is that an identity
	// is allocated and that identity is in the ipcache, but the result
	// is that the identity is allocated but the ipcache entry is missing.
	ipc.Lock()
	defer ipc.Unlock()
	for prefix, id := range identities {
		released, err := ipc.IdentityAllocator.Release(ctx, id, false)
		if err != nil {
			log.WithFields(logrus.Fields{
				logfields.Identity: id,
				logfields.CIDR:     prefix,
			}).WithError(err).Warning("Unable to release CIDR identity. Ignoring error. Identity may be leaked")
		}

		if released {
			ipc.deleteLocked(prefix.String(), source.Generated)
		}
	}
}

// ReleaseCIDRIdentitiesByCIDR releases the identities of a list of CIDRs.
// When the last use of the identity is released, the ipcache entry is deleted.
func (ipc *IPCache) ReleaseCIDRIdentitiesByCIDR(prefixes []netip.Prefix) {
	// TODO: Structure the code to pass context down from the Daemon.
	releaseCtx, cancel := context.WithTimeout(context.TODO(), option.Config.KVstoreConnectivityTimeout)
	defer cancel()

	identities := make(map[netip.Prefix]*identity.Identity, len(prefixes))
	for _, p := range prefixes {
		if id := ipc.IdentityAllocator.LookupIdentity(releaseCtx, cidr.GetCIDRLabels(p)); id != nil {
			identities[p] = id
		} else {
			log.Errorf("Unable to find identity of previously used CIDR %s", p.String())
		}
	}

	ipc.releaseCIDRIdentities(releaseCtx, identities)
}

// ReleaseCIDRIdentitiesByID releases the specified identities.
// When the last use of the identity is released, the ipcache entry is deleted.
func (ipc *IPCache) ReleaseCIDRIdentitiesByID(ctx context.Context, identities []identity.NumericIdentity) {
	fullIdentities := make(map[netip.Prefix]*identity.Identity, len(identities))
	for _, nid := range identities {
		if id := ipc.IdentityAllocator.LookupIdentityByID(ctx, nid); id != nil {
			cidr, ok := cidrLabelToPrefix(id.CIDRLabel.String())
			if !ok {
				log.WithFields(logrus.Fields{
					logfields.Identity: nid,
					logfields.Labels:   id.Labels,
				}).Warn("Unexpected release of non-CIDR identity, will leak this identity. Please report this issue to the developers.")
				continue
			}
			prefix, err := netip.ParsePrefix(strings.TrimPrefix(cidr, labels.LabelSourceCIDR+":"))
			if err != nil {
				log.WithFields(logrus.Fields{
					logfields.Identity: nid,
					logfields.Labels:   id.Labels,
				}).Warn("BUG: Cannot parse prefix from CIDR label during CIDR identity release. Please report this issue to the developers.")
				continue
			}
			fullIdentities[prefix] = id
		} else {
			log.WithFields(logrus.Fields{
				logfields.Identity: nid,
			}).Warn("Unexpected release of numeric identity that is no longer allocated")
		}
	}

	ipc.releaseCIDRIdentities(ctx, fullIdentities)
}
<|endoftext|>


<commit_before>package oauth

import (
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/couchdb/mango"
	"github.com/cozy/cozy-stack/pkg/crypto"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/permissions"

	jwt "gopkg.in/dgrijalva/jwt-go.v3"
)

const (
	// AndroidPlatform platform using Firebase Cloud Messaging (FCM)
	AndroidPlatform = "android"
	// IOSPlatform platform using APNS/2
	IOSPlatform = "ios"
)

// ClientSecretLen is the number of random bytes used for generating the client secret
const ClientSecretLen = 24 // #nosec

// Client is a struct for OAuth2 client. Most of the fields are described in
// the OAuth 2.0 Dynamic Client Registration Protocol. The exception is
// `client_kind`, and it is an optional field.
// See https://tools.ietf.org/html/rfc7591
//
// CouchID and ClientID are the same. They are just two ways to serialize to
// JSON, one for CouchDB and the other for the Dynamic Client Registration
// Protocol.
type Client struct {
	CouchID  string `json:"_id,omitempty"`  // Generated by CouchDB
	CouchRev string `json:"_rev,omitempty"` // Generated by CouchDB

	ClientID          string `json:"client_id,omitempty"`                 // Same as CouchID
	ClientSecret      string `json:"client_secret,omitempty"`             // Generated by the server
	SecretExpiresAt   int    `json:"client_secret_expires_at"`            // Forced by the server to 0 (no expiration)
	RegistrationToken string `json:"registration_access_token,omitempty"` // Generated by the server

	RedirectURIs    []string `json:"redirect_uris"`              // Declared by the client (mandatory)
	GrantTypes      []string `json:"grant_types"`                // Forced by the server to ["authorization_code", "refresh_token"]
	ResponseTypes   []string `json:"response_types"`             // Forced by the server to ["code"]
	ClientName      string   `json:"client_name"`                // Declared by the client (mandatory)
	ClientKind      string   `json:"client_kind,omitempty"`      // Declared by the client (optional, can be "desktop", "mobile", "browser", etc.)
	ClientURI       string   `json:"client_uri,omitempty"`       // Declared by the client (optional)
	LogoURI         string   `json:"logo_uri,omitempty"`         // Declared by the client (optional)
	PolicyURI       string   `json:"policy_uri,omitempty"`       // Declared by the client (optional)
	SoftwareID      string   `json:"software_id"`                // Declared by the client (mandatory)
	SoftwareVersion string   `json:"software_version,omitempty"` // Declared by the client (optional)

	// Notifications parameters
	NotificationPlatform    string `json:"notification_platform,omitempty"`     // Declared by the client (optional)
	NotificationDeviceToken string `json:"notification_device_token,omitempty"` // Declared by the client (optional)

	// XXX omitempty does not work for time.Time, thus the interface{} type
	SynchronizedAt interface{} `json:"synchronized_at,omitempty"` // Date of the last synchronization, updated by /settings/synchronized
}

// ID returns the client qualified identifier
func (c *Client) ID() string { return c.CouchID }

// Rev returns the client revision
func (c *Client) Rev() string { return c.CouchRev }

// DocType returns the client document type
func (c *Client) DocType() string { return consts.OAuthClients }

// Clone implements couchdb.Doc
func (c *Client) Clone() couchdb.Doc {
	cloned := *c
	cloned.RedirectURIs = make([]string, len(c.RedirectURIs))
	copy(cloned.RedirectURIs, c.RedirectURIs)

	cloned.GrantTypes = make([]string, len(c.GrantTypes))
	copy(cloned.GrantTypes, c.GrantTypes)

	cloned.ResponseTypes = make([]string, len(c.ResponseTypes))
	copy(cloned.ResponseTypes, c.ResponseTypes)
	return &cloned
}

// SetID changes the client qualified identifier
func (c *Client) SetID(id string) { c.CouchID = id }

// SetRev changes the client revision
func (c *Client) SetRev(rev string) { c.CouchRev = rev }

// TransformIDAndRev makes the translation from the JSON of CouchDB to the
// one used in the dynamic client registration protocol
func (c *Client) TransformIDAndRev() {
	c.ClientID = c.CouchID
	c.CouchID = ""
	c.CouchRev = ""
}

// GetAll loads all the clients from the database, without the secrets
func GetAll(i *instance.Instance) ([]*Client, error) {
	var clients []*Client
	req := &couchdb.AllDocsRequest{Limit: 100}
	if err := couchdb.GetAllDocs(i, consts.OAuthClients, req, &clients); err != nil {
		return nil, err
	}
	for _, client := range clients {
		client.ClientSecret = ""
	}
	return clients, nil
}

// GetNotifiables loads al the clients from the database containing a non-empty
// `notification_plaform` field.
func GetNotifiables(i *instance.Instance) ([]*Client, error) {
	var clients []*Client
	req := &couchdb.FindRequest{
		UseIndex: "by-plaform-notification",
		Selector: mango.Exists("notification_platform"),
	}
	err := couchdb.FindDocs(i, consts.OAuthClients, req, &clients)
	if err != nil {
		return nil, err
	}
	return clients, nil
}

// FindClient loads a client from the database
func FindClient(i *instance.Instance, id string) (*Client, error) {
	var c Client
	if err := couchdb.GetDoc(i, consts.OAuthClients, id, &c); err != nil {
		i.Logger().Errorf("[oauth] Failed to find the client %s: %s", id, err)
		return nil, err
	}
	if c.ClientID == "" {
		c.ClientID = c.CouchID
	}
	return &c, nil
}

// ClientRegistrationError is a Client Registration Error Response, as described
// in the Client Dynamic Registration Protocol
// See https://tools.ietf.org/html/rfc7591#section-3.2.2 for errors
type ClientRegistrationError struct {
	Code        int    `json:"-"`
	Error       string `json:"error"`
	Description string `json:"error_description,omitempty"`
}

func (c *Client) checkMandatoryFields(i *instance.Instance) *ClientRegistrationError {
	if len(c.RedirectURIs) == 0 {
		return &ClientRegistrationError{
			Code:        http.StatusBadRequest,
			Error:       "invalid_redirect_uri",
			Description: "redirect_uris is mandatory",
		}
	}
	for _, redirectURI := range c.RedirectURIs {
		u, err := url.Parse(redirectURI)
		if err != nil ||
			u.Host == i.Domain ||
			u.Fragment != "" {
			return &ClientRegistrationError{
				Code:        http.StatusBadRequest,
				Error:       "invalid_redirect_uri",
				Description: fmt.Sprintf("%s is invalid", redirectURI),
			}
		}
	}
	if c.ClientName == "" {
		return &ClientRegistrationError{
			Code:        http.StatusBadRequest,
			Error:       "invalid_client_metadata",
			Description: "client_name is mandatory",
		}
	}
	if c.SoftwareID == "" {
		return &ClientRegistrationError{
			Code:        http.StatusBadRequest,
			Error:       "invalid_client_metadata",
			Description: "software_id is mandatory",
		}
	}
	c.NotificationPlatform = strings.ToLower(c.NotificationPlatform)
	switch c.NotificationPlatform {
	case "", AndroidPlatform, IOSPlatform:
	default:
		return &ClientRegistrationError{
			Code:  http.StatusBadRequest,
			Error: "invalid_client_metadata",
		}
	}
	return nil
}

// Create is a function that sets some fields, and then save it in Couch.
func (c *Client) Create(i *instance.Instance) *ClientRegistrationError {
	if err := c.checkMandatoryFields(i); err != nil {
		return err
	}

	var results []*Client
	req := &couchdb.FindRequest{
		UseIndex: "by-client-name",
		Selector: mango.StartWith("client_name", c.ClientName),
	}
	err := couchdb.FindDocs(i, consts.OAuthClients, req, &results)
	if err != nil && !couchdb.IsNoDatabaseError(err) {
		return &ClientRegistrationError{
			Code:  http.StatusInternalServerError,
			Error: "internal_server_error",
		}
	}

	// Find the correct suffix to apply to the client name in case it is already
	// used.
	suffix := ""
	if len(results) > 0 {
		n := 1
		found := false
		prefix := c.ClientName + "-"
		for _, r := range results {
			name := r.ClientName
			if name == c.ClientName {
				found = true
				continue
			}
			if !strings.HasPrefix(name, prefix) {
				continue
			}
			var m int
			m, err = strconv.Atoi(name[len(prefix):])
			if err == nil && m > n {
				n = m
			}
		}
		if found {
			suffix = strconv.Itoa(n + 1)
		}
	}
	if suffix != "" {
		c.ClientName = c.ClientName + "-" + suffix
	}

	c.CouchID = ""
	c.CouchRev = ""
	c.ClientID = ""
	secret := crypto.GenerateRandomBytes(ClientSecretLen)
	c.ClientSecret = string(crypto.Base64Encode(secret))
	c.SecretExpiresAt = 0
	c.RegistrationToken = ""
	c.GrantTypes = []string{"authorization_code", "refresh_token"}
	c.ResponseTypes = []string{"code"}

	if err = couchdb.CreateDoc(i, c); err != nil {
		return &ClientRegistrationError{
			Code:  http.StatusInternalServerError,
			Error: "internal_server_error",
		}
	}

	c.RegistrationToken, err = crypto.NewJWT(i.OAuthSecret, jwt.StandardClaims{
		Audience: permissions.RegistrationTokenAudience,
		Issuer:   i.Domain,
		IssuedAt: time.Now().Unix(),
		Subject:  c.CouchID,
	})
	if err != nil {
		i.Logger().Errorf("[oauth] Failed to create the registration access token: %s", err)
		return &ClientRegistrationError{
			Code:  http.StatusInternalServerError,
			Error: "internal_server_error",
		}
	}

	c.TransformIDAndRev()
	return nil
}

// Update will update the client metadata
func (c *Client) Update(i *instance.Instance, old *Client) *ClientRegistrationError {
	if c.ClientID != old.ClientID {
		return &ClientRegistrationError{
			Code:        http.StatusBadRequest,
			Error:       "invalid_client_id",
			Description: "client_id is mandatory",
		}
	}

	if err := c.checkMandatoryFields(i); err != nil {
		return err
	}

	switch c.ClientSecret {
	case "":
		c.ClientSecret = old.ClientSecret
	case old.ClientSecret:
		secret := crypto.GenerateRandomBytes(ClientSecretLen)
		c.ClientSecret = string(crypto.Base64Encode(secret))
	default:
		return &ClientRegistrationError{
			Code:        http.StatusBadRequest,
			Error:       "invalid_client_secret",
			Description: "client_secret is invalid",
		}
	}

	c.CouchID = old.CouchID
	c.CouchRev = old.CouchRev
	c.ClientName = old.ClientName
	c.ClientID = ""
	c.SecretExpiresAt = 0
	c.RegistrationToken = ""
	c.GrantTypes = []string{"authorization_code", "refresh_token"}
	c.ResponseTypes = []string{"code"}
	if c.NotificationPlatform == "" {
		c.NotificationPlatform = old.NotificationPlatform
	}
	if c.NotificationDeviceToken == "" {
		c.NotificationDeviceToken = old.NotificationDeviceToken
	}

	if err := couchdb.UpdateDoc(i, c); err != nil {
		return &ClientRegistrationError{
			Code:  http.StatusInternalServerError,
			Error: "internal_server_error",
		}
	}

	c.TransformIDAndRev()
	return nil
}

// Delete is a function that unregister a client
func (c *Client) Delete(i *instance.Instance) *ClientRegistrationError {
	if err := couchdb.DeleteDoc(i, c); err != nil {
		return &ClientRegistrationError{
			Code:  http.StatusInternalServerError,
			Error: "internal_server_error",
		}
	}
	return nil
}

// AcceptRedirectURI returns true if the given URI matches the registered
// redirect_uris
func (c *Client) AcceptRedirectURI(u string) bool {
	for _, uri := range c.RedirectURIs {
		if u == uri {
			return true
		}
	}
	return false
}

// CreateJWT returns a new JSON Web Token for the given instance and audience
func (c *Client) CreateJWT(i *instance.Instance, audience, scope string) (string, error) {
	token, err := crypto.NewJWT(i.OAuthSecret, permissions.Claims{
		StandardClaims: jwt.StandardClaims{
			Audience: audience,
			Issuer:   i.Domain,
			IssuedAt: crypto.Timestamp(),
			Subject:  c.CouchID,
		},
		Scope: scope,
	})
	if err != nil {
		i.Logger().Errorf("[oauth] Failed to create the %s token: %s", audience, err)
	}
	return token, err
}

// ValidToken checks that the JWT is valid and returns the associate claims
// It is expected to be used for registration token and refresh token, and
// it doesn't check when they were issued as they don't expire.
func (c *Client) ValidToken(i *instance.Instance, audience, token string) (permissions.Claims, bool) {
	claims := permissions.Claims{}
	if token == "" {
		return claims, false
	}
	keyFunc := func(token *jwt.Token) (interface{}, error) {
		return i.OAuthSecret, nil
	}
	if err := crypto.ParseJWT(token, keyFunc, &claims); err != nil {
		i.Logger().Errorf("[oauth] Failed to verify the %s token: %s", audience, err)
		return claims, false
	}
	if claims.Expired() {
		i.Logger().Errorf("[oauth] Failed to verify the %s token: expired", audience)
		return claims, false
	}
	// Note: the refresh and registration tokens don't expire, no need to check its issue date
	if claims.Audience != audience {
		i.Logger().Errorf("[oauth] Unexpected audience for %s token: %s", audience, claims.Audience)
		return claims, false
	}
	if claims.Issuer != i.Domain {
		i.Logger().Errorf("[oauth] Expected %s issuer for %s token, but was: %s", audience, i.Domain, claims.Issuer)
		return claims, false
	}
	if claims.Subject != c.CouchID {
		i.Logger().Errorf("[oauth] Expected %s subject for %s token, but was: %s", audience, c.CouchID, claims.Subject)
		return claims, false
	}
	return claims, true
}

var (
	_ couchdb.Doc = &Client{}
)
<commit_msg>Fix index name for notifiables clients<commit_after>package oauth

import (
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/couchdb/mango"
	"github.com/cozy/cozy-stack/pkg/crypto"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/permissions"

	jwt "gopkg.in/dgrijalva/jwt-go.v3"
)

const (
	// AndroidPlatform platform using Firebase Cloud Messaging (FCM)
	AndroidPlatform = "android"
	// IOSPlatform platform using APNS/2
	IOSPlatform = "ios"
)

// ClientSecretLen is the number of random bytes used for generating the client secret
const ClientSecretLen = 24 // #nosec

// Client is a struct for OAuth2 client. Most of the fields are described in
// the OAuth 2.0 Dynamic Client Registration Protocol. The exception is
// `client_kind`, and it is an optional field.
// See https://tools.ietf.org/html/rfc7591
//
// CouchID and ClientID are the same. They are just two ways to serialize to
// JSON, one for CouchDB and the other for the Dynamic Client Registration
// Protocol.
type Client struct {
	CouchID  string `json:"_id,omitempty"`  // Generated by CouchDB
	CouchRev string `json:"_rev,omitempty"` // Generated by CouchDB

	ClientID          string `json:"client_id,omitempty"`                 // Same as CouchID
	ClientSecret      string `json:"client_secret,omitempty"`             // Generated by the server
	SecretExpiresAt   int    `json:"client_secret_expires_at"`            // Forced by the server to 0 (no expiration)
	RegistrationToken string `json:"registration_access_token,omitempty"` // Generated by the server

	RedirectURIs    []string `json:"redirect_uris"`              // Declared by the client (mandatory)
	GrantTypes      []string `json:"grant_types"`                // Forced by the server to ["authorization_code", "refresh_token"]
	ResponseTypes   []string `json:"response_types"`             // Forced by the server to ["code"]
	ClientName      string   `json:"client_name"`                // Declared by the client (mandatory)
	ClientKind      string   `json:"client_kind,omitempty"`      // Declared by the client (optional, can be "desktop", "mobile", "browser", etc.)
	ClientURI       string   `json:"client_uri,omitempty"`       // Declared by the client (optional)
	LogoURI         string   `json:"logo_uri,omitempty"`         // Declared by the client (optional)
	PolicyURI       string   `json:"policy_uri,omitempty"`       // Declared by the client (optional)
	SoftwareID      string   `json:"software_id"`                // Declared by the client (mandatory)
	SoftwareVersion string   `json:"software_version,omitempty"` // Declared by the client (optional)

	// Notifications parameters
	NotificationPlatform    string `json:"notification_platform,omitempty"`     // Declared by the client (optional)
	NotificationDeviceToken string `json:"notification_device_token,omitempty"` // Declared by the client (optional)

	// XXX omitempty does not work for time.Time, thus the interface{} type
	SynchronizedAt interface{} `json:"synchronized_at,omitempty"` // Date of the last synchronization, updated by /settings/synchronized
}

// ID returns the client qualified identifier
func (c *Client) ID() string { return c.CouchID }

// Rev returns the client revision
func (c *Client) Rev() string { return c.CouchRev }

// DocType returns the client document type
func (c *Client) DocType() string { return consts.OAuthClients }

// Clone implements couchdb.Doc
func (c *Client) Clone() couchdb.Doc {
	cloned := *c
	cloned.RedirectURIs = make([]string, len(c.RedirectURIs))
	copy(cloned.RedirectURIs, c.RedirectURIs)

	cloned.GrantTypes = make([]string, len(c.GrantTypes))
	copy(cloned.GrantTypes, c.GrantTypes)

	cloned.ResponseTypes = make([]string, len(c.ResponseTypes))
	copy(cloned.ResponseTypes, c.ResponseTypes)
	return &cloned
}

// SetID changes the client qualified identifier
func (c *Client) SetID(id string) { c.CouchID = id }

// SetRev changes the client revision
func (c *Client) SetRev(rev string) { c.CouchRev = rev }

// TransformIDAndRev makes the translation from the JSON of CouchDB to the
// one used in the dynamic client registration protocol
func (c *Client) TransformIDAndRev() {
	c.ClientID = c.CouchID
	c.CouchID = ""
	c.CouchRev = ""
}

// GetAll loads all the clients from the database, without the secrets
func GetAll(i *instance.Instance) ([]*Client, error) {
	var clients []*Client
	req := &couchdb.AllDocsRequest{Limit: 100}
	if err := couchdb.GetAllDocs(i, consts.OAuthClients, req, &clients); err != nil {
		return nil, err
	}
	for _, client := range clients {
		client.ClientSecret = ""
	}
	return clients, nil
}

// GetNotifiables loads al the clients from the database containing a non-empty
// `notification_plaform` field.
func GetNotifiables(i *instance.Instance) ([]*Client, error) {
	var clients []*Client
	req := &couchdb.FindRequest{
		UseIndex: "by-notification-platform",
		Selector: mango.Exists("notification_platform"),
	}
	err := couchdb.FindDocs(i, consts.OAuthClients, req, &clients)
	if err != nil {
		return nil, err
	}
	return clients, nil
}

// FindClient loads a client from the database
func FindClient(i *instance.Instance, id string) (*Client, error) {
	var c Client
	if err := couchdb.GetDoc(i, consts.OAuthClients, id, &c); err != nil {
		i.Logger().Errorf("[oauth] Failed to find the client %s: %s", id, err)
		return nil, err
	}
	if c.ClientID == "" {
		c.ClientID = c.CouchID
	}
	return &c, nil
}

// ClientRegistrationError is a Client Registration Error Response, as described
// in the Client Dynamic Registration Protocol
// See https://tools.ietf.org/html/rfc7591#section-3.2.2 for errors
type ClientRegistrationError struct {
	Code        int    `json:"-"`
	Error       string `json:"error"`
	Description string `json:"error_description,omitempty"`
}

func (c *Client) checkMandatoryFields(i *instance.Instance) *ClientRegistrationError {
	if len(c.RedirectURIs) == 0 {
		return &ClientRegistrationError{
			Code:        http.StatusBadRequest,
			Error:       "invalid_redirect_uri",
			Description: "redirect_uris is mandatory",
		}
	}
	for _, redirectURI := range c.RedirectURIs {
		u, err := url.Parse(redirectURI)
		if err != nil ||
			u.Host == i.Domain ||
			u.Fragment != "" {
			return &ClientRegistrationError{
				Code:        http.StatusBadRequest,
				Error:       "invalid_redirect_uri",
				Description: fmt.Sprintf("%s is invalid", redirectURI),
			}
		}
	}
	if c.ClientName == "" {
		return &ClientRegistrationError{
			Code:        http.StatusBadRequest,
			Error:       "invalid_client_metadata",
			Description: "client_name is mandatory",
		}
	}
	if c.SoftwareID == "" {
		return &ClientRegistrationError{
			Code:        http.StatusBadRequest,
			Error:       "invalid_client_metadata",
			Description: "software_id is mandatory",
		}
	}
	c.NotificationPlatform = strings.ToLower(c.NotificationPlatform)
	switch c.NotificationPlatform {
	case "", AndroidPlatform, IOSPlatform:
	default:
		return &ClientRegistrationError{
			Code:  http.StatusBadRequest,
			Error: "invalid_client_metadata",
		}
	}
	return nil
}

// Create is a function that sets some fields, and then save it in Couch.
func (c *Client) Create(i *instance.Instance) *ClientRegistrationError {
	if err := c.checkMandatoryFields(i); err != nil {
		return err
	}

	var results []*Client
	req := &couchdb.FindRequest{
		UseIndex: "by-client-name",
		Selector: mango.StartWith("client_name", c.ClientName),
	}
	err := couchdb.FindDocs(i, consts.OAuthClients, req, &results)
	if err != nil && !couchdb.IsNoDatabaseError(err) {
		return &ClientRegistrationError{
			Code:  http.StatusInternalServerError,
			Error: "internal_server_error",
		}
	}

	// Find the correct suffix to apply to the client name in case it is already
	// used.
	suffix := ""
	if len(results) > 0 {
		n := 1
		found := false
		prefix := c.ClientName + "-"
		for _, r := range results {
			name := r.ClientName
			if name == c.ClientName {
				found = true
				continue
			}
			if !strings.HasPrefix(name, prefix) {
				continue
			}
			var m int
			m, err = strconv.Atoi(name[len(prefix):])
			if err == nil && m > n {
				n = m
			}
		}
		if found {
			suffix = strconv.Itoa(n + 1)
		}
	}
	if suffix != "" {
		c.ClientName = c.ClientName + "-" + suffix
	}

	c.CouchID = ""
	c.CouchRev = ""
	c.ClientID = ""
	secret := crypto.GenerateRandomBytes(ClientSecretLen)
	c.ClientSecret = string(crypto.Base64Encode(secret))
	c.SecretExpiresAt = 0
	c.RegistrationToken = ""
	c.GrantTypes = []string{"authorization_code", "refresh_token"}
	c.ResponseTypes = []string{"code"}

	if err = couchdb.CreateDoc(i, c); err != nil {
		return &ClientRegistrationError{
			Code:  http.StatusInternalServerError,
			Error: "internal_server_error",
		}
	}

	c.RegistrationToken, err = crypto.NewJWT(i.OAuthSecret, jwt.StandardClaims{
		Audience: permissions.RegistrationTokenAudience,
		Issuer:   i.Domain,
		IssuedAt: time.Now().Unix(),
		Subject:  c.CouchID,
	})
	if err != nil {
		i.Logger().Errorf("[oauth] Failed to create the registration access token: %s", err)
		return &ClientRegistrationError{
			Code:  http.StatusInternalServerError,
			Error: "internal_server_error",
		}
	}

	c.TransformIDAndRev()
	return nil
}

// Update will update the client metadata
func (c *Client) Update(i *instance.Instance, old *Client) *ClientRegistrationError {
	if c.ClientID != old.ClientID {
		return &ClientRegistrationError{
			Code:        http.StatusBadRequest,
			Error:       "invalid_client_id",
			Description: "client_id is mandatory",
		}
	}

	if err := c.checkMandatoryFields(i); err != nil {
		return err
	}

	switch c.ClientSecret {
	case "":
		c.ClientSecret = old.ClientSecret
	case old.ClientSecret:
		secret := crypto.GenerateRandomBytes(ClientSecretLen)
		c.ClientSecret = string(crypto.Base64Encode(secret))
	default:
		return &ClientRegistrationError{
			Code:        http.StatusBadRequest,
			Error:       "invalid_client_secret",
			Description: "client_secret is invalid",
		}
	}

	c.CouchID = old.CouchID
	c.CouchRev = old.CouchRev
	c.ClientName = old.ClientName
	c.ClientID = ""
	c.SecretExpiresAt = 0
	c.RegistrationToken = ""
	c.GrantTypes = []string{"authorization_code", "refresh_token"}
	c.ResponseTypes = []string{"code"}
	if c.NotificationPlatform == "" {
		c.NotificationPlatform = old.NotificationPlatform
	}
	if c.NotificationDeviceToken == "" {
		c.NotificationDeviceToken = old.NotificationDeviceToken
	}

	if err := couchdb.UpdateDoc(i, c); err != nil {
		return &ClientRegistrationError{
			Code:  http.StatusInternalServerError,
			Error: "internal_server_error",
		}
	}

	c.TransformIDAndRev()
	return nil
}

// Delete is a function that unregister a client
func (c *Client) Delete(i *instance.Instance) *ClientRegistrationError {
	if err := couchdb.DeleteDoc(i, c); err != nil {
		return &ClientRegistrationError{
			Code:  http.StatusInternalServerError,
			Error: "internal_server_error",
		}
	}
	return nil
}

// AcceptRedirectURI returns true if the given URI matches the registered
// redirect_uris
func (c *Client) AcceptRedirectURI(u string) bool {
	for _, uri := range c.RedirectURIs {
		if u == uri {
			return true
		}
	}
	return false
}

// CreateJWT returns a new JSON Web Token for the given instance and audience
func (c *Client) CreateJWT(i *instance.Instance, audience, scope string) (string, error) {
	token, err := crypto.NewJWT(i.OAuthSecret, permissions.Claims{
		StandardClaims: jwt.StandardClaims{
			Audience: audience,
			Issuer:   i.Domain,
			IssuedAt: crypto.Timestamp(),
			Subject:  c.CouchID,
		},
		Scope: scope,
	})
	if err != nil {
		i.Logger().Errorf("[oauth] Failed to create the %s token: %s", audience, err)
	}
	return token, err
}

// ValidToken checks that the JWT is valid and returns the associate claims
// It is expected to be used for registration token and refresh token, and
// it doesn't check when they were issued as they don't expire.
func (c *Client) ValidToken(i *instance.Instance, audience, token string) (permissions.Claims, bool) {
	claims := permissions.Claims{}
	if token == "" {
		return claims, false
	}
	keyFunc := func(token *jwt.Token) (interface{}, error) {
		return i.OAuthSecret, nil
	}
	if err := crypto.ParseJWT(token, keyFunc, &claims); err != nil {
		i.Logger().Errorf("[oauth] Failed to verify the %s token: %s", audience, err)
		return claims, false
	}
	if claims.Expired() {
		i.Logger().Errorf("[oauth] Failed to verify the %s token: expired", audience)
		return claims, false
	}
	// Note: the refresh and registration tokens don't expire, no need to check its issue date
	if claims.Audience != audience {
		i.Logger().Errorf("[oauth] Unexpected audience for %s token: %s", audience, claims.Audience)
		return claims, false
	}
	if claims.Issuer != i.Domain {
		i.Logger().Errorf("[oauth] Expected %s issuer for %s token, but was: %s", audience, i.Domain, claims.Issuer)
		return claims, false
	}
	if claims.Subject != c.CouchID {
		i.Logger().Errorf("[oauth] Expected %s subject for %s token, but was: %s", audience, c.CouchID, claims.Subject)
		return claims, false
	}
	return claims, true
}

var (
	_ couchdb.Doc = &Client{}
)
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package version

// Base version information.
//
// This is the fallback data used when version information from git is not
// provided via go ldflags. It provides an approximation of the Kubernetes
// version for ad-hoc builds (e.g. `go build`) that cannot get the version
// information from git.
//
// The "-dev" suffix in the version info indicates that fact, and it means the
// current build is from a version greater that version. For example, v0.7-dev
// means version > 0.7 and < 0.8. (There's exceptions to this rule, see
// docs/releasing.md for more details.)
//
// When releasing a new Kubernetes version, this file should be updated to
// reflect the new version, and then a git annotated tag (using format vX.Y
// where X == Major version and Y == Minor version) should be created to point
// to the commit that updates pkg/version/base.go

var (
	// TODO: Deprecate gitMajor and gitMinor, use only gitVersion instead.
	gitMajor     string = "0"              // major version, always numeric
	gitMinor     string = "3"              // minor version, numeric possibly followed by "+"
	gitVersion   string = "v0.3"           // version from git, output of $(git describe)
	gitCommit    string = ""               // sha1 from git, output of $(git rev-parse HEAD)
	gitTreeState string = "not a git tree" // state of git tree, either "clean" or "dirty"
)
<commit_msg>Add the 0.3-dev commit.<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package version

// Base version information.
//
// This is the fallback data used when version information from git is not
// provided via go ldflags. It provides an approximation of the Kubernetes
// version for ad-hoc builds (e.g. `go build`) that cannot get the version
// information from git.
//
// The "-dev" suffix in the version info indicates that fact, and it means the
// current build is from a version greater that version. For example, v0.7-dev
// means version > 0.7 and < 0.8. (There's exceptions to this rule, see
// docs/releasing.md for more details.)
//
// When releasing a new Kubernetes version, this file should be updated to
// reflect the new version, and then a git annotated tag (using format vX.Y
// where X == Major version and Y == Minor version) should be created to point
// to the commit that updates pkg/version/base.go

var (
	// TODO: Deprecate gitMajor and gitMinor, use only gitVersion instead.
	gitMajor     string = "0"              // major version, always numeric
	gitMinor     string = "3+"             // minor version, numeric possibly followed by "+"
	gitVersion   string = "v0.3-dev"       // version from git, output of $(git describe)
	gitCommit    string = ""               // sha1 from git, output of $(git rev-parse HEAD)
	gitTreeState string = "not a git tree" // state of git tree, either "clean" or "dirty"
)
<|endoftext|>