<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsx

import (
	"fmt"
	"io"
	"math"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

// We will not send a request to GCS for less than this many bytes (unless the
// end of the object comes first).
const minReadSize = 1 << 20

// We will skip forwards in a GCS response at most this many bytes.
const skipForwardThreshold = 1 << 20

// An object that knows how to read ranges within a particular generation of a
// particular GCS object. May make optimizations when it e.g. detects large
// sequential reads.
//
// Not safe for concurrent access.
type RandomReader interface {
	// Panic if any internal invariants are violated.
	CheckInvariants()

	// Matches the semantics of io.ReaderAt, with the addition of context
	// support.
	ReadAt(ctx context.Context, p []byte, offset int64) (n int, err error)

	// Return the record for the object to which the reader is bound.
	Object() (o *gcs.Object)

	// Clean up any resources associated with the reader, which must not be used
	// again.
	Destroy()
}

// Create a random reader for the supplied object record that reads using the
// given bucket.
func NewRandomReader(
	o *gcs.Object,
	bucket gcs.Bucket) (rr RandomReader, err error) {
	rr = &randomReader{
		object: o,
		bucket: bucket,
		start:  -1,
		limit:  -1,
	}

	return
}

type randomReader struct {
	object *gcs.Object
	bucket gcs.Bucket

	// If non-nil, an in-flight read request and a function for cancelling it.
	//
	// INVARIANT: (reader == nil) == (cancel == nil)
	reader io.ReadCloser
	cancel func()

	// The range of the object that we expect reader to yield, when reader is
	// non-nil. When reader is nil, limit is the limit of the previous read
	// operation, or -1 if there has never been one.
	//
	// INVARIANT: start <= limit
	// INVARIANT: limit < 0 implies reader != nil
	start int64
	limit int64
}

func (rr *randomReader) CheckInvariants() {
	// INVARIANT: (reader == nil) == (cancel == nil)
	if (rr.reader == nil) != (rr.cancel == nil) {
		panic(fmt.Sprintf("Mismatch: %v vs. %v", rr.reader == nil, rr.cancel == nil))
	}

	// INVARIANT: start <= limit
	if !(rr.start <= rr.limit) {
		panic(fmt.Sprintf("Unexpected range: [%d, %d)", rr.start, rr.limit))
	}

	// INVARIANT: limit < 0 implies reader != nil
	if rr.limit < 0 && rr.reader != nil {
		panic(fmt.Sprintf("Unexpected non-nil reader with limit == %d", rr.limit))
	}
}

func (rr *randomReader) ReadAt(
	ctx context.Context,
	p []byte,
	offset int64) (n int, err error) {
	for len(p) > 0 {
		// Have we blown past the end of the object?
		if offset >= int64(rr.object.Size) {
			err = io.EOF
			return
		}

		// When the offset is AFTER the reader position, try to seek forward, within reason.
		// This happens when the kernel page cache serves some data. It's very common for
		// concurrent reads, often by only a few 128kB fuse read requests. The aim is to
		// re-use GCS connection and avoid throwing away already read data.
		// For parallel sequential reads to a single file, not throwing away the connections
		// is a 15x improvement in throughput.
		if rr.reader != nil && rr.start < offset && offset - rr.start < skipForwardThreshold {
		        bytes_to_skip := int64(offset - rr.start)
		        p := make([]byte, bytes_to_skip)
		        n, _ := rr.reader.Read(p)
		        rr.start += int64(n)
		}

		// If we have an existing reader but it's positioned at the wrong place,
		// clean it up and throw it away.
		if rr.reader != nil && rr.start != offset {
			rr.reader.Close()
			rr.reader = nil
			rr.cancel = nil
		}

		// If we don't have a reader, start a read operation.
		if rr.reader == nil {
			err = rr.startRead(offset, int64(len(p)))
			if err != nil {
				err = fmt.Errorf("startRead: %v", err)
				return
			}
		}

		// Now we have a reader positioned at the correct place. Consume as much from
		// it as possible.
		var tmp int
		tmp, err = rr.readFull(ctx, p)

		n += tmp
		p = p[tmp:]
		rr.start += int64(tmp)
		offset += int64(tmp)

		// Sanity check.
		if rr.start > rr.limit {
			err = fmt.Errorf("Reader returned %d too many bytes", rr.start-rr.limit)

			// Don't attempt to reuse the reader when it's behaving wackily.
			rr.reader.Close()
			rr.reader = nil
			rr.cancel = nil
			rr.start = -1
			rr.limit = -1

			return
		}

		// Are we finished with this reader now?
		if rr.start == rr.limit {
			rr.reader.Close()
			rr.reader = nil
			rr.cancel = nil
		}

		// Handle errors.
		switch {
		case err == io.EOF || err == io.ErrUnexpectedEOF:
			// For a non-empty buffer, ReadFull returns EOF or ErrUnexpectedEOF only
			// if the reader peters out early. That's fine, but it means we should
			// have hit the limit above.
			if rr.reader != nil {
				err = fmt.Errorf("Reader returned %d too few bytes", rr.limit-rr.start)
				return
			}

			err = nil

		case err != nil:
			// Propagate other errors.
			err = fmt.Errorf("readFull: %v", err)
			return
		}
	}

	return
}

func (rr *randomReader) Object() (o *gcs.Object) {
	o = rr.object
	return
}

func (rr *randomReader) Destroy() {
	// Close out the reader, if we have one.
	if rr.reader != nil {
		rr.reader.Close()
		rr.reader = nil
		rr.cancel = nil
	}
}

// Like io.ReadFull, but deals with the cancellation issues.
//
// REQUIRES: rr.reader != nil
func (rr *randomReader) readFull(
	ctx context.Context,
	p []byte) (n int, err error) {
	// Start a goroutine that will cancel the read operation we block on below if
	// the calling context is cancelled, but only if this method has not already
	// returned (to avoid souring the reader for the next read if this one is
	// successful, since the calling context will eventually be cancelled).
	readDone := make(chan struct{})
	defer close(readDone)

	go func() {
		select {
		case <-readDone:
			return

		case <-ctx.Done():
			select {
			case <-readDone:
				return

			default:
				rr.cancel()
			}
		}
	}()

	// Call through.
	n, err = io.ReadFull(rr.reader, p)

	return
}

// Ensure that rr.reader is set up for a range for which [start, start+size) is
// a prefix.
func (rr *randomReader) startRead(
	start int64,
	size int64) (err error) {
	// Make sure start and size are legal.
	if start < 0 || uint64(start) > rr.object.Size || size < 0 {
		err = fmt.Errorf(
			"Range [%d, %d) is illegal for %d-byte object",
			start,
			start+size,
			rr.object.Size)
		return
	}

	// We always read a decent amount from GCS, no matter how silly small the
	// user's read is, because GCS requests are expensive.
	actualSize := int64(size)
	if actualSize < minReadSize {
		actualSize = minReadSize
	}

	// If this read starts where the previous one left off, we take this as a
	// sign that the user is reading sequentially within the object. It's
	// probably worth it to just request the entire rest of the object, and let
	// them sip from the fire house with each call to ReadAt.
	if start == rr.limit {
		actualSize = math.MaxInt64
	}

	// Clip to the end of the object.
	if actualSize > int64(rr.object.Size)-start {
		actualSize = int64(rr.object.Size) - start
	}

	// Begin the read.
	ctx, cancel := context.WithCancel(context.Background())
	rc, err := rr.bucket.NewReader(
		ctx,
		&gcs.ReadObjectRequest{
			Name:       rr.object.Name,
			Generation: rr.object.Generation,
			Range: &gcs.ByteRange{
				Start: uint64(start),
				Limit: uint64(start + actualSize),
			},
		})

	if err != nil {
		err = fmt.Errorf("NewReader: %v", err)
		return
	}

	rr.reader = rc
	rr.cancel = cancel
	rr.start = start
	rr.limit = start + actualSize

	return
}
<commit_msg>Always request reads to the end of the object.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsx

import (
	"fmt"
	"io"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

// We will skip forwards in a GCS response at most this many bytes.
// About 6 MB of data is buffered anyway, so 8 MB seems like a good round number.
const skipForwardThreshold = 8 * 1 << 20

// An object that knows how to read ranges within a particular generation of a
// particular GCS object. Optimised for (large) sequential reads.
//
// Not safe for concurrent access.
type RandomReader interface {
	// Panic if any internal invariants are violated.
	CheckInvariants()

	// Matches the semantics of io.ReaderAt, with the addition of context
	// support.
	ReadAt(ctx context.Context, p []byte, offset int64) (n int, err error)

	// Return the record for the object to which the reader is bound.
	Object() (o *gcs.Object)

	// Clean up any resources associated with the reader, which must not be used
	// again.
	Destroy()
}

// Create a random reader for the supplied object record that reads using the
// given bucket.
func NewRandomReader(
	o *gcs.Object,
	bucket gcs.Bucket) (rr RandomReader, err error) {
	rr = &randomReader{
		object: o,
		bucket: bucket,
		start:  -1,
		limit:  -1,
	}

	return
}

type randomReader struct {
	object *gcs.Object
	bucket gcs.Bucket

	// If non-nil, an in-flight read request and a function for cancelling it.
	//
	// INVARIANT: (reader == nil) == (cancel == nil)
	reader io.ReadCloser
	cancel func()

	// The range of the object that we expect reader to yield, when reader is
	// non-nil. When reader is nil, limit is the limit of the previous read
	// operation, or -1 if there has never been one.
	//
	// INVARIANT: start <= limit
	// INVARIANT: limit < 0 implies reader != nil
	start int64
	limit int64
}

func (rr *randomReader) CheckInvariants() {
	// INVARIANT: (reader == nil) == (cancel == nil)
	if (rr.reader == nil) != (rr.cancel == nil) {
		panic(fmt.Sprintf("Mismatch: %v vs. %v", rr.reader == nil, rr.cancel == nil))
	}

	// INVARIANT: start <= limit
	if !(rr.start <= rr.limit) {
		panic(fmt.Sprintf("Unexpected range: [%d, %d)", rr.start, rr.limit))
	}

	// INVARIANT: limit < 0 implies reader != nil
	if rr.limit < 0 && rr.reader != nil {
		panic(fmt.Sprintf("Unexpected non-nil reader with limit == %d", rr.limit))
	}
}

func (rr *randomReader) ReadAt(
	ctx context.Context,
	p []byte,
	offset int64) (n int, err error) {
	for len(p) > 0 {
		// Have we blown past the end of the object?
		if offset >= int64(rr.object.Size) {
			err = io.EOF
			return
		}

		// When the offset is AFTER the reader position, try to seek forward, within reason.
		// This happens when the kernel page cache serves some data. It's very common for
		// concurrent reads, often by only a few 128kB fuse read requests. The aim is to
		// re-use GCS connection and avoid throwing away already read data.
		// For parallel sequential reads to a single file, not throwing away the connections
		// is a 15x improvement in throughput.
		if rr.reader != nil && rr.start < offset && offset - rr.start < skipForwardThreshold {
		        bytes_to_skip := int64(offset - rr.start)
		        p := make([]byte, bytes_to_skip)
		        n, _ := rr.reader.Read(p)
		        rr.start += int64(n)
		}

		// If we have an existing reader but it's positioned at the wrong place,
		// clean it up and throw it away.
		if rr.reader != nil && rr.start != offset {
			rr.reader.Close()
			rr.reader = nil
			rr.cancel = nil
		}

		// If we don't have a reader, start a read operation.
		if rr.reader == nil {
			err = rr.startRead(offset, int64(len(p)))
			if err != nil {
				err = fmt.Errorf("startRead: %v", err)
				return
			}
		}

		// Now we have a reader positioned at the correct place. Consume as much from
		// it as possible.
		var tmp int
		tmp, err = rr.readFull(ctx, p)

		n += tmp
		p = p[tmp:]
		rr.start += int64(tmp)
		offset += int64(tmp)

		// Sanity check.
		if rr.start > rr.limit {
			err = fmt.Errorf("Reader returned %d too many bytes", rr.start-rr.limit)

			// Don't attempt to reuse the reader when it's behaving wackily.
			rr.reader.Close()
			rr.reader = nil
			rr.cancel = nil
			rr.start = -1
			rr.limit = -1

			return
		}

		// Are we finished with this reader now?
		if rr.start == rr.limit {
			rr.reader.Close()
			rr.reader = nil
			rr.cancel = nil
		}

		// Handle errors.
		switch {
		case err == io.EOF || err == io.ErrUnexpectedEOF:
			// For a non-empty buffer, ReadFull returns EOF or ErrUnexpectedEOF only
			// if the reader peters out early. That's fine, but it means we should
			// have hit the limit above.
			if rr.reader != nil {
				err = fmt.Errorf("Reader returned %d too few bytes", rr.limit-rr.start)
				return
			}

			err = nil

		case err != nil:
			// Propagate other errors.
			err = fmt.Errorf("readFull: %v", err)
			return
		}
	}

	return
}

func (rr *randomReader) Object() (o *gcs.Object) {
	o = rr.object
	return
}

func (rr *randomReader) Destroy() {
	// Close out the reader, if we have one.
	if rr.reader != nil {
		rr.reader.Close()
		rr.reader = nil
		rr.cancel = nil
	}
}

// Like io.ReadFull, but deals with the cancellation issues.
//
// REQUIRES: rr.reader != nil
func (rr *randomReader) readFull(
	ctx context.Context,
	p []byte) (n int, err error) {
	// Start a goroutine that will cancel the read operation we block on below if
	// the calling context is cancelled, but only if this method has not already
	// returned (to avoid souring the reader for the next read if this one is
	// successful, since the calling context will eventually be cancelled).
	readDone := make(chan struct{})
	defer close(readDone)

	go func() {
		select {
		case <-readDone:
			return

		case <-ctx.Done():
			select {
			case <-readDone:
				return

			default:
				rr.cancel()
			}
		}
	}()

	// Call through.
	n, err = io.ReadFull(rr.reader, p)

	return
}

// Ensure that rr.reader is set up for a range for which [start, start+size) is
// a prefix.
func (rr *randomReader) startRead(
	start int64,
	size int64) (err error) {
	// Make sure start and size are legal.
	if start < 0 || uint64(start) > rr.object.Size || size < 0 {
		err = fmt.Errorf(
			"Range [%d, %d) is illegal for %d-byte object",
			start,
			start+size,
			rr.object.Size)
		return
	}

	// GCS requests are expensive. Always issue read requests to the end of
	// the object. Sequential reads will simply sip from the fire house
	// with each call to ReadAt. In practice, GCS will fill the TCP buffers
	// with about 6 MB of data. Requests from outside GCP will be charged
	// about 6MB of egress data, even if less data is read. Inside GCP
	// regions, GCS egress is free. This logic should limit the number of
	// GCS read requests, which are not free.

	// TODO: Investigate impact on random read workloads.

	// Begin the read.
	ctx, cancel := context.WithCancel(context.Background())
	rc, err := rr.bucket.NewReader(
		ctx,
		&gcs.ReadObjectRequest{
			Name:       rr.object.Name,
			Generation: rr.object.Generation,
			Range: &gcs.ByteRange{
				Start: uint64(start),
				Limit: uint64(rr.object.Size),
			},
		})

	if err != nil {
		err = fmt.Errorf("NewReader: %v", err)
		return
	}

	rr.reader = rc
	rr.cancel = cancel
	rr.start = start
	rr.limit = int64(rr.object.Size)

	return
}
<|endoftext|>


<commit_before>// Code generated by go-bindata.
// sources:
// resources/pictures.sql
// DO NOT EDIT!

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

// bindataRead reads the given file from disk. It returns an error on failure.
func bindataRead(path, name string) ([]byte, error) {
	buf, err := ioutil.ReadFile(path)
	if err != nil {
		err = fmt.Errorf("Error reading asset %s at %s: %v", name, path, err)
	}
	return buf, err
}

type asset struct {
	bytes []byte
	info  os.FileInfo
}

// resourcesPicturesSql reads file data from disk. It returns an error on failure.
func resourcesPicturesSql() (*asset, error) {
	path := "/Users/jlindsey/projects/go/src/github.com/SnapShotsApp/ibex/resources/pictures.sql"
	name := "resources/pictures.sql"
	bytes, err := bindataRead(path, name)
	if err != nil {
		return nil, err
	}

	fi, err := os.Stat(path)
	if err != nil {
		err = fmt.Errorf("Error reading asset info %s at %s: %v", name, path, err)
	}

	a := &asset{bytes: bytes, info: fi}
	return a, err
}

// Asset loads and returns the asset for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func Asset(name string) ([]byte, error) {
	cannonicalName := strings.Replace(name, "\\", "/", -1)
	if f, ok := _bindata[cannonicalName]; ok {
		a, err := f()
		if err != nil {
			return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err)
		}
		return a.bytes, nil
	}
	return nil, fmt.Errorf("Asset %s not found", name)
}

// MustAsset is like Asset but panics when Asset would return an error.
// It simplifies safe initialization of global variables.
func MustAsset(name string) []byte {
	a, err := Asset(name)
	if err != nil {
		panic("asset: Asset(" + name + "): " + err.Error())
	}

	return a
}

// AssetInfo loads and returns the asset info for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func AssetInfo(name string) (os.FileInfo, error) {
	cannonicalName := strings.Replace(name, "\\", "/", -1)
	if f, ok := _bindata[cannonicalName]; ok {
		a, err := f()
		if err != nil {
			return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err)
		}
		return a.info, nil
	}
	return nil, fmt.Errorf("AssetInfo %s not found", name)
}

// AssetNames returns the names of the assets.
func AssetNames() []string {
	names := make([]string, 0, len(_bindata))
	for name := range _bindata {
		names = append(names, name)
	}
	return names
}

// _bindata is a table, holding each asset generator, mapped to its name.
var _bindata = map[string]func() (*asset, error){
	"resources/pictures.sql": resourcesPicturesSql,
}

// AssetDir returns the file names below a certain
// directory embedded in the file by go-bindata.
// For example if you run go-bindata on data/... and data contains the
// following hierarchy:
//     data/
//       foo.txt
//       img/
//         a.png
//         b.png
// then AssetDir("data") would return []string{"foo.txt", "img"}
// AssetDir("data/img") would return []string{"a.png", "b.png"}
// AssetDir("foo.txt") and AssetDir("notexist") would return an error
// AssetDir("") will return []string{"data"}.
func AssetDir(name string) ([]string, error) {
	node := _bintree
	if len(name) != 0 {
		cannonicalName := strings.Replace(name, "\\", "/", -1)
		pathList := strings.Split(cannonicalName, "/")
		for _, p := range pathList {
			node = node.Children[p]
			if node == nil {
				return nil, fmt.Errorf("Asset %s not found", name)
			}
		}
	}
	if node.Func != nil {
		return nil, fmt.Errorf("Asset %s not found", name)
	}
	rv := make([]string, 0, len(node.Children))
	for childName := range node.Children {
		rv = append(rv, childName)
	}
	return rv, nil
}

type bintree struct {
	Func     func() (*asset, error)
	Children map[string]*bintree
}
var _bintree = &bintree{nil, map[string]*bintree{
	"resources": &bintree{nil, map[string]*bintree{
		"pictures.sql": &bintree{resourcesPicturesSql, map[string]*bintree{}},
	}},
}}

// RestoreAsset restores an asset under the given directory
func RestoreAsset(dir, name string) error {
	data, err := Asset(name)
	if err != nil {
		return err
	}
	info, err := AssetInfo(name)
	if err != nil {
		return err
	}
	err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755))
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode())
	if err != nil {
		return err
	}
	err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime())
	if err != nil {
		return err
	}
	return nil
}

// RestoreAssets restores an asset under the given directory recursively
func RestoreAssets(dir, name string) error {
	children, err := AssetDir(name)
	// File
	if err != nil {
		return RestoreAsset(dir, name)
	}
	// Dir
	for _, child := range children {
		err = RestoreAssets(dir, filepath.Join(name, child))
		if err != nil {
			return err
		}
	}
	return nil
}

func _filePath(dir, name string) string {
	cannonicalName := strings.Replace(name, "\\", "/", -1)
	return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
}

<commit_msg>Remove generated file<commit_after><|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pubsub

import (
	"net/http"
	"strings"
	"testing"
	"time"

	"google.golang.org/cloud"
)

func TestIsSec(t *testing.T) {
	tests := map[time.Duration]bool{
		time.Second:                    true,
		5 * time.Second:                true,
		time.Hour:                      true,
		time.Millisecond:               false,
		time.Second + time.Microsecond: false,
	}
	for dur, expected := range tests {
		if isSec(dur) != expected {
			t.Errorf("%v is more precise than a second", dur)
		}
	}
}

func TestEmptyAckID(t *testing.T) {
	ctx := cloud.NewContext("project-id", &http.Client{})
	id := []string{"test", ""}
	err := Ack(ctx, "sub", id...)

	if err == nil || !strings.Contains(err.Error(), "index 1") {
		t.Errorf("Ack should report an error indicating the id is empty. Got: %v", err)
	}
}
<commit_msg>pubsub: remove test for deprecated func.<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pubsub

import (
	"testing"
	"time"
)

func TestIsSec(t *testing.T) {
	tests := map[time.Duration]bool{
		time.Second:                    true,
		5 * time.Second:                true,
		time.Hour:                      true,
		time.Millisecond:               false,
		time.Second + time.Microsecond: false,
	}
	for dur, want := range tests {
		if got := isSec(dur); got != want {
			t.Errorf("isSec(%v) = %t; want %t", dur, got, want)
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package cluster to manage a Ceph cluster.
package cluster

import (
	"time"

	"github.com/pkg/errors"
	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
	"github.com/rook/rook/pkg/clusterd"
	"github.com/rook/rook/pkg/daemon/ceph/client"
	cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
	"github.com/rook/rook/pkg/operator/ceph/cluster/crash"
	"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
	"github.com/rook/rook/pkg/operator/ceph/config"
	"github.com/rook/rook/pkg/operator/ceph/csi"
	"github.com/rook/rook/pkg/operator/k8sutil"
	v1 "k8s.io/api/core/v1"
	kerrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
)

func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error {
	// Make sure the spec contains all the information we need
	err := validateExternalClusterSpec(cluster)
	if err != nil {
		return errors.Wrap(err, "failed to validate external cluster specs")
	}

	config.ConditionExport(c.context, c.namespacedName, cephv1.ConditionConnecting, v1.ConditionTrue, "ClusterConnecting", "Cluster is connecting")

	// loop until we find the secret necessary to connect to the external cluster
	// then populate clusterInfo
	cluster.Info = populateExternalClusterInfo(c.context, c.namespacedName.Namespace)

	// If the user to check the ceph health and status is not the admin,
	// we validate that ExternalCred has been populated correctly,
	// then we check if the key (whether admin or not) is encoded in base64
	if !isExternalHealthCheckUserAdmin(cluster.Info.AdminSecret) {
		if !cluster.Info.IsInitializedExternalCred(true) {
			return errors.New("invalid user health checker credentials")
		}
		if !cephconfig.IsKeyringBase64Encoded(cluster.Info.ExternalCred.Secret) {
			return errors.Errorf("invalid user health checker key %q", cluster.Info.ExternalCred.Username)
		}
	} else {
		// If the client.admin is used
		if !cephconfig.IsKeyringBase64Encoded(cluster.Info.AdminSecret) {
			return errors.Errorf("invalid user health checker key %q", client.AdminUsername)
		}
	}

	// Write connection info (ceph config file and keyring) for ceph commands
	if cluster.Spec.CephVersion.Image == "" {
		err = mon.WriteConnectionConfig(c.context, cluster.Info)
		if err != nil {
			logger.Errorf("failed to write config. attempting to continue. %v", err)
		}
	}

	// Validate versions (local and external)
	// If no image is specified we don't perform any checks
	if cluster.Spec.CephVersion.Image != "" {
		_, _, err = c.detectAndValidateCephVersion(cluster)
		if err != nil {
			return errors.Wrap(err, "failed to detect and validate ceph version")
		}

		// Write the rook-config-override configmap (used by various daemons to apply config overrides)
		// If we don't do this, daemons will never start, waiting forever for this configmap to be present
		//
		// Only do this when doing a bit of management...
		logger.Infof("creating %q configmap", k8sutil.ConfigOverrideName)
		err = populateConfigOverrideConfigMap(c.context, c.namespacedName.Namespace, cluster.ownerRef)
		if err != nil {
			return errors.Wrap(err, "failed to populate config override config map")
		}

		logger.Infof("creating %q secret", config.StoreName)
		err = config.GetStore(c.context, c.namespacedName.Namespace, &cluster.ownerRef).CreateOrUpdate(cluster.Info)
		if err != nil {
			return errors.Wrap(err, "failed to update the global config")
		}
	}

	// The cluster Identity must be established at this point
	if !cluster.Info.IsInitialized(true) {
		return errors.New("the cluster identity was not established")
	}
	logger.Info("external cluster identity established")

	// Create CSI Secrets only if the user has provided the admin key
	if cluster.Info.AdminSecret != mon.AdminSecretName {
		err = csi.CreateCSISecrets(c.context, c.namespacedName.Namespace, &cluster.ownerRef)
		if err != nil {
			return errors.Wrap(err, "failed to create csi kubernetes secrets")
		}
	}

	// Create CSI config map
	err = csi.CreateCsiConfigMap(c.namespacedName.Namespace, c.context.Clientset, &cluster.ownerRef)
	if err != nil {
		return errors.Wrap(err, "failed to create csi config map")
	}

	// Save CSI configmap
	err = csi.SaveClusterConfig(c.context.Clientset, c.namespacedName.Namespace, cluster.Info, c.csiConfigMutex)
	if err != nil {
		return errors.Wrap(err, "failed to update csi cluster config")
	}
	logger.Info("successfully updated csi config map")

	// Create Crash Collector Secret
	// In 14.2.5 the crash daemon will read the client.crash key instead of the admin key
	if !cluster.Spec.CrashCollector.Disable {
		err = crash.CreateCrashCollectorSecret(c.context, c.namespacedName.Namespace, &cluster.ownerRef)
		if err != nil {
			return errors.Wrap(err, "failed to create crash collector kubernetes secret")
		}
	}

	// Everything went well so let's update the CR's status to "connected"
	config.ConditionExport(c.context, c.namespacedName, cephv1.ConditionConnected, v1.ConditionTrue, "ClusterConnected", "Cluster connected successfully")

	// Mark initialization has done
	cluster.initCompleted = true

	return nil
}

func purgeExternalCluster(clientset kubernetes.Interface, namespace string) {
	// Purge the config maps
	cmsToDelete := []string{
		mon.EndpointConfigMapName,
		k8sutil.ConfigOverrideName,
	}
	for _, cm := range cmsToDelete {
		err := clientset.CoreV1().ConfigMaps(namespace).Delete(cm, &metav1.DeleteOptions{})
		if err != nil && !kerrors.IsNotFound(err) {
			logger.Errorf("failed to delete config map %q. %v", cm, err)
		}
	}

	// Purge the secrets
	secretsToDelete := []string{
		mon.AppName,
		mon.OperatorCreds,
		csi.CsiRBDNodeSecret,
		csi.CsiRBDProvisionerSecret,
		csi.CsiCephFSNodeSecret,
		csi.CsiCephFSProvisionerSecret,
		config.StoreName,
	}
	for _, secret := range secretsToDelete {
		err := clientset.CoreV1().Secrets(namespace).Delete(secret, &metav1.DeleteOptions{})
		if err != nil && !kerrors.IsNotFound(err) {
			logger.Errorf("failed to delete secret %q. %v", secret, err)
		}
	}
}

func validateExternalClusterSpec(cluster *cluster) error {
	if cluster.Spec.CephVersion.Image != "" {
		if cluster.Spec.DataDirHostPath == "" {
			return errors.New("dataDirHostPath must be specified")
		}
	}

	return nil
}

// Add validation in the code to fail if the external cluster has no OSDs keep waiting
func populateExternalClusterInfo(context *clusterd.Context, namespace string) *cephconfig.ClusterInfo {
	var clusterInfo *cephconfig.ClusterInfo
	for {
		var err error
		clusterInfo, _, _, err = mon.LoadClusterInfo(context, namespace)
		if err != nil {
			logger.Warningf("waiting for the connection info of the external cluster. retrying in %s.", externalConnectionRetry.String())
			time.Sleep(externalConnectionRetry)
			continue
		} else {
			// If an admin key was provided we don't need to load the other resources
			// Some people might want to give the admin key
			// The necessary users/keys/secrets will be created by Rook
			// This is also done to allow backward compatibility
			if isExternalHealthCheckUserAdmin(clusterInfo.AdminSecret) {
				break
			}
			externalCred, err := mon.ValidateAndLoadExternalClusterSecrets(context, namespace)
			if err != nil {
				logger.Warningf("waiting for the connection info of the external cluster. retrying in %s.", externalConnectionRetry.String())
				logger.Debugf("%v", err)
				time.Sleep(externalConnectionRetry)
				continue
			} else {
				clusterInfo.ExternalCred = externalCred
				logger.Infof("found the cluster info to connect to the external cluster. will use %q to check health and monitor status. mons=%+v", clusterInfo.ExternalCred.Username, clusterInfo.Monitors)
				break
			}
		}
	}

	return clusterInfo
}

func isExternalHealthCheckUserAdmin(adminSecret string) bool {
	return adminSecret != mon.AdminSecretName
}
<commit_msg>ceph: set external creds if admin key is available<commit_after>/*
Copyright 2020 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package cluster to manage a Ceph cluster.
package cluster

import (
	"time"

	"github.com/pkg/errors"
	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
	"github.com/rook/rook/pkg/clusterd"
	"github.com/rook/rook/pkg/daemon/ceph/client"
	cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
	"github.com/rook/rook/pkg/operator/ceph/cluster/crash"
	"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
	"github.com/rook/rook/pkg/operator/ceph/config"
	"github.com/rook/rook/pkg/operator/ceph/csi"
	"github.com/rook/rook/pkg/operator/k8sutil"
	v1 "k8s.io/api/core/v1"
	kerrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
)

func (c *ClusterController) configureExternalCephCluster(cluster *cluster) error {
	// Make sure the spec contains all the information we need
	err := validateExternalClusterSpec(cluster)
	if err != nil {
		return errors.Wrap(err, "failed to validate external cluster specs")
	}

	config.ConditionExport(c.context, c.namespacedName, cephv1.ConditionConnecting, v1.ConditionTrue, "ClusterConnecting", "Cluster is connecting")

	// loop until we find the secret necessary to connect to the external cluster
	// then populate clusterInfo
	cluster.Info = populateExternalClusterInfo(c.context, c.namespacedName.Namespace)

	// If the user to check the ceph health and status is not the admin,
	// we validate that ExternalCred has been populated correctly,
	// then we check if the key (whether admin or not) is encoded in base64
	if !isExternalHealthCheckUserAdmin(cluster.Info.AdminSecret) {
		if !cluster.Info.IsInitializedExternalCred(true) {
			return errors.New("invalid user health checker credentials")
		}
		if !cephconfig.IsKeyringBase64Encoded(cluster.Info.ExternalCred.Secret) {
			return errors.Errorf("invalid user health checker key %q", cluster.Info.ExternalCred.Username)
		}
	} else {
		// If the client.admin is used
		if !cephconfig.IsKeyringBase64Encoded(cluster.Info.AdminSecret) {
			return errors.Errorf("invalid user health checker key %q", client.AdminUsername)
		}
	}

	// Write connection info (ceph config file and keyring) for ceph commands
	if cluster.Spec.CephVersion.Image == "" {
		err = mon.WriteConnectionConfig(c.context, cluster.Info)
		if err != nil {
			logger.Errorf("failed to write config. attempting to continue. %v", err)
		}
	}

	// Validate versions (local and external)
	// If no image is specified we don't perform any checks
	if cluster.Spec.CephVersion.Image != "" {
		_, _, err = c.detectAndValidateCephVersion(cluster)
		if err != nil {
			return errors.Wrap(err, "failed to detect and validate ceph version")
		}

		// Write the rook-config-override configmap (used by various daemons to apply config overrides)
		// If we don't do this, daemons will never start, waiting forever for this configmap to be present
		//
		// Only do this when doing a bit of management...
		logger.Infof("creating %q configmap", k8sutil.ConfigOverrideName)
		err = populateConfigOverrideConfigMap(c.context, c.namespacedName.Namespace, cluster.ownerRef)
		if err != nil {
			return errors.Wrap(err, "failed to populate config override config map")
		}

		logger.Infof("creating %q secret", config.StoreName)
		err = config.GetStore(c.context, c.namespacedName.Namespace, &cluster.ownerRef).CreateOrUpdate(cluster.Info)
		if err != nil {
			return errors.Wrap(err, "failed to update the global config")
		}
	}

	// The cluster Identity must be established at this point
	if !cluster.Info.IsInitialized(true) {
		return errors.New("the cluster identity was not established")
	}
	logger.Info("external cluster identity established")

	// Create CSI Secrets only if the user has provided the admin key
	if cluster.Info.AdminSecret != mon.AdminSecretName {
		err = csi.CreateCSISecrets(c.context, c.namespacedName.Namespace, &cluster.ownerRef)
		if err != nil {
			return errors.Wrap(err, "failed to create csi kubernetes secrets")
		}
	}

	// Create CSI config map
	err = csi.CreateCsiConfigMap(c.namespacedName.Namespace, c.context.Clientset, &cluster.ownerRef)
	if err != nil {
		return errors.Wrap(err, "failed to create csi config map")
	}

	// Save CSI configmap
	err = csi.SaveClusterConfig(c.context.Clientset, c.namespacedName.Namespace, cluster.Info, c.csiConfigMutex)
	if err != nil {
		return errors.Wrap(err, "failed to update csi cluster config")
	}
	logger.Info("successfully updated csi config map")

	// Create Crash Collector Secret
	// In 14.2.5 the crash daemon will read the client.crash key instead of the admin key
	if !cluster.Spec.CrashCollector.Disable {
		err = crash.CreateCrashCollectorSecret(c.context, c.namespacedName.Namespace, &cluster.ownerRef)
		if err != nil {
			return errors.Wrap(err, "failed to create crash collector kubernetes secret")
		}
	}

	// Everything went well so let's update the CR's status to "connected"
	config.ConditionExport(c.context, c.namespacedName, cephv1.ConditionConnected, v1.ConditionTrue, "ClusterConnected", "Cluster connected successfully")

	// Mark initialization has done
	cluster.initCompleted = true

	return nil
}

func purgeExternalCluster(clientset kubernetes.Interface, namespace string) {
	// Purge the config maps
	cmsToDelete := []string{
		mon.EndpointConfigMapName,
		k8sutil.ConfigOverrideName,
	}
	for _, cm := range cmsToDelete {
		err := clientset.CoreV1().ConfigMaps(namespace).Delete(cm, &metav1.DeleteOptions{})
		if err != nil && !kerrors.IsNotFound(err) {
			logger.Errorf("failed to delete config map %q. %v", cm, err)
		}
	}

	// Purge the secrets
	secretsToDelete := []string{
		mon.AppName,
		mon.OperatorCreds,
		csi.CsiRBDNodeSecret,
		csi.CsiRBDProvisionerSecret,
		csi.CsiCephFSNodeSecret,
		csi.CsiCephFSProvisionerSecret,
		config.StoreName,
	}
	for _, secret := range secretsToDelete {
		err := clientset.CoreV1().Secrets(namespace).Delete(secret, &metav1.DeleteOptions{})
		if err != nil && !kerrors.IsNotFound(err) {
			logger.Errorf("failed to delete secret %q. %v", secret, err)
		}
	}
}

func validateExternalClusterSpec(cluster *cluster) error {
	if cluster.Spec.CephVersion.Image != "" {
		if cluster.Spec.DataDirHostPath == "" {
			return errors.New("dataDirHostPath must be specified")
		}
	}

	return nil
}

// Add validation in the code to fail if the external cluster has no OSDs keep waiting
func populateExternalClusterInfo(context *clusterd.Context, namespace string) *cephconfig.ClusterInfo {
	var clusterInfo *cephconfig.ClusterInfo
	for {
		var err error
		clusterInfo, _, _, err = mon.LoadClusterInfo(context, namespace)
		if err != nil {
			logger.Warningf("waiting for the connection info of the external cluster. retrying in %s.", externalConnectionRetry.String())
			time.Sleep(externalConnectionRetry)
			continue
		}
		// If an admin key was provided we don't need to load the other resources
		// Some people might want to give the admin key
		// The necessary users/keys/secrets will be created by Rook
		// This is also done to allow backward compatibility
		if isExternalHealthCheckUserAdmin(clusterInfo.AdminSecret) {
			clusterInfo.ExternalCred = cephconfig.ExternalCred{Username: client.AdminUsername, Secret: clusterInfo.AdminSecret}
			break
		}
		externalCred, err := mon.ValidateAndLoadExternalClusterSecrets(context, namespace)
		if err != nil {
			logger.Warningf("waiting for the connection info of the external cluster. retrying in %s.", externalConnectionRetry.String())
			logger.Debugf("%v", err)
			time.Sleep(externalConnectionRetry)
			continue
		}
		clusterInfo.ExternalCred = externalCred
		logger.Infof("found the cluster info to connect to the external cluster. will use %q to check health and monitor status. mons=%+v", clusterInfo.ExternalCred.Username, clusterInfo.Monitors)
		break
	}

	return clusterInfo
}

func isExternalHealthCheckUserAdmin(adminSecret string) bool {
	return adminSecret != mon.AdminSecretName
}
<|endoftext|>


<commit_before>package arts

import (
	"code.google.com/p/cascadia"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
	"errors"
	"fmt"
	"regexp"
	"sort"
	"strings"
)

var headlinePats = struct {
	considerSel         cascadia.Selector // elements to consider
	titleSel            cascadia.Selector
	metaTitlesSel       cascadia.Selector // meta tags which have title
	itemPropHeadlineSel cascadia.Selector
}{
	cascadia.MustCompile("h1,h2,h3,h4,h5,h6,div,span,th,td"),
	cascadia.MustCompile("title"),
	cascadia.MustCompile(`meta[property="og:title"], meta[name="wp_twitter-title"]`),
	cascadia.MustCompile(`[itemprop="headline"]`),
}

func grabHeadline(root *html.Node, art_url string) (string, *html.Node, error) {
	dbug := Debug.HeadlineLogger

	var candidates = make(candidateList, 0, 100)

	indicative := regexp.MustCompile(`(?i)entry-title|headline|title`)

	cookedSlug := toAlphanumeric(regexp.MustCompile("[-_]+").ReplaceAllLiteralString(getSlug(art_url), " "))
	dbug.Printf("slug: '%s'\n", cookedSlug)

	//html.Render(os.Stderr, root)
	//dumpTree(root, 0)

	var cookedTitle string
	t := headlinePats.titleSel.MatchFirst(root)
	if t != nil {
		cookedTitle = toAlphanumeric(getTextContent(t))
	}

	// check for any interesting meta tags (og:title etc...)
	// remember that some sites append the site name to this (eg telegraph,
	// rolling stone) so we can't just take it verbatim. But it gives us clues...
	type metaTitle struct {
		cooked string
		node   *html.Node
	}
	metaTitles := []metaTitle{}
	for _, metaTitleNode := range headlinePats.metaTitlesSel.MatchAll(root) {
		cooked := toAlphanumeric(getAttr(metaTitleNode, "content"))
		metaTitles = append(metaTitles, metaTitle{cooked, metaTitleNode})
	}

	// TODO: early-out on hatom or schema.org article

	for _, el := range headlinePats.considerSel.MatchAll(root) {

		txt := compressSpace(getTextContent(el))
		if len(txt) >= 500 {
			continue // too long
		}
		if len(txt) < 3 {
			continue // too short
		}

		cookedTxt := toAlphanumeric(txt)

		c := newStandardCandidate(el, txt)

		// TEST: schema.org headline
		if headlinePats.itemPropHeadlineSel.Match(el) {
			c.addPoints(2, `itemprop="headline"`)
		}

		// TEST: is it a headliney element?
		tag := el.DataAtom
		if tag == atom.H1 || tag == atom.H2 || tag == atom.H3 || tag == atom.H4 {
			c.addPoints(2, "headliney")
		}
		if tag == atom.Span || tag == atom.Td {
			c.addPoints(-2, "not headliney")
		}

		// TEST: likely-looking class or id?
		cls := getAttr(el, "class")
		if cls != "" && (indicative.FindStringIndex(cls) != nil) {
			c.addPoints(2, "indicative class")
		}

		id := getAttr(el, "id")
		if id != "" && (indicative.FindStringIndex(id) != nil) {
			c.addPoints(2, "indicative id")
		}

		if len(cookedTxt) > 0 {
			// TEST: beginning of <title>?
			if strings.HasPrefix(cookedTitle, cookedTxt) {
				c.addPoints(2, "appears at start of <title>")
			}

			if wordCount(cookedTxt) >= 3 {

				// TEST: appears in page <title>?
				{
					value := jaccardWordCompare(cookedTxt, cookedTitle)
					c.addPoints((value*4)-1, "score against <title>")
				}

				// TEST: like the slug?
				if cookedSlug != "" {
					value := jaccardWordCompare(cookedTxt, cookedSlug)
					c.addPoints((value*4)-1, "score against slug")
				}
			}

			// TEST: likely-looking meta tags
			for _, metaTitle := range metaTitles {
				value := jaccardWordCompare(cookedTxt, metaTitle.cooked)
				c.addPoints((value*4)-1, fmt.Sprintf("score against %s", describeNode(metaTitle.node)))
			}
		}

		// TEST: inside an obvious sidebar or <aside>?

		/* TODO!!!!!!!!!!!!!!!!!!!!!!!
		if s.Closest("aside").Length() > 0 {
			c.addPoints(-3, "contained within <aside>")
		}
		if s.Closest("#sidebar").Length() > 0 {
			c.addPoints(-3, "contained within #sidebar")
		}

		// TEST: within article container?
		if insideArticle(s) {
			c.addPoints(1, "within article container")
		}
		*/

		// IDEAS:
		//  promote if within <article> <header>?

		if c.total() > 0 {
			candidates = append(candidates, c)
		}
	}

	sort.Sort(Reverse{candidates})

	dbug.Printf("HEADLINE %d candidates\n", len(candidates))
	// show the top ten, with reasons
	//	if len(candidates) > 20 {
	//		candidates = candidates[0:20]
	//	}
	for _, c := range candidates {
		c.dump(dbug)
	}

	if len(candidates) > 0 {
		return candidates[0].txt(), candidates[0].node(), nil
	}
	return "", nil, errors.New("couldn't find a headline")
}

/*
func insideArticle(s *goquery.Selection) bool {
	if s.Closest("article, #post, .article, .story-body").Length() > 0 {
		return true
	}

	return false
}
*/
<commit_msg>headline extract: penalty for unlikely class/id<commit_after>package arts

import (
	"code.google.com/p/cascadia"
	"errors"
	"fmt"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
	"regexp"
	"sort"
	"strings"
)

var headlinePats = struct {
	considerSel         cascadia.Selector // elements to consider
	titleSel            cascadia.Selector
	metaTitlesSel       cascadia.Selector // meta tags which have title
	itemPropHeadlineSel cascadia.Selector
	indicativePat       *regexp.Regexp
	unIndicativePat     *regexp.Regexp
}{
	cascadia.MustCompile("h1,h2,h3,h4,h5,h6,div,span,th,td"),
	cascadia.MustCompile("title"),
	cascadia.MustCompile(`meta[property="og:title"], meta[name="wp_twitter-title"]`),
	cascadia.MustCompile(`[itemprop="headline"]`),
	regexp.MustCompile(`(?i)entry-title|headline|title`),
	regexp.MustCompile(`(?i)feed-title`),
}

func grabHeadline(root *html.Node, art_url string) (string, *html.Node, error) {
	dbug := Debug.HeadlineLogger

	var candidates = make(candidateList, 0, 100)

	cookedSlug := toAlphanumeric(regexp.MustCompile("[-_]+").ReplaceAllLiteralString(getSlug(art_url), " "))
	dbug.Printf("slug: '%s'\n", cookedSlug)

	//html.Render(os.Stderr, root)
	//dumpTree(root, 0)

	var cookedTitle string
	t := headlinePats.titleSel.MatchFirst(root)
	if t != nil {
		cookedTitle = toAlphanumeric(getTextContent(t))
	}

	// check for any interesting meta tags (og:title etc...)
	// remember that some sites append the site name to this (eg telegraph,
	// rolling stone) so we can't just take it verbatim. But it gives us clues...
	type metaTitle struct {
		cooked string
		node   *html.Node
	}
	metaTitles := []metaTitle{}
	for _, metaTitleNode := range headlinePats.metaTitlesSel.MatchAll(root) {
		cooked := toAlphanumeric(getAttr(metaTitleNode, "content"))
		metaTitles = append(metaTitles, metaTitle{cooked, metaTitleNode})
	}

	// TODO: early-out on hatom or schema.org article

	for _, el := range headlinePats.considerSel.MatchAll(root) {

		txt := compressSpace(getTextContent(el))
		if len(txt) >= 500 {
			continue // too long
		}
		if len(txt) < 3 {
			continue // too short
		}

		cookedTxt := toAlphanumeric(txt)

		c := newStandardCandidate(el, txt)

		// TEST: schema.org headline
		if headlinePats.itemPropHeadlineSel.Match(el) {
			c.addPoints(2, `itemprop="headline"`)
		}

		// TEST: is it a headliney element?
		tag := el.DataAtom
		if tag == atom.H1 || tag == atom.H2 || tag == atom.H3 || tag == atom.H4 {
			c.addPoints(2, "headliney")
		}
		if tag == atom.Span || tag == atom.Td {
			c.addPoints(-2, "not headliney")
		}

		// TEST: likely-looking class or id?
		cls := getAttr(el, "class")
		id := getAttr(el, "id")
		if cls != "" && (headlinePats.indicativePat.FindStringIndex(cls) != nil) {
			c.addPoints(2, "indicative class")
		}
		if id != "" && (headlinePats.indicativePat.FindStringIndex(id) != nil) {
			c.addPoints(2, "indicative id")
		}

		// TEST: unlikely-looking class or id?
		if cls != "" && (headlinePats.unIndicativePat.FindStringIndex(cls) != nil) {
			c.addPoints(-1, "unindicative class")
		}
		if id != "" && (headlinePats.unIndicativePat.FindStringIndex(id) != nil) {
			c.addPoints(-1, "unindicative id")
		}
		if len(cookedTxt) > 0 {
			// TEST: beginning of <title>?
			if strings.HasPrefix(cookedTitle, cookedTxt) {
				c.addPoints(2, "appears at start of <title>")
			}

			if wordCount(cookedTxt) >= 3 {

				// TEST: appears in page <title>?
				{
					value := jaccardWordCompare(cookedTxt, cookedTitle)
					c.addPoints((value*4)-1, "score against <title>")
				}

				// TEST: like the slug?
				if cookedSlug != "" {
					value := jaccardWordCompare(cookedTxt, cookedSlug)
					c.addPoints((value*4)-1, "score against slug")
				}
			}

			// TEST: likely-looking meta tags
			for _, metaTitle := range metaTitles {
				value := jaccardWordCompare(cookedTxt, metaTitle.cooked)
				c.addPoints((value*4)-1, fmt.Sprintf("score against %s", describeNode(metaTitle.node)))
			}
		}

		// TEST: inside an obvious sidebar or <aside>?

		/* TODO!!!!!!!!!!!!!!!!!!!!!!!
		if s.Closest("aside").Length() > 0 {
			c.addPoints(-3, "contained within <aside>")
		}
		if s.Closest("#sidebar").Length() > 0 {
			c.addPoints(-3, "contained within #sidebar")
		}

		// TEST: within article container?
		if insideArticle(s) {
			c.addPoints(1, "within article container")
		}
		*/

		// IDEAS:
		//  promote if within <article> <header>?

		if c.total() > 0 {
			candidates = append(candidates, c)
		}
	}

	sort.Sort(Reverse{candidates})

	dbug.Printf("HEADLINE %d candidates\n", len(candidates))
	// show the top ten, with reasons
	//	if len(candidates) > 20 {
	//		candidates = candidates[0:20]
	//	}
	for _, c := range candidates {
		c.dump(dbug)
	}

	if len(candidates) > 0 {
		return candidates[0].txt(), candidates[0].node(), nil
	}
	return "", nil, errors.New("couldn't find a headline")
}

/*
func insideArticle(s *goquery.Selection) bool {
	if s.Closest("article, #post, .article, .story-body").Length() > 0 {
		return true
	}

	return false
}
*/
<|endoftext|>


<commit_before>package secrets_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"crypto/rand"

	"github.com/adobe-platform/porter/secrets"
)

var _ = Describe("Secrets", func() {

	It("Encrypt/decrypt flow works", func() {

		symmetricKey, err := secrets.GenerateKey()
		Expect(err).To(BeNil())

		payload := []byte("Super secret message")
		encPayload, err := secrets.Encrypt(payload, symmetricKey)
		Expect(err).To(BeNil())

		originalPayload, err := secrets.Decrypt(encPayload, symmetricKey)
		Expect(err).To(BeNil())

		Expect(originalPayload).To(Equal(payload))
	})

})
<commit_msg>remove extra import<commit_after>package secrets_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/adobe-platform/porter/secrets"
)

var _ = Describe("Secrets", func() {

	It("Encrypt/decrypt flow works", func() {

		symmetricKey, err := secrets.GenerateKey()
		Expect(err).To(BeNil())

		payload := []byte("Super secret message")
		encPayload, err := secrets.Encrypt(payload, symmetricKey)
		Expect(err).To(BeNil())

		originalPayload, err := secrets.Decrypt(encPayload, symmetricKey)
		Expect(err).To(BeNil())

		Expect(originalPayload).To(Equal(payload))
	})

})
<|endoftext|>


<commit_before>package integration

import (
	"encoding/json"
	"fmt"
	"github.com/astaxie/beego"
	"github.com/astaxie/beego/orm"
	"github.com/robxu9/kahinah/models"
	"github.com/robxu9/kahinah/util"
	"io/ioutil"
	"log"
	"menteslibres.net/gosexy/dig"
	"menteslibres.net/gosexy/to"
	"net/http"
	"strings"
	"time"
)

const (
	ABF_URL          = "https://abf.io/api/v1"
	ABF_HANDLER_NAME = "abf"
)

var (
	user      = beego.AppConfig.String("abf_user")
	pass      = beego.AppConfig.String("abf_pass")
	platforms *util.Set
	client    = &http.Client{}
)

func init() {
	platforms = util.NewSet()
	for _, v := range strings.Split(beego.AppConfig.String("abf_platforms"), ";") {
		platforms.Add(v)
	}
}

type ABF byte

func (a ABF) Ping() error {
	err := a.pingBuildCompleted()
	err2 := a.pingTestingBuilds()

	// FUTURE TODO: ping published & rejected builds to ensure consistency

	if err != nil && err2 != nil {
		return fmt.Errorf("abf: two errors: %s | %s", err, err2)
	} else if err != nil {
		return err
	} else if err2 != nil {
		return err2
	}

	return nil
}

func (a ABF) pingBuildCompleted() error {
	// regular usage: use 0 (Build has been completed)
	// below: use 12000 ([testing] build has been published)
	// FIXME - remove hardcoded openmandriva2013.0 filter (but find alternative, we need said filter)
	req, err := http.NewRequest("GET", ABF_URL+"/build_lists.json?per_page=100&filter[status]=0&filter[ownership]=index&filter[platform_id]=668", nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	var result map[string]interface{}

	err = json.Unmarshal(body, &result)
	if err != nil {
		return err
	}

	var lists []interface{}

	err = dig.Get(&result, &lists, "build_lists")
	if err != nil {
		return err
	}

	o := orm.NewOrm()

	for _, v := range lists {
		asserted := v.(map[string]interface{})
		id := dig.Uint64(&asserted, "id")

		num, err := o.QueryTable(new(models.BuildList)).Filter("Handler", ABF_HANDLER_NAME).Filter("HandleId", id).Count()
		if num <= 0 || err != nil {
			list, err := a.getBuildList(id)
			if err != nil {
				log.Printf("abf: Error retrieving build list %s: %s\n", id, err)
			}

			if !platforms.Contains(list.Platform) {
				// ignore
				continue
			}

			// Now send it to testing
			go a.sendToTesting(id)

			_, err = o.Insert(list)
			if err != nil {
				log.Printf("abf: Error saving build list %s: %s\n", id, err)
			}

			for _, listpkg := range list.Packages {
				listpkg.List = list
				o.Insert(listpkg)
			}
		}
	}

	return nil
}

func (a ABF) pingTestingBuilds() error {
	// FIXME - remove hardcoded openmandriva2013.0 filter (but find alternative, we need said filter)
	req, err := http.NewRequest("GET", ABF_URL+"/build_lists.json?per_page=100&filter[status]=12000&filter[ownership]=index&filter[platform_id]=668", nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	var result map[string]interface{}

	err = json.Unmarshal(body, &result)
	if err != nil {
		return err
	}

	var lists []interface{}

	err = dig.Get(&result, &lists, "build_lists")
	if err != nil {
		return err
	}

	o := orm.NewOrm()

	for _, v := range lists {
		asserted := v.(map[string]interface{})
		id := dig.Uint64(&asserted, "id")

		num, err := o.QueryTable(new(models.BuildList)).Filter("Handler", ABF_HANDLER_NAME).Filter("HandleId", id).Count()
		if num <= 0 || err != nil {
			list, err := a.getBuildList(id)
			if err != nil {
				log.Printf("abf: Error retrieving build list %s: %s\n", id, err)
			}

			if !platforms.Contains(list.Platform) {
				// ignore
				continue
			}

			_, err = o.Insert(list)
			if err != nil {
				log.Printf("abf: Error saving build list %s: %s\n", id, err)
			}

			for _, listpkg := range list.Packages {
				listpkg.List = list
				o.Insert(listpkg)
			}
		}
	}

	return nil
}

func (a ABF) PingParams(m map[string]string) error {
	return fmt.Errorf("abf: PingParams not supported yet.")
}

func (a ABF) Url(sid string) string {
	return "https://abf.io/build_lists/" + sid
}

func (a ABF) Publish(sid string) error {
	id := to.Uint64(sid)
	req, err := http.NewRequest("PUT", ABF_URL+"/build_lists/"+to.String(id)+"/publish.json", nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)
	req.Header.Add("Content-Length", 0)

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	return nil
}

func (a ABF) Reject(sid string) error {
	id := to.Uint64(sid)
	req, err := http.NewRequest("PUT", ABF_URL+"/build_lists/"+to.String(id)+"/reject_publish.json", nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)
	req.Header.Add("Content-Length", 0)

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	return nil
}

func (a ABF) sendToTesting(id uint64) error {
	req, err := http.NewRequest("PUT", ABF_URL+"/build_lists/"+to.String(id)+"/publish_into_testing.json", nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)
	req.Header.Add("Content-Length", 0)

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	return nil
}

func (a ABF) getBuildList(id uint64) (*models.BuildList, error) {
	req, err := http.NewRequest("GET", ABF_URL+"/build_lists/"+to.String(id)+".json", nil)
	if err != nil {
		return nil, err
	}

	req.SetBasicAuth(user, pass)

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var result map[string]interface{}

	err = json.Unmarshal(body, &result)
	if err != nil {
		return nil, err
	}

	var list map[string]interface{}

	dig.Get(&result, &list, "build_list")

	pkgs := make([]interface{}, 0)
	dig.Get(&list, &pkgs, "packages")

	pkg := make([]*models.BuildListPkg, 0)

	pkgrep := func(m map[string]interface{}) *models.BuildListPkg {
		pkgType := dig.String(&m, "type")
		if strings.HasSuffix(dig.String(&m, "name"), "-debuginfo") {
			pkgType = "debuginfo"
		}

		return &models.BuildListPkg{
			Name:    dig.String(&m, "name"),
			Type:    pkgType,
			Epoch:   dig.Int64(&m, "epoch"),
			Version: dig.String(&m, "version"),
			Release: dig.String(&m, "release"),
			Url:     dig.String(&m, "url"),
		}
	}

	for _, v := range pkgs {
		asserted := v.(map[string]interface{})
		pkg = append(pkg, pkgrep(asserted))
	}

	changelog := ""

	logs := make([]interface{}, 0)
	dig.Get(&list, &logs, "logs")

	for _, v := range logs {
		asserted := v.(map[string]interface{})
		if dig.String(&asserted, "file_name") == "changelog.log" {
			changelog = dig.String(&asserted, "url")
			break
		}
	}

	user := a.getUser(dig.Uint64(&list, "user", "id"))

	bl := models.BuildList{
		HandleId: to.String(dig.Uint64(&list, "id")),
		Handler:  ABF_HANDLER_NAME,

		//Platform:     dig.String(&list, "build_for_platform", "name"),
		Platform:     dig.String(&list, "save_to_repository", "platform", "name"),
		Repo:         dig.String(&list, "save_to_repository", "name"),
		Architecture: dig.String(&list, "arch", "name"),
		Name:         dig.String(&list, "project", "name"),
		Submitter:    user,
		Type:         dig.String(&list, "update_type"),
		Status:       models.STATUS_TESTING,
		Changelog:    changelog, // url
		Packages:     pkg,
		BuildDate:    time.Unix(dig.Int64(&list, "updated_at"), 0)}

	return &bl, nil

}

func (a ABF) getUser(id uint64) *models.User {

	o := orm.NewOrm()

	var userModel *models.User

	var userIntegration models.UserIntegration
	err := o.QueryTable(new(models.UserIntegration)).Filter("Service", ABF_HANDLER_NAME).Filter("ServiceUserId", id).One(&userIntegration)
	if err == orm.ErrNoRows {

		req, err := http.NewRequest("GET", ABF_URL+"/users/"+to.String(id)+".json", nil)
		if err != nil {
			return nil
		}

		req.SetBasicAuth(user, pass)

		resp, err := client.Do(req)
		if err != nil {
			return nil
		}

		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil
		}

		var result map[string]interface{}

		err = json.Unmarshal(body, &result)
		if err != nil {
			return nil
		}

		email := dig.String(&result, "user", "email")

		userModel = models.FindUser(email)
		if userModel.Integration == nil {
			userModel.Integration = make([]*models.UserIntegration, 0)
		}

		integration := models.UserIntegration{
			User:          userModel,
			Service:       ABF_HANDLER_NAME,
			ServiceUserId: to.String(id),
		}

		o.Insert(&integration)

		userModel.Integration = append(userModel.Integration, &integration)
		userModel.Save()

	} else if err != nil {
		panic(err)
	} else {
		o.LoadRelated(&userIntegration, "User")
		userModel = userIntegration.User
	}

	return userModel

}
<commit_msg>oh fck. switch from int to string<commit_after>package integration

import (
	"encoding/json"
	"fmt"
	"github.com/astaxie/beego"
	"github.com/astaxie/beego/orm"
	"github.com/robxu9/kahinah/models"
	"github.com/robxu9/kahinah/util"
	"io/ioutil"
	"log"
	"menteslibres.net/gosexy/dig"
	"menteslibres.net/gosexy/to"
	"net/http"
	"strings"
	"time"
)

const (
	ABF_URL          = "https://abf.io/api/v1"
	ABF_HANDLER_NAME = "abf"
)

var (
	user      = beego.AppConfig.String("abf_user")
	pass      = beego.AppConfig.String("abf_pass")
	platforms *util.Set
	client    = &http.Client{}
)

func init() {
	platforms = util.NewSet()
	for _, v := range strings.Split(beego.AppConfig.String("abf_platforms"), ";") {
		platforms.Add(v)
	}
}

type ABF byte

func (a ABF) Ping() error {
	err := a.pingBuildCompleted()
	err2 := a.pingTestingBuilds()

	// FUTURE TODO: ping published & rejected builds to ensure consistency

	if err != nil && err2 != nil {
		return fmt.Errorf("abf: two errors: %s | %s", err, err2)
	} else if err != nil {
		return err
	} else if err2 != nil {
		return err2
	}

	return nil
}

func (a ABF) pingBuildCompleted() error {
	// regular usage: use 0 (Build has been completed)
	// below: use 12000 ([testing] build has been published)
	// FIXME - remove hardcoded openmandriva2013.0 filter (but find alternative, we need said filter)
	req, err := http.NewRequest("GET", ABF_URL+"/build_lists.json?per_page=100&filter[status]=0&filter[ownership]=index&filter[platform_id]=668", nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	var result map[string]interface{}

	err = json.Unmarshal(body, &result)
	if err != nil {
		return err
	}

	var lists []interface{}

	err = dig.Get(&result, &lists, "build_lists")
	if err != nil {
		return err
	}

	o := orm.NewOrm()

	for _, v := range lists {
		asserted := v.(map[string]interface{})
		id := dig.Uint64(&asserted, "id")

		num, err := o.QueryTable(new(models.BuildList)).Filter("Handler", ABF_HANDLER_NAME).Filter("HandleId", id).Count()
		if num <= 0 || err != nil {
			list, err := a.getBuildList(id)
			if err != nil {
				log.Printf("abf: Error retrieving build list %s: %s\n", id, err)
			}

			if !platforms.Contains(list.Platform) {
				// ignore
				continue
			}

			// Now send it to testing
			go a.sendToTesting(id)

			_, err = o.Insert(list)
			if err != nil {
				log.Printf("abf: Error saving build list %s: %s\n", id, err)
			}

			for _, listpkg := range list.Packages {
				listpkg.List = list
				o.Insert(listpkg)
			}
		}
	}

	return nil
}

func (a ABF) pingTestingBuilds() error {
	// FIXME - remove hardcoded openmandriva2013.0 filter (but find alternative, we need said filter)
	req, err := http.NewRequest("GET", ABF_URL+"/build_lists.json?per_page=100&filter[status]=12000&filter[ownership]=index&filter[platform_id]=668", nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	var result map[string]interface{}

	err = json.Unmarshal(body, &result)
	if err != nil {
		return err
	}

	var lists []interface{}

	err = dig.Get(&result, &lists, "build_lists")
	if err != nil {
		return err
	}

	o := orm.NewOrm()

	for _, v := range lists {
		asserted := v.(map[string]interface{})
		id := dig.Uint64(&asserted, "id")

		num, err := o.QueryTable(new(models.BuildList)).Filter("Handler", ABF_HANDLER_NAME).Filter("HandleId", id).Count()
		if num <= 0 || err != nil {
			list, err := a.getBuildList(id)
			if err != nil {
				log.Printf("abf: Error retrieving build list %s: %s\n", id, err)
			}

			if !platforms.Contains(list.Platform) {
				// ignore
				continue
			}

			_, err = o.Insert(list)
			if err != nil {
				log.Printf("abf: Error saving build list %s: %s\n", id, err)
			}

			for _, listpkg := range list.Packages {
				listpkg.List = list
				o.Insert(listpkg)
			}
		}
	}

	return nil
}

func (a ABF) PingParams(m map[string]string) error {
	return fmt.Errorf("abf: PingParams not supported yet.")
}

func (a ABF) Url(sid string) string {
	return "https://abf.io/build_lists/" + sid
}

func (a ABF) Publish(sid string) error {
	id := to.Uint64(sid)
	req, err := http.NewRequest("PUT", ABF_URL+"/build_lists/"+to.String(id)+"/publish.json", nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)
	req.Header.Add("Content-Length", "0")

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	return nil
}

func (a ABF) Reject(sid string) error {
	id := to.Uint64(sid)
	req, err := http.NewRequest("PUT", ABF_URL+"/build_lists/"+to.String(id)+"/reject_publish.json", nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)
	req.Header.Add("Content-Length", "0")

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	return nil
}

func (a ABF) sendToTesting(id uint64) error {
	req, err := http.NewRequest("PUT", ABF_URL+"/build_lists/"+to.String(id)+"/publish_into_testing.json", nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(user, pass)
	req.Header.Add("Content-Length", "0")

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	return nil
}

func (a ABF) getBuildList(id uint64) (*models.BuildList, error) {
	req, err := http.NewRequest("GET", ABF_URL+"/build_lists/"+to.String(id)+".json", nil)
	if err != nil {
		return nil, err
	}

	req.SetBasicAuth(user, pass)

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var result map[string]interface{}

	err = json.Unmarshal(body, &result)
	if err != nil {
		return nil, err
	}

	var list map[string]interface{}

	dig.Get(&result, &list, "build_list")

	pkgs := make([]interface{}, 0)
	dig.Get(&list, &pkgs, "packages")

	pkg := make([]*models.BuildListPkg, 0)

	pkgrep := func(m map[string]interface{}) *models.BuildListPkg {
		pkgType := dig.String(&m, "type")
		if strings.HasSuffix(dig.String(&m, "name"), "-debuginfo") {
			pkgType = "debuginfo"
		}

		return &models.BuildListPkg{
			Name:    dig.String(&m, "name"),
			Type:    pkgType,
			Epoch:   dig.Int64(&m, "epoch"),
			Version: dig.String(&m, "version"),
			Release: dig.String(&m, "release"),
			Url:     dig.String(&m, "url"),
		}
	}

	for _, v := range pkgs {
		asserted := v.(map[string]interface{})
		pkg = append(pkg, pkgrep(asserted))
	}

	changelog := ""

	logs := make([]interface{}, 0)
	dig.Get(&list, &logs, "logs")

	for _, v := range logs {
		asserted := v.(map[string]interface{})
		if dig.String(&asserted, "file_name") == "changelog.log" {
			changelog = dig.String(&asserted, "url")
			break
		}
	}

	user := a.getUser(dig.Uint64(&list, "user", "id"))

	bl := models.BuildList{
		HandleId: to.String(dig.Uint64(&list, "id")),
		Handler:  ABF_HANDLER_NAME,

		//Platform:     dig.String(&list, "build_for_platform", "name"),
		Platform:     dig.String(&list, "save_to_repository", "platform", "name"),
		Repo:         dig.String(&list, "save_to_repository", "name"),
		Architecture: dig.String(&list, "arch", "name"),
		Name:         dig.String(&list, "project", "name"),
		Submitter:    user,
		Type:         dig.String(&list, "update_type"),
		Status:       models.STATUS_TESTING,
		Changelog:    changelog, // url
		Packages:     pkg,
		BuildDate:    time.Unix(dig.Int64(&list, "updated_at"), 0)}

	return &bl, nil

}

func (a ABF) getUser(id uint64) *models.User {

	o := orm.NewOrm()

	var userModel *models.User

	var userIntegration models.UserIntegration
	err := o.QueryTable(new(models.UserIntegration)).Filter("Service", ABF_HANDLER_NAME).Filter("ServiceUserId", id).One(&userIntegration)
	if err == orm.ErrNoRows {

		req, err := http.NewRequest("GET", ABF_URL+"/users/"+to.String(id)+".json", nil)
		if err != nil {
			return nil
		}

		req.SetBasicAuth(user, pass)

		resp, err := client.Do(req)
		if err != nil {
			return nil
		}

		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil
		}

		var result map[string]interface{}

		err = json.Unmarshal(body, &result)
		if err != nil {
			return nil
		}

		email := dig.String(&result, "user", "email")

		userModel = models.FindUser(email)
		if userModel.Integration == nil {
			userModel.Integration = make([]*models.UserIntegration, 0)
		}

		integration := models.UserIntegration{
			User:          userModel,
			Service:       ABF_HANDLER_NAME,
			ServiceUserId: to.String(id),
		}

		o.Insert(&integration)

		userModel.Integration = append(userModel.Integration, &integration)
		userModel.Save()

	} else if err != nil {
		panic(err)
	} else {
		o.LoadRelated(&userIntegration, "User")
		userModel = userIntegration.User
	}

	return userModel

}
<|endoftext|>


<commit_before>package c2

import (
	"log"
	"os"
)

// Don't send enything to this logger unless it's an error that deserves immediate attention
var Err *log.Logger

// Warnings, or general information.
var Info *log.Logger

var Debug *log.Logger

func init() {
	Err = log.New(os.Stderr, "", log.Ldate|log.Ltime|log.Lshortfile)
	Info = log.New(os.Stdout, "", log.Ldate|log.Ltime)
	Debug = log.New(os.Stdout, "", log.Lshortfile)
}
<commit_msg>distinguish log messages w/prefix<commit_after>package c2

import (
	"log"
	"os"
)

// Don't send enything to this logger unless it's an error that deserves immediate attention
var Err *log.Logger

// Warnings, or general information.
var Info *log.Logger

var Debug *log.Logger

func init() {
	Err = log.New(os.Stderr, "ERRO ", log.Ldate|log.Ltime|log.Lshortfile)
	Info = log.New(os.Stdout, "INFO ", log.Ldate|log.Ltime)
	Debug = log.New(os.Stdout, "DEBG ", log.Lshortfile)
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cipher

import (
	"crypto/subtle"
	"errors"
)

// AEAD is a cipher mode providing authenticated encryption with associated
// data.
type AEAD interface {
	// NonceSize returns the size of the nonce that must be passed to Seal
	// and Open.
	NonceSize() int

	// Overhead returns the maximum difference between the lengths of a
	// plaintext and ciphertext.
	Overhead() int

	// Seal encrypts and authenticates plaintext, authenticates the
	// additional data and appends the result to dst, returning the updated
	// slice. The nonce must be NonceSize() bytes long and unique for all
	// time, for a given key.
	//
	// The plaintext and dst may alias exactly or not at all.
	Seal(dst, nonce, plaintext, data []byte) []byte

	// Open decrypts and authenticates ciphertext, authenticates the
	// additional data and, if successful, appends the resulting plaintext
	// to dst, returning the updated slice and true. On error, nil and
	// false is returned. The nonce must be NonceSize() bytes long and both
	// it and the additional data must match the value passed to Seal.
	//
	// The ciphertext and dst may alias exactly or not at all.
	Open(dst, nonce, ciphertext, data []byte) ([]byte, error)
}

// gcmFieldElement represents a value in GF(2¹²⁸). In order to reflect the GCM
// standard and make getUint64 suitable for marshaling these values, the bits
// are stored backwards. For example:
//   the coefficient of x⁰ can be obtained by v.low >> 63.
//   the coefficient of x⁶³ can be obtained by v.low & 1.
//   the coefficient of x⁶⁴ can be obtained by v.high >> 63.
//   the coefficient of x¹²⁷ can be obtained by v.high & 1.
type gcmFieldElement struct {
	low, high uint64
}

// gcm represents a Galois Counter Mode with a specific key. See
// http://csrc.nist.gov/groups/ST/toolkit/BCM/documents/proposedmodes/gcm/gcm-revised-spec.pdf
type gcm struct {
	cipher Block
	// productTable contains the first sixteen powers of the key, H.
	// However, they are in bit reversed order. See NewGCM.
	productTable [16]gcmFieldElement
}

// NewGCM returns the given 128-bit, block cipher wrapped in Galois Counter Mode.
func NewGCM(cipher Block) (AEAD, error) {
	if cipher.BlockSize() != gcmBlockSize {
		return nil, errors.New("cipher: NewGCM requires 128-bit block cipher")
	}

	var key [gcmBlockSize]byte
	cipher.Encrypt(key[:], key[:])

	g := &gcm{cipher: cipher}

	// We precompute 16 multiples of |key|. However, when we do lookups
	// into this table we'll be using bits from a field element and
	// therefore the bits will be in the reverse order. So normally one
	// would expect, say, 4*key to be in index 4 of the table but due to
	// this bit ordering it will actually be in index 0010 (base 2) = 2.
	x := gcmFieldElement{
		getUint64(key[:8]),
		getUint64(key[8:]),
	}
	g.productTable[reverseBits(1)] = x

	for i := 2; i < 16; i += 2 {
		g.productTable[reverseBits(i)] = gcmDouble(&g.productTable[reverseBits(i/2)])
		g.productTable[reverseBits(i+1)] = gcmAdd(&g.productTable[reverseBits(i)], &x)
	}

	return g, nil
}

const (
	gcmBlockSize = 16
	gcmTagSize   = 16
	gcmNonceSize = 12
)

func (*gcm) NonceSize() int {
	return gcmNonceSize
}

func (*gcm) Overhead() int {
	return gcmTagSize
}

func (g *gcm) Seal(dst, nonce, plaintext, data []byte) []byte {
	if len(nonce) != gcmNonceSize {
		panic("cipher: incorrect nonce length given to GCM")
	}

	ret, out := sliceForAppend(dst, len(plaintext)+gcmTagSize)

	// See GCM spec, section 7.1.
	var counter, tagMask [gcmBlockSize]byte
	copy(counter[:], nonce)
	counter[gcmBlockSize-1] = 1

	g.cipher.Encrypt(tagMask[:], counter[:])
	gcmInc32(&counter)

	g.counterCrypt(out, plaintext, &counter)
	g.auth(out[len(plaintext):], out[:len(plaintext)], data, &tagMask)

	return ret
}

var errOpen = errors.New("cipher: message authentication failed")

func (g *gcm) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) {
	if len(nonce) != gcmNonceSize {
		panic("cipher: incorrect nonce length given to GCM")
	}

	if len(ciphertext) < gcmTagSize {
		return nil, errOpen
	}
	tag := ciphertext[len(ciphertext)-gcmTagSize:]
	ciphertext = ciphertext[:len(ciphertext)-gcmTagSize]

	// See GCM spec, section 7.1.
	var counter, tagMask [gcmBlockSize]byte
	copy(counter[:], nonce)
	counter[gcmBlockSize-1] = 1

	g.cipher.Encrypt(tagMask[:], counter[:])
	gcmInc32(&counter)

	var expectedTag [gcmTagSize]byte
	g.auth(expectedTag[:], ciphertext, data, &tagMask)

	if subtle.ConstantTimeCompare(expectedTag[:], tag) != 1 {
		return nil, errOpen
	}

	ret, out := sliceForAppend(dst, len(ciphertext))
	g.counterCrypt(out, ciphertext, &counter)

	return ret, nil
}

// reverseBits reverses the order of the bits of 4-bit number in i.
func reverseBits(i int) int {
	i = ((i << 2) & 0xc) | ((i >> 2) & 0x3)
	i = ((i << 1) & 0xa) | ((i >> 1) & 0x5)
	return i
}

// gcmAdd adds two elements of GF(2¹²⁸) and returns the sum.
func gcmAdd(x, y *gcmFieldElement) gcmFieldElement {
	// Addition in a characteristic 2 field is just XOR.
	return gcmFieldElement{x.low ^ y.low, x.high ^ y.high}
}

// gcmDouble returns the result of doubling an element of GF(2¹²⁸).
func gcmDouble(x *gcmFieldElement) (double gcmFieldElement) {
	msbSet := x.high&1 == 1

	// Because of the bit-ordering, doubling is actually a right shift.
	double.high = x.high >> 1
	double.high |= x.low << 63
	double.low = x.low >> 1

	// If the most-significant bit was set before shifting then it,
	// conceptually, becomes a term of x^128. This is greater than the
	// irreducible polynomial so the result has to be reduced. The
	// irreducible polynomial is 1+x+x^2+x^7+x^128. We can subtract that to
	// eliminate the term at x^128 which also means subtracting the other
	// four terms. In characteristic 2 fields, subtraction == addition ==
	// XOR.
	if msbSet {
		double.low ^= 0xe100000000000000
	}

	return
}

var gcmReductionTable = []uint16{
	0x0000, 0x1c20, 0x3840, 0x2460, 0x7080, 0x6ca0, 0x48c0, 0x54e0,
	0xe100, 0xfd20, 0xd940, 0xc560, 0x9180, 0x8da0, 0xa9c0, 0xb5e0,
}

// mul sets y to y*H, where H is the GCM key, fixed during NewGCM.
func (g *gcm) mul(y *gcmFieldElement) {
	var z gcmFieldElement

	for i := 0; i < 2; i++ {
		word := y.high
		if i == 1 {
			word = y.low
		}

		// Multiplication works by multiplying z by 16 and adding in
		// one of the precomputed multiples of H.
		for j := 0; j < 64; j += 4 {
			msw := z.high & 0xf
			z.high >>= 4
			z.high |= z.low << 60
			z.low >>= 4
			z.low ^= uint64(gcmReductionTable[msw]) << 48

			// the values in |table| are ordered for
			// little-endian bit positions. See the comment
			// in NewGCM.
			t := &g.productTable[word&0xf]

			z.low ^= t.low
			z.high ^= t.high
			word >>= 4
		}
	}

	*y = z
}

// updateBlocks extends y with more polynomial terms from blocks, based on
// Horner's rule. There must be a multiple of gcmBlockSize bytes in blocks.
func (g *gcm) updateBlocks(y *gcmFieldElement, blocks []byte) {
	for len(blocks) > 0 {
		y.low ^= getUint64(blocks)
		y.high ^= getUint64(blocks[8:])
		g.mul(y)
		blocks = blocks[gcmBlockSize:]
	}
}

// update extends y with more polynomial terms from data. If data is not a
// multiple of gcmBlockSize bytes long then the remainder is zero padded.
func (g *gcm) update(y *gcmFieldElement, data []byte) {
	fullBlocks := (len(data) >> 4) << 4
	g.updateBlocks(y, data[:fullBlocks])

	if len(data) != fullBlocks {
		var partialBlock [gcmBlockSize]byte
		copy(partialBlock[:], data[fullBlocks:])
		g.updateBlocks(y, partialBlock[:])
	}
}

// gcmInc32 treats the final four bytes of counterBlock as a big-endian value
// and increments it.
func gcmInc32(counterBlock *[16]byte) {
	for i := gcmBlockSize - 1; i >= gcmBlockSize-4; i-- {
		counterBlock[i]++
		if counterBlock[i] != 0 {
			break
		}
	}
}

// sliceForAppend takes a slice and a requested number of bytes. It returns a
// slice with the contents of the given slice followed by that many bytes and a
// second slice that aliases into it and contains only the extra bytes. If the
// original slice has sufficient capacity then no allocation is performed.
func sliceForAppend(in []byte, n int) (head, tail []byte) {
	if total := len(in) + n; cap(in) >= total {
		head = in[:total]
	} else {
		head = make([]byte, total)
		copy(head, in)
	}
	tail = head[len(in):]
	return
}

// counterCrypt crypts in to out using g.cipher in counter mode.
func (g *gcm) counterCrypt(out, in []byte, counter *[gcmBlockSize]byte) {
	var mask [gcmBlockSize]byte

	for len(in) >= gcmBlockSize {
		g.cipher.Encrypt(mask[:], counter[:])
		gcmInc32(counter)

		xorWords(out, in, mask[:])
		out = out[gcmBlockSize:]
		in = in[gcmBlockSize:]
	}

	if len(in) > 0 {
		g.cipher.Encrypt(mask[:], counter[:])
		gcmInc32(counter)
		xorBytes(out, in, mask[:])
	}
}

// auth calculates GHASH(ciphertext, additionalData), masks the result with
// tagMask and writes the result to out.
func (g *gcm) auth(out, ciphertext, additionalData []byte, tagMask *[gcmTagSize]byte) {
	var y gcmFieldElement
	g.update(&y, additionalData)
	g.update(&y, ciphertext)

	y.low ^= uint64(len(additionalData)) * 8
	y.high ^= uint64(len(ciphertext)) * 8

	g.mul(&y)

	putUint64(out, y.low)
	putUint64(out[8:], y.high)

	xorWords(out, out, tagMask[:])
}

func getUint64(data []byte) uint64 {
	r := uint64(data[0])<<56 |
		uint64(data[1])<<48 |
		uint64(data[2])<<40 |
		uint64(data[3])<<32 |
		uint64(data[4])<<24 |
		uint64(data[5])<<16 |
		uint64(data[6])<<8 |
		uint64(data[7])
	return r
}

func putUint64(out []byte, v uint64) {
	out[0] = byte(v >> 56)
	out[1] = byte(v >> 48)
	out[2] = byte(v >> 40)
	out[3] = byte(v >> 32)
	out[4] = byte(v >> 24)
	out[5] = byte(v >> 16)
	out[6] = byte(v >> 8)
	out[7] = byte(v)
}
<commit_msg>crypto/cipher: fix AEAD.Open documentation nit<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cipher

import (
	"crypto/subtle"
	"errors"
)

// AEAD is a cipher mode providing authenticated encryption with associated
// data.
type AEAD interface {
	// NonceSize returns the size of the nonce that must be passed to Seal
	// and Open.
	NonceSize() int

	// Overhead returns the maximum difference between the lengths of a
	// plaintext and ciphertext.
	Overhead() int

	// Seal encrypts and authenticates plaintext, authenticates the
	// additional data and appends the result to dst, returning the updated
	// slice. The nonce must be NonceSize() bytes long and unique for all
	// time, for a given key.
	//
	// The plaintext and dst may alias exactly or not at all.
	Seal(dst, nonce, plaintext, data []byte) []byte

	// Open decrypts and authenticates ciphertext, authenticates the
	// additional data and, if successful, appends the resulting plaintext
	// to dst, returning the updated slice. The nonce must be NonceSize()
	// bytes long and both it and the additional data must match the
	// value passed to Seal.
	//
	// The ciphertext and dst may alias exactly or not at all.
	Open(dst, nonce, ciphertext, data []byte) ([]byte, error)
}

// gcmFieldElement represents a value in GF(2¹²⁸). In order to reflect the GCM
// standard and make getUint64 suitable for marshaling these values, the bits
// are stored backwards. For example:
//   the coefficient of x⁰ can be obtained by v.low >> 63.
//   the coefficient of x⁶³ can be obtained by v.low & 1.
//   the coefficient of x⁶⁴ can be obtained by v.high >> 63.
//   the coefficient of x¹²⁷ can be obtained by v.high & 1.
type gcmFieldElement struct {
	low, high uint64
}

// gcm represents a Galois Counter Mode with a specific key. See
// http://csrc.nist.gov/groups/ST/toolkit/BCM/documents/proposedmodes/gcm/gcm-revised-spec.pdf
type gcm struct {
	cipher Block
	// productTable contains the first sixteen powers of the key, H.
	// However, they are in bit reversed order. See NewGCM.
	productTable [16]gcmFieldElement
}

// NewGCM returns the given 128-bit, block cipher wrapped in Galois Counter Mode.
func NewGCM(cipher Block) (AEAD, error) {
	if cipher.BlockSize() != gcmBlockSize {
		return nil, errors.New("cipher: NewGCM requires 128-bit block cipher")
	}

	var key [gcmBlockSize]byte
	cipher.Encrypt(key[:], key[:])

	g := &gcm{cipher: cipher}

	// We precompute 16 multiples of |key|. However, when we do lookups
	// into this table we'll be using bits from a field element and
	// therefore the bits will be in the reverse order. So normally one
	// would expect, say, 4*key to be in index 4 of the table but due to
	// this bit ordering it will actually be in index 0010 (base 2) = 2.
	x := gcmFieldElement{
		getUint64(key[:8]),
		getUint64(key[8:]),
	}
	g.productTable[reverseBits(1)] = x

	for i := 2; i < 16; i += 2 {
		g.productTable[reverseBits(i)] = gcmDouble(&g.productTable[reverseBits(i/2)])
		g.productTable[reverseBits(i+1)] = gcmAdd(&g.productTable[reverseBits(i)], &x)
	}

	return g, nil
}

const (
	gcmBlockSize = 16
	gcmTagSize   = 16
	gcmNonceSize = 12
)

func (*gcm) NonceSize() int {
	return gcmNonceSize
}

func (*gcm) Overhead() int {
	return gcmTagSize
}

func (g *gcm) Seal(dst, nonce, plaintext, data []byte) []byte {
	if len(nonce) != gcmNonceSize {
		panic("cipher: incorrect nonce length given to GCM")
	}

	ret, out := sliceForAppend(dst, len(plaintext)+gcmTagSize)

	// See GCM spec, section 7.1.
	var counter, tagMask [gcmBlockSize]byte
	copy(counter[:], nonce)
	counter[gcmBlockSize-1] = 1

	g.cipher.Encrypt(tagMask[:], counter[:])
	gcmInc32(&counter)

	g.counterCrypt(out, plaintext, &counter)
	g.auth(out[len(plaintext):], out[:len(plaintext)], data, &tagMask)

	return ret
}

var errOpen = errors.New("cipher: message authentication failed")

func (g *gcm) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) {
	if len(nonce) != gcmNonceSize {
		panic("cipher: incorrect nonce length given to GCM")
	}

	if len(ciphertext) < gcmTagSize {
		return nil, errOpen
	}
	tag := ciphertext[len(ciphertext)-gcmTagSize:]
	ciphertext = ciphertext[:len(ciphertext)-gcmTagSize]

	// See GCM spec, section 7.1.
	var counter, tagMask [gcmBlockSize]byte
	copy(counter[:], nonce)
	counter[gcmBlockSize-1] = 1

	g.cipher.Encrypt(tagMask[:], counter[:])
	gcmInc32(&counter)

	var expectedTag [gcmTagSize]byte
	g.auth(expectedTag[:], ciphertext, data, &tagMask)

	if subtle.ConstantTimeCompare(expectedTag[:], tag) != 1 {
		return nil, errOpen
	}

	ret, out := sliceForAppend(dst, len(ciphertext))
	g.counterCrypt(out, ciphertext, &counter)

	return ret, nil
}

// reverseBits reverses the order of the bits of 4-bit number in i.
func reverseBits(i int) int {
	i = ((i << 2) & 0xc) | ((i >> 2) & 0x3)
	i = ((i << 1) & 0xa) | ((i >> 1) & 0x5)
	return i
}

// gcmAdd adds two elements of GF(2¹²⁸) and returns the sum.
func gcmAdd(x, y *gcmFieldElement) gcmFieldElement {
	// Addition in a characteristic 2 field is just XOR.
	return gcmFieldElement{x.low ^ y.low, x.high ^ y.high}
}

// gcmDouble returns the result of doubling an element of GF(2¹²⁸).
func gcmDouble(x *gcmFieldElement) (double gcmFieldElement) {
	msbSet := x.high&1 == 1

	// Because of the bit-ordering, doubling is actually a right shift.
	double.high = x.high >> 1
	double.high |= x.low << 63
	double.low = x.low >> 1

	// If the most-significant bit was set before shifting then it,
	// conceptually, becomes a term of x^128. This is greater than the
	// irreducible polynomial so the result has to be reduced. The
	// irreducible polynomial is 1+x+x^2+x^7+x^128. We can subtract that to
	// eliminate the term at x^128 which also means subtracting the other
	// four terms. In characteristic 2 fields, subtraction == addition ==
	// XOR.
	if msbSet {
		double.low ^= 0xe100000000000000
	}

	return
}

var gcmReductionTable = []uint16{
	0x0000, 0x1c20, 0x3840, 0x2460, 0x7080, 0x6ca0, 0x48c0, 0x54e0,
	0xe100, 0xfd20, 0xd940, 0xc560, 0x9180, 0x8da0, 0xa9c0, 0xb5e0,
}

// mul sets y to y*H, where H is the GCM key, fixed during NewGCM.
func (g *gcm) mul(y *gcmFieldElement) {
	var z gcmFieldElement

	for i := 0; i < 2; i++ {
		word := y.high
		if i == 1 {
			word = y.low
		}

		// Multiplication works by multiplying z by 16 and adding in
		// one of the precomputed multiples of H.
		for j := 0; j < 64; j += 4 {
			msw := z.high & 0xf
			z.high >>= 4
			z.high |= z.low << 60
			z.low >>= 4
			z.low ^= uint64(gcmReductionTable[msw]) << 48

			// the values in |table| are ordered for
			// little-endian bit positions. See the comment
			// in NewGCM.
			t := &g.productTable[word&0xf]

			z.low ^= t.low
			z.high ^= t.high
			word >>= 4
		}
	}

	*y = z
}

// updateBlocks extends y with more polynomial terms from blocks, based on
// Horner's rule. There must be a multiple of gcmBlockSize bytes in blocks.
func (g *gcm) updateBlocks(y *gcmFieldElement, blocks []byte) {
	for len(blocks) > 0 {
		y.low ^= getUint64(blocks)
		y.high ^= getUint64(blocks[8:])
		g.mul(y)
		blocks = blocks[gcmBlockSize:]
	}
}

// update extends y with more polynomial terms from data. If data is not a
// multiple of gcmBlockSize bytes long then the remainder is zero padded.
func (g *gcm) update(y *gcmFieldElement, data []byte) {
	fullBlocks := (len(data) >> 4) << 4
	g.updateBlocks(y, data[:fullBlocks])

	if len(data) != fullBlocks {
		var partialBlock [gcmBlockSize]byte
		copy(partialBlock[:], data[fullBlocks:])
		g.updateBlocks(y, partialBlock[:])
	}
}

// gcmInc32 treats the final four bytes of counterBlock as a big-endian value
// and increments it.
func gcmInc32(counterBlock *[16]byte) {
	for i := gcmBlockSize - 1; i >= gcmBlockSize-4; i-- {
		counterBlock[i]++
		if counterBlock[i] != 0 {
			break
		}
	}
}

// sliceForAppend takes a slice and a requested number of bytes. It returns a
// slice with the contents of the given slice followed by that many bytes and a
// second slice that aliases into it and contains only the extra bytes. If the
// original slice has sufficient capacity then no allocation is performed.
func sliceForAppend(in []byte, n int) (head, tail []byte) {
	if total := len(in) + n; cap(in) >= total {
		head = in[:total]
	} else {
		head = make([]byte, total)
		copy(head, in)
	}
	tail = head[len(in):]
	return
}

// counterCrypt crypts in to out using g.cipher in counter mode.
func (g *gcm) counterCrypt(out, in []byte, counter *[gcmBlockSize]byte) {
	var mask [gcmBlockSize]byte

	for len(in) >= gcmBlockSize {
		g.cipher.Encrypt(mask[:], counter[:])
		gcmInc32(counter)

		xorWords(out, in, mask[:])
		out = out[gcmBlockSize:]
		in = in[gcmBlockSize:]
	}

	if len(in) > 0 {
		g.cipher.Encrypt(mask[:], counter[:])
		gcmInc32(counter)
		xorBytes(out, in, mask[:])
	}
}

// auth calculates GHASH(ciphertext, additionalData), masks the result with
// tagMask and writes the result to out.
func (g *gcm) auth(out, ciphertext, additionalData []byte, tagMask *[gcmTagSize]byte) {
	var y gcmFieldElement
	g.update(&y, additionalData)
	g.update(&y, ciphertext)

	y.low ^= uint64(len(additionalData)) * 8
	y.high ^= uint64(len(ciphertext)) * 8

	g.mul(&y)

	putUint64(out, y.low)
	putUint64(out[8:], y.high)

	xorWords(out, out, tagMask[:])
}

func getUint64(data []byte) uint64 {
	r := uint64(data[0])<<56 |
		uint64(data[1])<<48 |
		uint64(data[2])<<40 |
		uint64(data[3])<<32 |
		uint64(data[4])<<24 |
		uint64(data[5])<<16 |
		uint64(data[6])<<8 |
		uint64(data[7])
	return r
}

func putUint64(out []byte, v uint64) {
	out[0] = byte(v >> 56)
	out[1] = byte(v >> 48)
	out[2] = byte(v >> 40)
	out[3] = byte(v >> 32)
	out[4] = byte(v >> 24)
	out[5] = byte(v >> 16)
	out[6] = byte(v >> 8)
	out[7] = byte(v)
}
<|endoftext|>


<commit_before>package slack

import (
	//	"github.com/gorilla/websocket"
	"github.com/james-bowman/websocket"
	"log"
	"sync"
	"time"
)

const (
	// Time allowed to write a message to the peer.
	writeWait = 10 * time.Second

	// Time allowed to read the next pong message from the peer.
	pongWait = 60 * time.Second

	// Send pings to peer with this period. Must be less than pongWait.
	pingPeriod = (pongWait * 9) / 10

	// Maximum message size allowed from peer.
	maxMessageSize = 1024
)

// Connection type represents the duplex websocket connection to Slack
type Connection struct {
	// The websocket connection.
	ws *websocket.Conn

	// waitgroup to wait for all go routines to terminate before attempting to reconnect.
	wg sync.WaitGroup

	// channel used to signal termination to socket writer go routine.
	finish chan struct{}

	// Buffered channel of outbound messages.
	out chan []byte

	// Buffered channel of inbound messages.
	in chan []byte

	// information about the current Slack connection and team settings.
	config Config
}

// write a message with the given message type and payload.
func (c *Connection) write(mt int, payload []byte) error {
	c.ws.SetWriteDeadline(time.Now().Add(writeWait))
	return c.ws.WriteMessage(mt, payload)
}

// socketWriter writes queued messages to the websocket connection.
func (c *Connection) socketWriter() {
	c.wg.Add(1)
	ticker := time.NewTicker(pingPeriod)
	defer func() {
		log.Println("Closing socket writer")
		ticker.Stop()
		c.ws.Close()
		c.wg.Done()
	}()
	for {
		select {
		case message, ok := <-c.out:
			if !ok {
				// channel closed so close the websocket
				c.write(websocket.CloseMessage, []byte{})
				log.Println("Closing socket")
				return
			}
			if err := c.write(websocket.TextMessage, message); err != nil {
				// error writing to websocket
				log.Printf("Error writing to slack websocket: %s", err)
				return
			}
		case <-ticker.C:
			// if idle send a ping
			if err := c.write(websocket.PingMessage, []byte{}); err != nil {
				log.Printf("Error sending ping on slack websocket: %s", err)
				return
			}
		case <-c.finish:
			return
		}
	}
}

// socketReader reads messages from the websocket connection and queues them for processing.
func (c *Connection) socketReader() {
	c.wg.Add(1)
	defer func() {
		log.Println("Closing socket reader")
		c.ws.Close()
		c.wg.Done()
	}()
	c.ws.SetReadDeadline(time.Now().Add(pongWait))
	c.ws.SetPongHandler(func(string) error { c.ws.SetReadDeadline(time.Now().Add(pongWait)); return nil })
	for {
		_, message, err := c.ws.ReadMessage()
		if err != nil {
			log.Printf("Error reading from slack websocket: %s", err)
			break
		}
		c.in <- message
	}
}

// Write the specified event to Slack.  The event is queued and then sent asynchronously.
func (c *Connection) Write(data []byte) {
	c.out <- data
}

// Read the next event from the connection to Slack or block until one is available
func (c *Connection) Read() []byte {
	return <-c.in
}

// start the connection.  Starts receiving and sending events from/to Slack on the connection and,
// in the event the connection is lost, will attempt to automatically reconnect to Slack.
func (c *Connection) start(reconnectionHandler func() (*Config, *websocket.Conn, error)) {
	go func() {
		for {
			c.finish = make(chan struct{})

			go c.socketWriter()
			c.socketReader()

			close(c.finish)
			c.wg.Wait()

			connected := false
			var ws *websocket.Conn
			var config *Config

			for i := 1; !connected; i = i * 2 {
				log.Printf("Lost connection to Slack - reconnecting in %d seconds", i)

				// wait 10 seconds before trying to reconnect
				time.Sleep(time.Duration(i) * time.Second)

				var err error
				config, ws, err = reconnectionHandler()

				if err != nil {
					log.Printf("Error reconnecting: %s", err)
				} else {
					log.Printf("Connected to Slack")
					connected = true
				}
			}

			c.ws = ws
			c.config = *config
		}
	}()
}
<commit_msg>removed max message size as it is no longer used here<commit_after>package slack

import (
	//	"github.com/gorilla/websocket"
	"github.com/james-bowman/websocket"
	"log"
	"sync"
	"time"
)

const (
	// Time allowed to write a message to the peer.
	writeWait = 10 * time.Second

	// Time allowed to read the next pong message from the peer.
	pongWait = 60 * time.Second

	// Send pings to peer with this period. Must be less than pongWait.
	pingPeriod = (pongWait * 9) / 10
)

// Connection type represents the duplex websocket connection to Slack
type Connection struct {
	// The websocket connection.
	ws *websocket.Conn

	// waitgroup to wait for all go routines to terminate before attempting to reconnect.
	wg sync.WaitGroup

	// channel used to signal termination to socket writer go routine.
	finish chan struct{}

	// Buffered channel of outbound messages.
	out chan []byte

	// Buffered channel of inbound messages.
	in chan []byte

	// information about the current Slack connection and team settings.
	config Config
}

// write a message with the given message type and payload.
func (c *Connection) write(mt int, payload []byte) error {
	c.ws.SetWriteDeadline(time.Now().Add(writeWait))
	return c.ws.WriteMessage(mt, payload)
}

// socketWriter writes queued messages to the websocket connection.
func (c *Connection) socketWriter() {
	c.wg.Add(1)
	ticker := time.NewTicker(pingPeriod)
	defer func() {
		log.Println("Closing socket writer")
		ticker.Stop()
		c.ws.Close()
		c.wg.Done()
	}()
	for {
		select {
		case message, ok := <-c.out:
			if !ok {
				// channel closed so close the websocket
				c.write(websocket.CloseMessage, []byte{})
				log.Println("Closing socket")
				return
			}
			if err := c.write(websocket.TextMessage, message); err != nil {
				// error writing to websocket
				log.Printf("Error writing to slack websocket: %s", err)
				return
			}
		case <-ticker.C:
			// if idle send a ping
			if err := c.write(websocket.PingMessage, []byte{}); err != nil {
				log.Printf("Error sending ping on slack websocket: %s", err)
				return
			}
		case <-c.finish:
			return
		}
	}
}

// socketReader reads messages from the websocket connection and queues them for processing.
func (c *Connection) socketReader() {
	c.wg.Add(1)
	defer func() {
		log.Println("Closing socket reader")
		c.ws.Close()
		c.wg.Done()
	}()
	c.ws.SetReadDeadline(time.Now().Add(pongWait))
	c.ws.SetPongHandler(func(string) error { c.ws.SetReadDeadline(time.Now().Add(pongWait)); return nil })
	for {
		_, message, err := c.ws.ReadMessage()
		if err != nil {
			log.Printf("Error reading from slack websocket: %s", err)
			break
		}
		c.in <- message
	}
}

// Write the specified event to Slack.  The event is queued and then sent asynchronously.
func (c *Connection) Write(data []byte) {
	c.out <- data
}

// Read the next event from the connection to Slack or block until one is available
func (c *Connection) Read() []byte {
	return <-c.in
}

// start the connection.  Starts receiving and sending events from/to Slack on the connection and,
// in the event the connection is lost, will attempt to automatically reconnect to Slack.
func (c *Connection) start(reconnectionHandler func() (*Config, *websocket.Conn, error)) {
	go func() {
		for {
			c.finish = make(chan struct{})

			go c.socketWriter()
			c.socketReader()

			close(c.finish)
			c.wg.Wait()

			connected := false
			var ws *websocket.Conn
			var config *Config

			for i := 1; !connected; i = i * 2 {
				log.Printf("Lost connection to Slack - reconnecting in %d seconds", i)

				// wait 10 seconds before trying to reconnect
				time.Sleep(time.Duration(i) * time.Second)

				var err error
				config, ws, err = reconnectionHandler()

				if err != nil {
					log.Printf("Error reconnecting: %s", err)
				} else {
					log.Printf("Connected to Slack")
					connected = true
				}
			}

			c.ws = ws
			c.config = *config
		}
	}()
}
<|endoftext|>