<commit_before>package aws

import (
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ssm"
	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsSsmAssociation() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsSsmAssociationCreate,
		Read:   resourceAwsSsmAssociationRead,
		Update: resourceAwsSsmAssocationUpdate,
		Delete: resourceAwsSsmAssociationDelete,

		MigrateState:  resourceAwsSsmAssociationMigrateState,
		SchemaVersion: 1,

		Schema: map[string]*schema.Schema{
			"association_name": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"association_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"instance_id": {
				Type:     schema.TypeString,
				ForceNew: true,
				Optional: true,
			},
			"document_version": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"name": {
				Type:     schema.TypeString,
				ForceNew: true,
				Required: true,
			},
			"parameters": {
				Type:     schema.TypeMap,
				Optional: true,
				Computed: true,
			},
			"schedule_expression": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"output_location": {
				Type:     schema.TypeList,
				MaxItems: 1,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"s3_bucket_name": {
							Type:     schema.TypeString,
							Required: true,
						},
						"s3_key_prefix": {
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
			},
			"targets": {
				Type:     schema.TypeList,
				Optional: true,
				Computed: true,
				MaxItems: 5,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"key": {
							Type:     schema.TypeString,
							Required: true,
						},
						"values": {
							Type:     schema.TypeList,
							Required: true,
							Elem:     &schema.Schema{Type: schema.TypeString},
						},
					},
				},
			},
		},
	}
}

func resourceAwsSsmAssociationCreate(d *schema.ResourceData, meta interface{}) error {
	ssmconn := meta.(*AWSClient).ssmconn

	log.Printf("[DEBUG] SSM association create: %s", d.Id())

	assosciationInput := &ssm.CreateAssociationInput{
		Name: aws.String(d.Get("name").(string)),
	}

	if v, ok := d.GetOk("association_name"); ok {
		assosciationInput.AssociationName = aws.String(v.(string))
	}

	if v, ok := d.GetOk("instance_id"); ok {
		assosciationInput.InstanceId = aws.String(v.(string))
	}

	if v, ok := d.GetOk("document_version"); ok {
		assosciationInput.DocumentVersion = aws.String(v.(string))
	}

	if v, ok := d.GetOk("schedule_expression"); ok {
		assosciationInput.ScheduleExpression = aws.String(v.(string))
	}

	if v, ok := d.GetOk("parameters"); ok {
		assosciationInput.Parameters = expandSSMDocumentParameters(v.(map[string]interface{}))
	}

	if _, ok := d.GetOk("targets"); ok {
		assosciationInput.Targets = expandAwsSsmTargets(d)
	}

	if v, ok := d.GetOk("output_location"); ok {
		assosciationInput.OutputLocation = expandSSMAssociationOutputLocation(v.([]interface{}))
	}

	resp, err := ssmconn.CreateAssociation(assosciationInput)
	if err != nil {
		return errwrap.Wrapf("[ERROR] Error creating SSM association: {{err}}", err)
	}

	if resp.AssociationDescription == nil {
		return fmt.Errorf("[ERROR] AssociationDescription was nil")
	}

	d.SetId(*resp.AssociationDescription.AssociationId)
	d.Set("association_id", resp.AssociationDescription.AssociationId)

	return resourceAwsSsmAssociationRead(d, meta)
}

func resourceAwsSsmAssociationRead(d *schema.ResourceData, meta interface{}) error {
	ssmconn := meta.(*AWSClient).ssmconn

	log.Printf("[DEBUG] Reading SSM Association: %s", d.Id())

	params := &ssm.DescribeAssociationInput{
		AssociationId: aws.String(d.Id()),
	}

	resp, err := ssmconn.DescribeAssociation(params)

	if err != nil {
		return errwrap.Wrapf("[ERROR] Error reading SSM association: {{err}}", err)
	}
	if resp.AssociationDescription == nil {
		return fmt.Errorf("[ERROR] AssociationDescription was nil")
	}

	association := resp.AssociationDescription
	d.Set("association_name", association.AssociationName)
	d.Set("instance_id", association.InstanceId)
	d.Set("name", association.Name)
	d.Set("parameters", association.Parameters)
	d.Set("association_id", association.AssociationId)
	d.Set("schedule_expression", association.ScheduleExpression)
	d.Set("document_version", association.DocumentVersion)

	if err := d.Set("targets", flattenAwsSsmTargets(association.Targets)); err != nil {
		return fmt.Errorf("[DEBUG] Error setting targets error: %#v", err)
	}

	if err := d.Set("output_location", flattenAwsSsmAssociationOutoutLocation(association.OutputLocation)); err != nil {
		return fmt.Errorf("[DEBUG] Error setting output_location error: %#v", err)
	}

	return nil
}

func resourceAwsSsmAssocationUpdate(d *schema.ResourceData, meta interface{}) error {
	ssmconn := meta.(*AWSClient).ssmconn

	log.Printf("[DEBUG] SSM association update: %s", d.Id())

	associationInput := &ssm.UpdateAssociationInput{
		AssociationId: aws.String(d.Get("association_id").(string)),
	}

	if d.HasChange("association_name") {
		associationInput.AssociationName = aws.String(d.Get("association_name").(string))
	}

	if d.HasChange("schedule_expression") {
		associationInput.ScheduleExpression = aws.String(d.Get("schedule_expression").(string))
	}

	if d.HasChange("document_version") {
		associationInput.DocumentVersion = aws.String(d.Get("document_version").(string))
	}

	if d.HasChange("parameters") {
		associationInput.Parameters = expandSSMDocumentParameters(d.Get("parameters").(map[string]interface{}))
	}

	if d.HasChange("output_location") {
		associationInput.OutputLocation = expandSSMAssociationOutputLocation(d.Get("output_location").([]interface{}))
	}

	if d.HasChange("targets") {
		associationInput.Targets = expandAwsSsmTargets(d)
	}

	_, err := ssmconn.UpdateAssociation(associationInput)
	if err != nil {
		return errwrap.Wrapf("[ERROR] Error updating SSM association: {{err}}", err)
	}

	return resourceAwsSsmAssociationRead(d, meta)
}

func resourceAwsSsmAssociationDelete(d *schema.ResourceData, meta interface{}) error {
	ssmconn := meta.(*AWSClient).ssmconn

	log.Printf("[DEBUG] Deleting SSM Assosciation: %s", d.Id())

	params := &ssm.DeleteAssociationInput{
		AssociationId: aws.String(d.Get("association_id").(string)),
	}

	_, err := ssmconn.DeleteAssociation(params)

	if err != nil {
		return errwrap.Wrapf("[ERROR] Error deleting SSM association: {{err}}", err)
	}

	return nil
}

func expandSSMDocumentParameters(params map[string]interface{}) map[string][]*string {
	var docParams = make(map[string][]*string)
	for k, v := range params {
		values := make([]*string, 1)
		values[0] = aws.String(v.(string))
		docParams[k] = values
	}

	return docParams
}

func expandSSMAssociationOutputLocation(config []interface{}) *ssm.InstanceAssociationOutputLocation {
	if config == nil {
		return nil
	}

	//We only allow 1 Item so we can grab the first in the list only
	locationConfig := config[0].(map[string]interface{})

	S3OutputLocation := &ssm.S3OutputLocation{
		OutputS3BucketName: aws.String(locationConfig["s3_bucket_name"].(string)),
	}

	if v, ok := locationConfig["s3_key_prefix"]; ok {
		S3OutputLocation.OutputS3KeyPrefix = aws.String(v.(string))
	}

	return &ssm.InstanceAssociationOutputLocation{
		S3Location: S3OutputLocation,
	}
}

func flattenAwsSsmAssociationOutoutLocation(location *ssm.InstanceAssociationOutputLocation) []map[string]interface{} {
	if location == nil {
		return nil
	}

	result := make([]map[string]interface{}, 0)
	item := make(map[string]interface{})

	item["s3_bucket_name"] = *location.S3Location.OutputS3BucketName

	if location.S3Location.OutputS3KeyPrefix != nil {
		item["s3_key_prefix"] = *location.S3Location.OutputS3KeyPrefix
	}

	result = append(result, item)

	return result
}
<commit_msg>resource/ssm_association: remove resource from state in case of outside deletion<commit_after>package aws

import (
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ssm"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsSsmAssociation() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsSsmAssociationCreate,
		Read:   resourceAwsSsmAssociationRead,
		Update: resourceAwsSsmAssocationUpdate,
		Delete: resourceAwsSsmAssociationDelete,

		MigrateState:  resourceAwsSsmAssociationMigrateState,
		SchemaVersion: 1,

		Schema: map[string]*schema.Schema{
			"association_name": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"association_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"instance_id": {
				Type:     schema.TypeString,
				ForceNew: true,
				Optional: true,
			},
			"document_version": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"name": {
				Type:     schema.TypeString,
				ForceNew: true,
				Required: true,
			},
			"parameters": {
				Type:     schema.TypeMap,
				Optional: true,
				Computed: true,
			},
			"schedule_expression": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"output_location": {
				Type:     schema.TypeList,
				MaxItems: 1,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"s3_bucket_name": {
							Type:     schema.TypeString,
							Required: true,
						},
						"s3_key_prefix": {
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
			},
			"targets": {
				Type:     schema.TypeList,
				Optional: true,
				Computed: true,
				MaxItems: 5,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"key": {
							Type:     schema.TypeString,
							Required: true,
						},
						"values": {
							Type:     schema.TypeList,
							Required: true,
							Elem:     &schema.Schema{Type: schema.TypeString},
						},
					},
				},
			},
		},
	}
}

func resourceAwsSsmAssociationCreate(d *schema.ResourceData, meta interface{}) error {
	ssmconn := meta.(*AWSClient).ssmconn

	log.Printf("[DEBUG] SSM association create: %s", d.Id())

	assosciationInput := &ssm.CreateAssociationInput{
		Name: aws.String(d.Get("name").(string)),
	}

	if v, ok := d.GetOk("association_name"); ok {
		assosciationInput.AssociationName = aws.String(v.(string))
	}

	if v, ok := d.GetOk("instance_id"); ok {
		assosciationInput.InstanceId = aws.String(v.(string))
	}

	if v, ok := d.GetOk("document_version"); ok {
		assosciationInput.DocumentVersion = aws.String(v.(string))
	}

	if v, ok := d.GetOk("schedule_expression"); ok {
		assosciationInput.ScheduleExpression = aws.String(v.(string))
	}

	if v, ok := d.GetOk("parameters"); ok {
		assosciationInput.Parameters = expandSSMDocumentParameters(v.(map[string]interface{}))
	}

	if _, ok := d.GetOk("targets"); ok {
		assosciationInput.Targets = expandAwsSsmTargets(d)
	}

	if v, ok := d.GetOk("output_location"); ok {
		assosciationInput.OutputLocation = expandSSMAssociationOutputLocation(v.([]interface{}))
	}

	resp, err := ssmconn.CreateAssociation(assosciationInput)
	if err != nil {
		return fmt.Errorf("[ERROR] Error creating SSM association: %s", err)
	}

	if resp.AssociationDescription == nil {
		return fmt.Errorf("[ERROR] AssociationDescription was nil")
	}

	d.SetId(*resp.AssociationDescription.AssociationId)
	d.Set("association_id", resp.AssociationDescription.AssociationId)

	return resourceAwsSsmAssociationRead(d, meta)
}

func resourceAwsSsmAssociationRead(d *schema.ResourceData, meta interface{}) error {
	ssmconn := meta.(*AWSClient).ssmconn

	log.Printf("[DEBUG] Reading SSM Association: %s", d.Id())

	params := &ssm.DescribeAssociationInput{
		AssociationId: aws.String(d.Id()),
	}

	resp, err := ssmconn.DescribeAssociation(params)

	if err != nil {
		if isAWSErr(err, ssm.ErrCodeAssociationDoesNotExist, "") {
			d.SetId("")
			return nil
		}
		return fmt.Errorf("[ERROR] Error reading SSM association: %s", err)
	}
	if resp.AssociationDescription == nil {
		return fmt.Errorf("[ERROR] AssociationDescription was nil")
	}

	association := resp.AssociationDescription
	d.Set("association_name", association.AssociationName)
	d.Set("instance_id", association.InstanceId)
	d.Set("name", association.Name)
	d.Set("parameters", association.Parameters)
	d.Set("association_id", association.AssociationId)
	d.Set("schedule_expression", association.ScheduleExpression)
	d.Set("document_version", association.DocumentVersion)

	if err := d.Set("targets", flattenAwsSsmTargets(association.Targets)); err != nil {
		return fmt.Errorf("[DEBUG] Error setting targets error: %#v", err)
	}

	if err := d.Set("output_location", flattenAwsSsmAssociationOutoutLocation(association.OutputLocation)); err != nil {
		return fmt.Errorf("[DEBUG] Error setting output_location error: %#v", err)
	}

	return nil
}

func resourceAwsSsmAssocationUpdate(d *schema.ResourceData, meta interface{}) error {
	ssmconn := meta.(*AWSClient).ssmconn

	log.Printf("[DEBUG] SSM association update: %s", d.Id())

	associationInput := &ssm.UpdateAssociationInput{
		AssociationId: aws.String(d.Get("association_id").(string)),
	}

	if d.HasChange("association_name") {
		associationInput.AssociationName = aws.String(d.Get("association_name").(string))
	}

	if d.HasChange("schedule_expression") {
		associationInput.ScheduleExpression = aws.String(d.Get("schedule_expression").(string))
	}

	if d.HasChange("document_version") {
		associationInput.DocumentVersion = aws.String(d.Get("document_version").(string))
	}

	if d.HasChange("parameters") {
		associationInput.Parameters = expandSSMDocumentParameters(d.Get("parameters").(map[string]interface{}))
	}

	if d.HasChange("output_location") {
		associationInput.OutputLocation = expandSSMAssociationOutputLocation(d.Get("output_location").([]interface{}))
	}

	if d.HasChange("targets") {
		associationInput.Targets = expandAwsSsmTargets(d)
	}

	_, err := ssmconn.UpdateAssociation(associationInput)
	if err != nil {
		return fmt.Errorf("[ERROR] Error updating SSM association: %s", err)
	}

	return resourceAwsSsmAssociationRead(d, meta)
}

func resourceAwsSsmAssociationDelete(d *schema.ResourceData, meta interface{}) error {
	ssmconn := meta.(*AWSClient).ssmconn

	log.Printf("[DEBUG] Deleting SSM Assosciation: %s", d.Id())

	params := &ssm.DeleteAssociationInput{
		AssociationId: aws.String(d.Get("association_id").(string)),
	}

	_, err := ssmconn.DeleteAssociation(params)

	if err != nil {
		return fmt.Errorf("[ERROR] Error deleting SSM association: %s", err)
	}

	return nil
}

func expandSSMDocumentParameters(params map[string]interface{}) map[string][]*string {
	var docParams = make(map[string][]*string)
	for k, v := range params {
		values := make([]*string, 1)
		values[0] = aws.String(v.(string))
		docParams[k] = values
	}

	return docParams
}

func expandSSMAssociationOutputLocation(config []interface{}) *ssm.InstanceAssociationOutputLocation {
	if config == nil {
		return nil
	}

	//We only allow 1 Item so we can grab the first in the list only
	locationConfig := config[0].(map[string]interface{})

	S3OutputLocation := &ssm.S3OutputLocation{
		OutputS3BucketName: aws.String(locationConfig["s3_bucket_name"].(string)),
	}

	if v, ok := locationConfig["s3_key_prefix"]; ok {
		S3OutputLocation.OutputS3KeyPrefix = aws.String(v.(string))
	}

	return &ssm.InstanceAssociationOutputLocation{
		S3Location: S3OutputLocation,
	}
}

func flattenAwsSsmAssociationOutoutLocation(location *ssm.InstanceAssociationOutputLocation) []map[string]interface{} {
	if location == nil {
		return nil
	}

	result := make([]map[string]interface{}, 0)
	item := make(map[string]interface{})

	item["s3_bucket_name"] = *location.S3Location.OutputS3BucketName

	if location.S3Location.OutputS3KeyPrefix != nil {
		item["s3_key_prefix"] = *location.S3Location.OutputS3KeyPrefix
	}

	result = append(result, item)

	return result
}
<|endoftext|>


<commit_before>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by client-gen. DO NOT EDIT.

package fake

import (
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/discovery"
	fakediscovery "k8s.io/client-go/discovery/fake"
	"k8s.io/client-go/testing"
	clientset "k8s.io/metrics/pkg/client/clientset/versioned"
	metricsv1alpha1 "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1alpha1"
	fakemetricsv1alpha1 "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1alpha1/fake"
	metricsv1beta1 "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1"
	fakemetricsv1beta1 "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1/fake"
)

// NewSimpleClientset returns a clientset that will respond with the provided objects.
// It's backed by a very simple object tracker that processes creates, updates and deletions as-is,
// without applying any validations and/or defaults. It shouldn't be considered a replacement
// for a real clientset and is mostly useful in simple unit tests.
func NewSimpleClientset(objects ...runtime.Object) *Clientset {
	o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder())
	for _, obj := range objects {
		if err := o.Add(obj); err != nil {
			panic(err)
		}
	}

	cs := &Clientset{}
	cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake}
	cs.AddReactor("*", "*", testing.ObjectReaction(o))
	cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) {
		gvr := action.GetResource()
		ns := action.GetNamespace()
		watch, err := o.Watch(gvr, ns)
		if err != nil {
			return false, nil, err
		}
		return true, watch, nil
	})

	return cs
}

// Clientset implements clientset.Interface. Meant to be embedded into a
// struct to get a default implementation. This makes faking out just the method
// you want to test easier.
type Clientset struct {
	testing.Fake
	discovery *fakediscovery.FakeDiscovery
}

func (c *Clientset) Discovery() discovery.DiscoveryInterface {
	return c.discovery
}

var _ clientset.Interface = &Clientset{}

// MetricsV1alpha1 retrieves the MetricsV1alpha1Client
func (c *Clientset) MetricsV1alpha1() metricsv1alpha1.MetricsV1alpha1Interface {
	return &fakemetricsv1alpha1.FakeMetricsV1alpha1{Fake: &c.Fake}
}

// MetricsV1beta1 retrieves the MetricsV1beta1Client
func (c *Clientset) MetricsV1beta1() metricsv1beta1.MetricsV1beta1Interface {
	return &fakemetricsv1beta1.FakeMetricsV1beta1{Fake: &c.Fake}
}
<commit_msg>Expose object tracker for fake clientsets<commit_after>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by client-gen. DO NOT EDIT.

package fake

import (
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/discovery"
	fakediscovery "k8s.io/client-go/discovery/fake"
	"k8s.io/client-go/testing"
	clientset "k8s.io/metrics/pkg/client/clientset/versioned"
	metricsv1alpha1 "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1alpha1"
	fakemetricsv1alpha1 "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1alpha1/fake"
	metricsv1beta1 "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1"
	fakemetricsv1beta1 "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1/fake"
)

// NewSimpleClientset returns a clientset that will respond with the provided objects.
// It's backed by a very simple object tracker that processes creates, updates and deletions as-is,
// without applying any validations and/or defaults. It shouldn't be considered a replacement
// for a real clientset and is mostly useful in simple unit tests.
func NewSimpleClientset(objects ...runtime.Object) *Clientset {
	o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder())
	for _, obj := range objects {
		if err := o.Add(obj); err != nil {
			panic(err)
		}
	}

	cs := &Clientset{tracker: o}
	cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake}
	cs.AddReactor("*", "*", testing.ObjectReaction(o))
	cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) {
		gvr := action.GetResource()
		ns := action.GetNamespace()
		watch, err := o.Watch(gvr, ns)
		if err != nil {
			return false, nil, err
		}
		return true, watch, nil
	})

	return cs
}

// Clientset implements clientset.Interface. Meant to be embedded into a
// struct to get a default implementation. This makes faking out just the method
// you want to test easier.
type Clientset struct {
	testing.Fake
	discovery *fakediscovery.FakeDiscovery
	tracker   testing.ObjectTracker
}

func (c *Clientset) Discovery() discovery.DiscoveryInterface {
	return c.discovery
}

func (c *Clientset) Tracker() testing.ObjectTracker {
	return c.tracker
}

var _ clientset.Interface = &Clientset{}

// MetricsV1alpha1 retrieves the MetricsV1alpha1Client
func (c *Clientset) MetricsV1alpha1() metricsv1alpha1.MetricsV1alpha1Interface {
	return &fakemetricsv1alpha1.FakeMetricsV1alpha1{Fake: &c.Fake}
}

// MetricsV1beta1 retrieves the MetricsV1beta1Client
func (c *Clientset) MetricsV1beta1() metricsv1beta1.MetricsV1beta1Interface {
	return &fakemetricsv1beta1.FakeMetricsV1beta1{Fake: &c.Fake}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package knativeserving

import (
	"fmt"
	"context"

	mf "github.com/jcrossley3/manifestival"
	"go.uber.org/zap"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/equality"
	"k8s.io/apimachinery/pkg/api/errors"
	apierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"

	"k8s.io/client-go/tools/cache"

	"knative.dev/pkg/controller"
	servingv1alpha1 "knative.dev/serving-operator/pkg/apis/serving/v1alpha1"
	listers "knative.dev/serving-operator/pkg/client/listers/serving/v1alpha1"
	"knative.dev/serving-operator/pkg/reconciler"
	"knative.dev/serving-operator/pkg/reconciler/knativeserving/common"
	"knative.dev/serving-operator/version"
)

const (
	finalizerName  = "delete-knative-serving-manifest"
	creationChange = "creation"
	editChange     = "edit"
	deletionChange = "deletion"
)

var (
	// Platform-specific behavior to affect the installation
	platform common.Platforms
)

// Reconciler implements controller.Reconciler for Knativeserving resources.
type Reconciler struct {
	*reconciler.Base
	// Listers index properties about resources
	knativeServingLister listers.KnativeServingLister
	config               mf.Manifest
	servings             map[string]int64
}

// Check that our Reconciler implements controller.Reconciler
var _ controller.Reconciler = (*Reconciler)(nil)

// Reconcile compares the actual state with the desired, and attempts to
// converge the two. It then updates the Status block of the Knativeserving resource
// with the current status of the resource.
func (r *Reconciler) Reconcile(ctx context.Context, key string) error {
	// Convert the namespace/name string into a distinct namespace and name
	namespace, name, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		r.Logger.Errorf("invalid resource key: %s", key)
		return nil
	}
	// Get the KnativeServing resource with this namespace/name.
	original, err := r.knativeServingLister.KnativeServings(namespace).Get(name)
	if apierrs.IsNotFound(err) {
		return nil
	} else if err != nil {
		r.Logger.Error(err, "Error getting KnativeServing")
		return err
	}
	if original.GetDeletionTimestamp() != nil {
		if _, ok := r.servings[key]; ok {
			delete(r.servings, key)
			r.StatsReporter.ReportKnativeservingChange(key, deletionChange)
		}
		return r.delete(original)
	}
	// Keep track of the number and generation of KnativeServings in the cluster.
	newGen := original.Generation
	if oldGen, ok := r.servings[key]; ok {
		if newGen > oldGen {
			r.StatsReporter.ReportKnativeservingChange(key, editChange)
		} else if newGen < oldGen {
			return fmt.Errorf("reconciling obsolete generation of KnativeServing %s: newGen = %d and oldGen = %d", key, newGen, oldGen)
		}
	} else {
		// No metrics are emitted when newGen > 1: the first reconciling of
		// a new operator on an existing KnativeServing resource.
		if newGen == 1 {
			r.StatsReporter.ReportKnativeservingChange(key, creationChange)
		}
	}
	r.servings[key] = newGen

	// Don't modify the informers copy.
	knativeServing := original.DeepCopy()

	// Reconcile this copy of the KnativeServing resource and then write back any status
	// updates regardless of whether the reconciliation errored out.
	reconcileErr := r.reconcile(ctx, knativeServing)
	if equality.Semantic.DeepEqual(original.Status, knativeServing.Status) {
		// If we didn't change anything then don't call updateStatus.
		// This is important because the copy we loaded from the informer's
		// cache may be stale and we don't want to overwrite a prior update
		// to status with this stale state.
	} else if err = r.updateStatus(knativeServing); err != nil {
		r.Logger.Warnw("Failed to update knativeServing status", zap.Error(err))
		r.Recorder.Eventf(knativeServing, corev1.EventTypeWarning, "UpdateFailed",
			"Failed to update status for KnativeServing %q: %v", knativeServing.Name, err)
		return err
	}
	if reconcileErr != nil {
		r.Recorder.Event(knativeServing, corev1.EventTypeWarning, "InternalError", reconcileErr.Error())
		return reconcileErr
	}
	return nil
}

func (r *Reconciler) reconcile(ctx context.Context, ks *servingv1alpha1.KnativeServing) error {
	reqLogger := r.Logger.With(zap.String("Request.Namespace", ks.Namespace)).With("Request.Name", ks.Name)
	reqLogger.Infow("Reconciling KnativeServing", "status", ks.Status)

	stages := []func(*mf.Manifest, *servingv1alpha1.KnativeServing) error{
		r.ensureFinalizer,
		r.initStatus,
		r.install,
		r.checkDeployments,
		r.deleteObsoleteResources,
	}

	manifest, err := r.transform(ks)
	if err != nil {
		ks.Status.MarkInstallFailed(err.Error())
		return err
	}

	for _, stage := range stages {
		if err := stage(manifest, ks); err != nil {
			return err
		}
	}
	reqLogger.Infow("Reconcile stages complete", "status", ks.Status)
	return nil
}

// Transform the resources
func (r *Reconciler) transform(instance *servingv1alpha1.KnativeServing) (*mf.Manifest, error) {
	r.Logger.Debug("Transforming manifest")
	transforms, err := platform.Transformers(r.KubeClientSet, instance, r.Logger)
	if err != nil {
		return nil, err
	}
	return r.config.Transform(transforms...)
}

// Update the status subresource
func (r *Reconciler) updateStatus(instance *servingv1alpha1.KnativeServing) error {
	afterUpdate, err := r.KnativeServingClientSet.OperatorV1alpha1().KnativeServings(instance.Namespace).UpdateStatus(instance)

	if err != nil {
		return err
	}
	// TODO: We shouldn't rely on mutability and return the updated entities from functions instead.
	afterUpdate.DeepCopyInto(instance)
	return nil
}

// Initialize status conditions
func (r *Reconciler) initStatus(_ *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	r.Logger.Debug("Initializing status")
	if len(instance.Status.Conditions) == 0 {
		instance.Status.InitializeConditions()
		if err := r.updateStatus(instance); err != nil {
			return err
		}
	}
	return nil
}

// Apply the manifest resources
func (r *Reconciler) install(manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	r.Logger.Debug("Installing manifest")
	if err := manifest.ApplyAll(); err != nil {
		instance.Status.MarkInstallFailed(err.Error())
		return err
	}
	instance.Status.MarkInstallSucceeded()
	instance.Status.Version = version.Version
	return nil
}

// Check for all deployments available
func (r *Reconciler) checkDeployments(manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	r.Logger.Debug("Checking deployments")
	available := func(d *appsv1.Deployment) bool {
		for _, c := range d.Status.Conditions {
			if c.Type == appsv1.DeploymentAvailable && c.Status == corev1.ConditionTrue {
				return true
			}
		}
		return false
	}
	for _, u := range manifest.Resources {
		if u.GetKind() == "Deployment" {
			deployment, err := r.KubeClientSet.AppsV1().Deployments(u.GetNamespace()).Get(u.GetName(), metav1.GetOptions{})
			if err != nil {
				instance.Status.MarkDeploymentsNotReady()
				if errors.IsNotFound(err) {
					return nil
				}
				return err
			}
			if !available(deployment) {
				instance.Status.MarkDeploymentsNotReady()
				return nil
			}
		}
	}
	instance.Status.MarkDeploymentsAvailable()
	return nil
}

// ensureFinalizer attaches a "delete manifest" finalizer to the instance
func (r *Reconciler) ensureFinalizer(manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	for _, finalizer := range instance.GetFinalizers() {
		if finalizer == finalizerName {
			return nil
		}
	}
	instance.SetFinalizers(append(instance.GetFinalizers(), finalizerName))
	instance, err := r.KnativeServingClientSet.OperatorV1alpha1().KnativeServings(instance.Namespace).Update(instance)
	return err
}

// delete all the resources in the release manifest
func (r *Reconciler) delete(instance *servingv1alpha1.KnativeServing) error {
	if len(instance.GetFinalizers()) == 0 || instance.GetFinalizers()[0] != finalizerName {
		return nil
	}
	if len(r.servings) == 0 {
		if err := r.config.DeleteAll(&metav1.DeleteOptions{}); err != nil {
			return err
		}
	}
	// The deletionTimestamp might've changed. Fetch the resource again.
	refetched, err := r.knativeServingLister.KnativeServings(instance.Namespace).Get(instance.Name)
	if err != nil {
		return err
	}
	refetched.SetFinalizers(refetched.GetFinalizers()[1:])
	_, err = r.KnativeServingClientSet.OperatorV1alpha1().KnativeServings(refetched.Namespace).Update(refetched)
	return err
}

// Delete obsolete resources from previous versions
func (r *Reconciler) deleteObsoleteResources(manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	// istio-system resources from 0.3
	resource := &unstructured.Unstructured{}
	resource.SetNamespace("istio-system")
	resource.SetName("knative-ingressgateway")
	resource.SetAPIVersion("v1")
	resource.SetKind("Service")
	if err := manifest.Delete(resource, &metav1.DeleteOptions{}); err != nil {
		return err
	}
	resource.SetAPIVersion("apps/v1")
	resource.SetKind("Deployment")
	if err := manifest.Delete(resource, &metav1.DeleteOptions{}); err != nil {
		return err
	}
	resource.SetAPIVersion("autoscaling/v1")
	resource.SetKind("HorizontalPodAutoscaler")
	if err := manifest.Delete(resource, &metav1.DeleteOptions{}); err != nil {
		return err
	}
	// config-controller from 0.5
	resource.SetNamespace(instance.GetNamespace())
	resource.SetName("config-controller")
	resource.SetAPIVersion("v1")
	resource.SetKind("ConfigMap")
	if err := manifest.Delete(resource, &metav1.DeleteOptions{}); err != nil {
		return err
	}
	return nil
}
<commit_msg>golang format tools (#244)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package knativeserving

import (
	"context"
	"fmt"

	mf "github.com/jcrossley3/manifestival"
	"go.uber.org/zap"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/equality"
	"k8s.io/apimachinery/pkg/api/errors"
	apierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"

	"k8s.io/client-go/tools/cache"

	"knative.dev/pkg/controller"
	servingv1alpha1 "knative.dev/serving-operator/pkg/apis/serving/v1alpha1"
	listers "knative.dev/serving-operator/pkg/client/listers/serving/v1alpha1"
	"knative.dev/serving-operator/pkg/reconciler"
	"knative.dev/serving-operator/pkg/reconciler/knativeserving/common"
	"knative.dev/serving-operator/version"
)

const (
	finalizerName  = "delete-knative-serving-manifest"
	creationChange = "creation"
	editChange     = "edit"
	deletionChange = "deletion"
)

var (
	// Platform-specific behavior to affect the installation
	platform common.Platforms
)

// Reconciler implements controller.Reconciler for Knativeserving resources.
type Reconciler struct {
	*reconciler.Base
	// Listers index properties about resources
	knativeServingLister listers.KnativeServingLister
	config               mf.Manifest
	servings             map[string]int64
}

// Check that our Reconciler implements controller.Reconciler
var _ controller.Reconciler = (*Reconciler)(nil)

// Reconcile compares the actual state with the desired, and attempts to
// converge the two. It then updates the Status block of the Knativeserving resource
// with the current status of the resource.
func (r *Reconciler) Reconcile(ctx context.Context, key string) error {
	// Convert the namespace/name string into a distinct namespace and name
	namespace, name, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		r.Logger.Errorf("invalid resource key: %s", key)
		return nil
	}
	// Get the KnativeServing resource with this namespace/name.
	original, err := r.knativeServingLister.KnativeServings(namespace).Get(name)
	if apierrs.IsNotFound(err) {
		return nil
	} else if err != nil {
		r.Logger.Error(err, "Error getting KnativeServing")
		return err
	}
	if original.GetDeletionTimestamp() != nil {
		if _, ok := r.servings[key]; ok {
			delete(r.servings, key)
			r.StatsReporter.ReportKnativeservingChange(key, deletionChange)
		}
		return r.delete(original)
	}
	// Keep track of the number and generation of KnativeServings in the cluster.
	newGen := original.Generation
	if oldGen, ok := r.servings[key]; ok {
		if newGen > oldGen {
			r.StatsReporter.ReportKnativeservingChange(key, editChange)
		} else if newGen < oldGen {
			return fmt.Errorf("reconciling obsolete generation of KnativeServing %s: newGen = %d and oldGen = %d", key, newGen, oldGen)
		}
	} else {
		// No metrics are emitted when newGen > 1: the first reconciling of
		// a new operator on an existing KnativeServing resource.
		if newGen == 1 {
			r.StatsReporter.ReportKnativeservingChange(key, creationChange)
		}
	}
	r.servings[key] = newGen

	// Don't modify the informers copy.
	knativeServing := original.DeepCopy()

	// Reconcile this copy of the KnativeServing resource and then write back any status
	// updates regardless of whether the reconciliation errored out.
	reconcileErr := r.reconcile(ctx, knativeServing)
	if equality.Semantic.DeepEqual(original.Status, knativeServing.Status) {
		// If we didn't change anything then don't call updateStatus.
		// This is important because the copy we loaded from the informer's
		// cache may be stale and we don't want to overwrite a prior update
		// to status with this stale state.
	} else if err = r.updateStatus(knativeServing); err != nil {
		r.Logger.Warnw("Failed to update knativeServing status", zap.Error(err))
		r.Recorder.Eventf(knativeServing, corev1.EventTypeWarning, "UpdateFailed",
			"Failed to update status for KnativeServing %q: %v", knativeServing.Name, err)
		return err
	}
	if reconcileErr != nil {
		r.Recorder.Event(knativeServing, corev1.EventTypeWarning, "InternalError", reconcileErr.Error())
		return reconcileErr
	}
	return nil
}

func (r *Reconciler) reconcile(ctx context.Context, ks *servingv1alpha1.KnativeServing) error {
	reqLogger := r.Logger.With(zap.String("Request.Namespace", ks.Namespace)).With("Request.Name", ks.Name)
	reqLogger.Infow("Reconciling KnativeServing", "status", ks.Status)

	stages := []func(*mf.Manifest, *servingv1alpha1.KnativeServing) error{
		r.ensureFinalizer,
		r.initStatus,
		r.install,
		r.checkDeployments,
		r.deleteObsoleteResources,
	}

	manifest, err := r.transform(ks)
	if err != nil {
		ks.Status.MarkInstallFailed(err.Error())
		return err
	}

	for _, stage := range stages {
		if err := stage(manifest, ks); err != nil {
			return err
		}
	}
	reqLogger.Infow("Reconcile stages complete", "status", ks.Status)
	return nil
}

// Transform the resources
func (r *Reconciler) transform(instance *servingv1alpha1.KnativeServing) (*mf.Manifest, error) {
	r.Logger.Debug("Transforming manifest")
	transforms, err := platform.Transformers(r.KubeClientSet, instance, r.Logger)
	if err != nil {
		return nil, err
	}
	return r.config.Transform(transforms...)
}

// Update the status subresource
func (r *Reconciler) updateStatus(instance *servingv1alpha1.KnativeServing) error {
	afterUpdate, err := r.KnativeServingClientSet.OperatorV1alpha1().KnativeServings(instance.Namespace).UpdateStatus(instance)

	if err != nil {
		return err
	}
	// TODO: We shouldn't rely on mutability and return the updated entities from functions instead.
	afterUpdate.DeepCopyInto(instance)
	return nil
}

// Initialize status conditions
func (r *Reconciler) initStatus(_ *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	r.Logger.Debug("Initializing status")
	if len(instance.Status.Conditions) == 0 {
		instance.Status.InitializeConditions()
		if err := r.updateStatus(instance); err != nil {
			return err
		}
	}
	return nil
}

// Apply the manifest resources
func (r *Reconciler) install(manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	r.Logger.Debug("Installing manifest")
	if err := manifest.ApplyAll(); err != nil {
		instance.Status.MarkInstallFailed(err.Error())
		return err
	}
	instance.Status.MarkInstallSucceeded()
	instance.Status.Version = version.Version
	return nil
}

// Check for all deployments available
func (r *Reconciler) checkDeployments(manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	r.Logger.Debug("Checking deployments")
	available := func(d *appsv1.Deployment) bool {
		for _, c := range d.Status.Conditions {
			if c.Type == appsv1.DeploymentAvailable && c.Status == corev1.ConditionTrue {
				return true
			}
		}
		return false
	}
	for _, u := range manifest.Resources {
		if u.GetKind() == "Deployment" {
			deployment, err := r.KubeClientSet.AppsV1().Deployments(u.GetNamespace()).Get(u.GetName(), metav1.GetOptions{})
			if err != nil {
				instance.Status.MarkDeploymentsNotReady()
				if errors.IsNotFound(err) {
					return nil
				}
				return err
			}
			if !available(deployment) {
				instance.Status.MarkDeploymentsNotReady()
				return nil
			}
		}
	}
	instance.Status.MarkDeploymentsAvailable()
	return nil
}

// ensureFinalizer attaches a "delete manifest" finalizer to the instance
func (r *Reconciler) ensureFinalizer(manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	for _, finalizer := range instance.GetFinalizers() {
		if finalizer == finalizerName {
			return nil
		}
	}
	instance.SetFinalizers(append(instance.GetFinalizers(), finalizerName))
	instance, err := r.KnativeServingClientSet.OperatorV1alpha1().KnativeServings(instance.Namespace).Update(instance)
	return err
}

// delete all the resources in the release manifest
func (r *Reconciler) delete(instance *servingv1alpha1.KnativeServing) error {
	if len(instance.GetFinalizers()) == 0 || instance.GetFinalizers()[0] != finalizerName {
		return nil
	}
	if len(r.servings) == 0 {
		if err := r.config.DeleteAll(&metav1.DeleteOptions{}); err != nil {
			return err
		}
	}
	// The deletionTimestamp might've changed. Fetch the resource again.
	refetched, err := r.knativeServingLister.KnativeServings(instance.Namespace).Get(instance.Name)
	if err != nil {
		return err
	}
	refetched.SetFinalizers(refetched.GetFinalizers()[1:])
	_, err = r.KnativeServingClientSet.OperatorV1alpha1().KnativeServings(refetched.Namespace).Update(refetched)
	return err
}

// Delete obsolete resources from previous versions
func (r *Reconciler) deleteObsoleteResources(manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	// istio-system resources from 0.3
	resource := &unstructured.Unstructured{}
	resource.SetNamespace("istio-system")
	resource.SetName("knative-ingressgateway")
	resource.SetAPIVersion("v1")
	resource.SetKind("Service")
	if err := manifest.Delete(resource, &metav1.DeleteOptions{}); err != nil {
		return err
	}
	resource.SetAPIVersion("apps/v1")
	resource.SetKind("Deployment")
	if err := manifest.Delete(resource, &metav1.DeleteOptions{}); err != nil {
		return err
	}
	resource.SetAPIVersion("autoscaling/v1")
	resource.SetKind("HorizontalPodAutoscaler")
	if err := manifest.Delete(resource, &metav1.DeleteOptions{}); err != nil {
		return err
	}
	// config-controller from 0.5
	resource.SetNamespace(instance.GetNamespace())
	resource.SetName("config-controller")
	resource.SetAPIVersion("v1")
	resource.SetKind("ConfigMap")
	if err := manifest.Delete(resource, &metav1.DeleteOptions{}); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package read

import "github.com/qlova/uct/compiler"
import "github.com/qlova/ilang/syntax/symbols"

import "github.com/qlova/ilang/types/letter"
import "github.com/qlova/ilang/types/text"

var Expression = compiler.Expression {
	Name: compiler.Translatable{
		compiler.English: "read",
		compiler.Maori: "rīti",
	},

	OnScan: func(c *compiler.Compiler) compiler.Type {
		c.Expecting(symbols.FunctionCallBegin)

		if c.ScanIf(symbols.FunctionCallEnd) {
			c.Int('\n')
			c.List()
			c.Open()
			c.Flip()
			c.Read()
			return text.Type
		}

		var argument = c.ScanExpression()
		c.Expecting(symbols.FunctionCallEnd)

		switch {
			case letter.Type.Equals(argument):

				c.List()
				c.Open()
				c.Flip()
				c.Read()

				return text.Type

			default:
				c.Unimplemented()

		}

		return compiler.Type{}
	},
}
<commit_msg>Add Moari language to read.<commit_after>package read

import "github.com/qlova/uct/compiler"
import "github.com/qlova/ilang/syntax/symbols"

import "github.com/qlova/ilang/types/letter"
import "github.com/qlova/ilang/types/text"

var Name = compiler.Translatable{
	compiler.English: "read",
	compiler.Maori: "rīti",
}

var Expression = compiler.Expression {
	Name: Name,

	OnScan: func(c *compiler.Compiler) compiler.Type {
		c.Expecting(symbols.FunctionCallBegin)

		if c.ScanIf(symbols.FunctionCallEnd) {
			c.Int('\n')
			c.List()
			c.Open()
			c.Flip()
			c.Read()
			return text.Type
		}

		var argument = c.ScanExpression()
		c.Expecting(symbols.FunctionCallEnd)

		switch {
			case letter.Type.Equals(argument):

				c.List()
				c.Open()
				c.Flip()
				c.Read()

				return text.Type

			default:
				c.Unimplemented()

		}

		return compiler.Type{}
	},
}
<|endoftext|>


<commit_before>// Package pagerduty provides an API library for interacting with PagerDuty.
// Much of this code is inspired by https://github.com/google/go-github <3
package pagerduty

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"reflect"

	"github.com/google/go-querystring/query"
)

// Client is an API client
type Client struct {
	client    *http.Client
	Subdomain string
	APIKey    string
	BaseURL   *url.URL

	Incidents *IncidentsService
	Users     *UsersService
}

// New returns a Client with the default http.Client
func New(sub, key string) *Client {
	return NewClient(sub, key, nil)
}

// NewClient returns a default cliente
func NewClient(sub, key string, httpClient *http.Client) *Client {
	if httpClient == nil {
		httpClient = http.DefaultClient
	}
	u, _ := url.Parse(fmt.Sprintf("https://%s.pagerduty.com/api/v1/", sub))
	client := &Client{
		client:    httpClient,
		APIKey:    key,
		Subdomain: sub,
		BaseURL:   u,
	}

	client.Incidents = &IncidentsService{client: client}
	client.Users = &UsersService{client: client}

	return client
}

// addOptions adds the parameters in opt as URL query parameters to s.  opt
// must be a struct whose fields may contain "url" tags.
func addOptions(s string, opt interface{}) (string, error) {
	v := reflect.ValueOf(opt)
	if v.Kind() == reflect.Ptr && v.IsNil() {
		return s, nil
	}

	u, err := url.Parse(s)
	if err != nil {
		return s, err
	}

	qs, err := query.Values(opt)
	if err != nil {
		return s, err
	}

	u.RawQuery = qs.Encode()

	return u.String(), nil
}

// NewRequest builds an http.Request, resolves relative URLs, and sets HTTP headers
func (c *Client) NewRequest(meth string, path string, input interface{}) (*http.Request, error) {
	ref, err := url.Parse(path)
	if err != nil {
		return nil, err
	}

	u := c.BaseURL.ResolveReference(ref)

	buf := new(bytes.Buffer)
	if input != nil {
		if err := json.NewEncoder(buf).Encode(input); err != nil {
			return nil, err
		}
	}

	req, err := http.NewRequest(meth, u.String(), buf)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Authorization", `Token token=`+c.APIKey)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/json")

	return req, nil
}

// Get makes a GET request
func (c *Client) Get(path string, output interface{}) (*http.Response, error) {
	req, err := c.NewRequest("GET", path, nil)
	if err != nil {
		return nil, err
	}

	return c.Do(req, output)
}

// Post makes a POST request
func (c *Client) Post(path string, input, output interface{}) (*http.Response, error) {
	req, err := c.NewRequest("POST", path, input)
	if err != nil {
		return nil, err
	}

	return c.Do(req, output)
}

// Put makes a PUT request
func (c *Client) Put(path string, input, output interface{}) (*http.Response, error) {
	req, err := c.NewRequest("PUT", path, input)
	if err != nil {
		return nil, err
	}

	return c.Do(req, output)
}

// Delete makes a DELETE request
func (c *Client) Delete(path string, input, output interface{}) (*http.Response, error) {
	req, err := c.NewRequest("DELETE", path, input)
	if err != nil {
		return nil, err
	}

	return c.Do(req, output)
}

// Do performs the request
func (c *Client) Do(req *http.Request, output interface{}) (*http.Response, error) {
	res, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()

	err = CheckResponse(res)
	if err != nil {
		return res, err
	}
	if output != nil {
		if w, ok := output.(io.Writer); ok {
			io.Copy(w, res.Body)
		} else {
			err = json.NewDecoder(res.Body).Decode(output)
		}
	}

	return res, err
}

// An ErrorResponse represents one or more errors created by an API request.
type ErrorResponse struct {
	Response *http.Response
	Message  string    `json:"message"`
	Code     ErrorCode `json:"code"`
	Errors   []string  `json:"errors"`
}

func (r *ErrorResponse) Error() string {
	return fmt.Sprintf("%v %v: %d %v %+v", r.Response.Request.Method, r.Response.Request.URL, r.Response.StatusCode, r.Message, r.Errors)
}

// CheckResponse checks the API response for errors, returning them if
// present. A response is considered to have an error if it has a status code
// outside the 200 range.
func CheckResponse(r *http.Response) error {
	if c := r.StatusCode; 200 <= c && c <= 299 {
		return nil
	}

	v := &ErrorResponse{Response: r}
	d, err := ioutil.ReadAll(r.Body)
	if err == nil && d != nil {
		json.Unmarshal(d, v)
	}

	return v
}
<commit_msg>fix wonky comments<commit_after>// Package pagerduty provides an API library for interacting with PagerDuty.
// Much of this code is inspired by https://github.com/google/go-github <3
package pagerduty

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"reflect"

	"github.com/google/go-querystring/query"
)

// Client is an API client
type Client struct {
	client    *http.Client
	Subdomain string
	APIKey    string
	BaseURL   *url.URL

	Incidents *IncidentsService
	Users     *UsersService
}

// New returns a Client with the default http.Client
func New(sub, key string) *Client {
	return NewClient(sub, key, nil)
}

// NewClient returns a default client
func NewClient(sub, key string, httpClient *http.Client) *Client {
	if httpClient == nil {
		httpClient = http.DefaultClient
	}
	u, _ := url.Parse(fmt.Sprintf("https://%s.pagerduty.com/api/v1/", sub))
	client := &Client{
		client:    httpClient,
		APIKey:    key,
		Subdomain: sub,
		BaseURL:   u,
	}

	client.Incidents = &IncidentsService{client: client}
	client.Users = &UsersService{client: client}

	return client
}

// addOptions adds the parameters in opt as URL query parameters to s.
// opt must be a struct whose fields may contain "url" tags.
func addOptions(s string, opt interface{}) (string, error) {
	v := reflect.ValueOf(opt)
	if v.Kind() == reflect.Ptr && v.IsNil() {
		return s, nil
	}

	u, err := url.Parse(s)
	if err != nil {
		return s, err
	}

	qs, err := query.Values(opt)
	if err != nil {
		return s, err
	}

	u.RawQuery = qs.Encode()

	return u.String(), nil
}

// NewRequest builds an http.Request, resolves relative URLs, and sets HTTP headers
func (c *Client) NewRequest(meth string, path string, input interface{}) (*http.Request, error) {
	ref, err := url.Parse(path)
	if err != nil {
		return nil, err
	}

	u := c.BaseURL.ResolveReference(ref)

	buf := new(bytes.Buffer)
	if input != nil {
		if err := json.NewEncoder(buf).Encode(input); err != nil {
			return nil, err
		}
	}

	req, err := http.NewRequest(meth, u.String(), buf)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Authorization", `Token token=`+c.APIKey)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/json")

	return req, nil
}

// Get makes a GET request
func (c *Client) Get(path string, output interface{}) (*http.Response, error) {
	req, err := c.NewRequest("GET", path, nil)
	if err != nil {
		return nil, err
	}

	return c.Do(req, output)
}

// Post makes a POST request
func (c *Client) Post(path string, input, output interface{}) (*http.Response, error) {
	req, err := c.NewRequest("POST", path, input)
	if err != nil {
		return nil, err
	}

	return c.Do(req, output)
}

// Put makes a PUT request
func (c *Client) Put(path string, input, output interface{}) (*http.Response, error) {
	req, err := c.NewRequest("PUT", path, input)
	if err != nil {
		return nil, err
	}

	return c.Do(req, output)
}

// Delete makes a DELETE request
func (c *Client) Delete(path string, input, output interface{}) (*http.Response, error) {
	req, err := c.NewRequest("DELETE", path, input)
	if err != nil {
		return nil, err
	}

	return c.Do(req, output)
}

// Do performs the request
func (c *Client) Do(req *http.Request, output interface{}) (*http.Response, error) {
	res, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()

	err = CheckResponse(res)
	if err != nil {
		return res, err
	}
	if output != nil {
		if w, ok := output.(io.Writer); ok {
			io.Copy(w, res.Body)
		} else {
			err = json.NewDecoder(res.Body).Decode(output)
		}
	}

	return res, err
}

// An ErrorResponse represents one or more errors created by an API request.
type ErrorResponse struct {
	Response *http.Response
	Message  string    `json:"message"`
	Code     ErrorCode `json:"code"`
	Errors   []string  `json:"errors"`
}

func (r *ErrorResponse) Error() string {
	return fmt.Sprintf("%v %v: %d %v %+v", r.Response.Request.Method, r.Response.Request.URL, r.Response.StatusCode, r.Message, r.Errors)
}

// CheckResponse checks the API response for errors, returning them if
// present. A response is considered to have an error if it has a status code
// outside the 200 range.
func CheckResponse(r *http.Response) error {
	if c := r.StatusCode; 200 <= c && c <= 299 {
		return nil
	}

	v := &ErrorResponse{Response: r}
	d, err := ioutil.ReadAll(r.Body)
	if err == nil && d != nil {
		json.Unmarshal(d, v)
	}

	return v
}
<|endoftext|>


<commit_before>package amv

import (
	"strings"

	"github.com/animenotifier/notify.moe/arn"
	"github.com/animenotifier/notify.moe/assets"
)

func getOpenGraph(amv *arn.AMV) *arn.OpenGraph {
	openGraph := &arn.OpenGraph{
		Tags: map[string]string{
			"og:title":     amv.Title.ByUser(nil) + " (AMV)",
			"og:url":       "https://" + assets.Domain + amv.Link(),
			"og:site_name": assets.Domain,
			"og:type":      "video.other",
		},
		Meta: map[string]string{},
	}

	openGraph.Tags["og:description"] = strings.Join(amv.Tags, ", ")

	if amv.File != "" {
		openGraph.Tags["og:video"] = "https://" + assets.Domain + "/videos/amvs/" + amv.File
		openGraph.Tags["og:video:type"] = "video/webm"
		openGraph.Tags["og:video:width"] = "640"
		openGraph.Tags["og:video:height"] = "360"

		openGraph.Meta["twitter:player"] = openGraph.Tags["og:video"]
		openGraph.Meta["twitter:player:width"] = openGraph.Tags["og:video:width"]
		openGraph.Meta["twitter:player:height"] = openGraph.Tags["og:video:height"]
		openGraph.Meta["twitter:player:stream"] = openGraph.Tags["og:video"]
		openGraph.Meta["twitter:player:stream:content_type"] = openGraph.Tags["og:video:type"]
	}

	return openGraph
}
<commit_msg>Updated AMV OpenGraph data<commit_after>package amv

import (
	"strings"

	"github.com/animenotifier/notify.moe/arn"
	"github.com/animenotifier/notify.moe/assets"
)

func getOpenGraph(amv *arn.AMV) *arn.OpenGraph {
	openGraph := &arn.OpenGraph{
		Tags: map[string]string{
			"og:title":     amv.Title.ByUser(nil) + " (AMV)",
			"og:url":       "https://" + assets.Domain + amv.Link(),
			"og:site_name": assets.Domain,
			"og:type":      "video.other",
		},
		Meta: map[string]string{},
	}

	openGraph.Tags["og:description"] = strings.Join(amv.Tags, ", ")

	if amv.File != "" {
		openGraph.Tags["og:video"] = amv.VideoLink()
		openGraph.Tags["og:video:type"] = "video/webm"
		openGraph.Tags["og:video:width"] = "640"
		openGraph.Tags["og:video:height"] = "360"

		openGraph.Meta["twitter:player"] = openGraph.Tags["og:video"]
		openGraph.Meta["twitter:player:width"] = openGraph.Tags["og:video:width"]
		openGraph.Meta["twitter:player:height"] = openGraph.Tags["og:video:height"]
		openGraph.Meta["twitter:player:stream"] = openGraph.Tags["og:video"]
		openGraph.Meta["twitter:player:stream:content_type"] = openGraph.Tags["og:video:type"]
	}

	return openGraph
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/http"

	"github.com/gorilla/mux"
	"github.com/gorilla/rpc/v2"
	"github.com/gorilla/rpc/v2/json"
	"github.com/nofdev/fastforward/provisioning/api/rpc/json/openstack"
)

func main() {
	s := rpc.NewServer()
	log.Printf("Playback API started")
	s.RegisterCodec(json.NewCodec(), "application/json")
	openstack := new(openstack.OpenStack)
	s.RegisterService(openstack, "")
	log.Printf("Register OpenStack service")
	r := mux.NewRouter()
	r.Handle("/v1", s)
	log.Printf("Handle API version 1")
	log.Printf("Listen on port 7001")
	http.ListenAndServe(":7001", r)
}<commit_msg>Update doc<commit_after>// Package main is the JSON-RPC 2.0 API server for playback.
package main

import (
	"log"
	"net/http"

	"github.com/gorilla/mux"
	"github.com/gorilla/rpc/v2"
	"github.com/gorilla/rpc/v2/json"
	"github.com/nofdev/fastforward/provisioning/api/rpc/json/openstack"
)

func main() {
	s := rpc.NewServer()
	log.Printf("Playback API started")
	s.RegisterCodec(json.NewCodec(), "application/json")
	openstack := new(openstack.OpenStack)
	s.RegisterService(openstack, "")
	log.Printf("Register OpenStack service")
	r := mux.NewRouter()
	r.Handle("/v1", s)
	log.Printf("Handle API version 1")
	log.Printf("Listen on port 7001")
	http.ListenAndServe(":7001", r)
}<|endoftext|>


<commit_before>package systems

import (
	"fmt"
	"sync"

	"time"

	am "github.com/Ariemeth/quantum-pulse/assets"
	"github.com/Ariemeth/quantum-pulse/components"
	"github.com/Ariemeth/quantum-pulse/entity"
	"github.com/go-gl/gl/v4.1-core/gl"
	"github.com/go-gl/glfw/v3.2/glfw"
)

const (
	// SystemTypeRenderer is the name of the renderer system.
	SystemTypeRenderer = "renderer"
)

// Renderer provides the interface needed to process the rendering of Entities.  Each time Process is called all Entities will be rendered.
type Renderer interface {
	System
	// Process renders all renderable entities.
	Process()
	// LoadCamera sets the camera to be used by the display.
	LoadCamera(camera components.Camera)
	Stop()
}

type renderer struct {
	entities     map[string]renderable
	assets       *am.AssetManager
	camera       components.Camera
	mainFunc     func(f func())
	window       *glfw.Window
	remove       chan entity.Entity
	add          chan entity.Entity
	quit         chan interface{}
	runningLock  sync.Mutex
	isRunning    bool
	requirements []string
}

// NewRenderer creates a new renderer system.  The renderer system handles rendering all renderable Entities to the screen.
func NewRenderer(assetManager *am.AssetManager, mainFunc func(f func()), window *glfw.Window) Renderer {
	r := renderer{
		entities:     make(map[string]renderable),
		assets:       assetManager,
		camera:       components.NewCamera(),
		mainFunc:     mainFunc,
		window:       window,
		remove:       make(chan entity.Entity, 0),
		add:          make(chan entity.Entity, 0),
		quit:         make(chan interface{}),
		requirements: []string{""}, //TODO eventually add the actual requirements
	}

	r.Start()

	return &r
}

// SystemType retrieves the type of system such as renderer, mover, etc.
func (r *renderer) SystemType() string {
	return SystemTypeRenderer
}

// AddEntity adds an entity to the renderer to be capable of being rendered to the screen.
func (r *renderer) AddEntity(e entity.Entity) {
	r.add <- e
}

// RemoveEntity removes an entity from the renderer.  Once removed the Entity will no longer be rendered to the screen unless it is added back to the renderer.
func (r *renderer) RemoveEntity(e entity.Entity) {
	r.remove <- e
}

// addEntity adds an Entity to the system.  Each system will have a component requirement that must be met before the Entity can be added.
func (r *renderer) addEntity(e entity.Entity) {
	mesh, isMesh := e.Component(components.ComponentTypeMesh).(components.Mesh)
	transform, isTransform := e.Component(components.ComponentTypeTransform).(components.Transform)

	if isMesh && isTransform {
		rend := renderable{
			Mesh:      mesh,
			Transform: transform,
		}
		// Set up the shader
		md := mesh.Data()

		var shader am.Shader
		var err error
		r.mainFunc(func() {
			shader, err = r.assets.Shaders().LoadProgramFromFile(md.VertShaderFile, md.FragShaderFile, false)
		})
		if err != nil {
			fmt.Printf("Unable to load shaders %s,%s", md.VertShaderFile, md.FragShaderFile)
			return
		}

		r.mainFunc(func() {
			md.ProgramID = shader.ProgramID()
			md.VAO = shader.CreateVAO(mesh)
		})

		// Load and set the texture if it exists.
		var texture uint32
		r.mainFunc(func() {
			texture, err = r.assets.Textures().LoadTexture(md.TextureFile, md.TextureFile)
		})
		if err != nil {
			fmt.Printf("Unable to load texture:%s", md.TextureFile)
			return
		}

		md.TextureID = texture
		// Add the shader and texture ids to the mesh component.
		mesh.Set(md)

		r.entities[e.ID()] = rend
	}

}

// removeEntity removes an Entity from the system.
func (r *renderer) removeEntity(e entity.Entity) {
	delete(r.entities, e.ID())
}

// Start will begin attempting to Render Entities that have been added.  This routine is expected to wait for various channel inputs and act accordingly. The renderer needs to run on the main thread.
func (r *renderer) Start() {
	defer r.runningLock.Unlock()
	r.runningLock.Lock()

	if r.isRunning {
		return
	}
	r.isRunning = true

	go func() {
		for {
			select {
			case ent := <-r.add:
				fmt.Printf("Adding %s to the renderer.\n", ent.ID())
				r.addEntity(ent)
			case ent := <-r.remove:
				fmt.Printf("Removing %s to the renderer.\n", ent.ID())
				r.removeEntity(ent)
			case <-r.quit:
				r.isRunning = false
				return
			case <-time.After(15 * time.Millisecond):
				r.Process()
			}
		}
	}()
}

// Stop Will stop the renderer from rendering any of its Entities.
func (r *renderer) Stop() {
	defer r.runningLock.Unlock()
	r.runningLock.Lock()
	if r.isRunning {
		r.quit <- true
	}
}

// IsRunning is useful to check if the renderer is running its process routine.
func (r *renderer) IsRunning() bool {
	defer r.runningLock.Unlock()
	r.runningLock.Lock()
	return r.isRunning
}

func (r *renderer) Process() {

	r.mainFunc(func() {
		r.runningLock.Lock()
		defer r.runningLock.Unlock()

		gl.Clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT)

		for _, ent := range r.entities {
			md := ent.Mesh.Data()
			shader, status := r.assets.Shaders().GetShaderProgram(md.ProgramID)
			if !status {
				// if the shader is not loaded there is no point in trying to render it to the screen.
				continue
			}
			gl.UseProgram(md.ProgramID)

			gl.BindVertexArray(md.VAO)
			// set the camera uniform.  TODO this only needs to be done once per shader
			projection := r.camera.Projection()
			gl.UniformMatrix4fv(shader.GetUniformLoc(am.ProjectionUniform), 1, false, &projection[0])
			// set the view uniform. TODO this only needs to be done once per shader
			view := r.camera.View()
			gl.UniformMatrix4fv(shader.GetUniformLoc(am.CameraUniform), 1, false, &view[0])

			td := ent.Transform.Data()
			gl.UniformMatrix4fv(shader.GetUniformLoc(am.ModelUniform), 1, false, &td[0])
			gl.ActiveTexture(gl.TEXTURE0)
			gl.Uniform1i(shader.GetUniformLoc(am.TextureUniform), 0)

			texture, isLoaded := r.assets.Textures().GetTexture(md.TextureFile)
			if isLoaded {
				gl.BindTexture(gl.TEXTURE_2D, texture)
			}

			if md.Indexed {
				gl.DrawElements(gl.TRIANGLE_FAN, int32(len(md.Indices)), gl.UNSIGNED_INT, gl.PtrOffset(0))
			} else {
				gl.DrawArrays(gl.TRIANGLES, 0, int32(len(md.Verts))/md.VertSize)
			}

			gl.BindVertexArray(0)
		}

		// Maintenance
		r.window.SwapBuffers()
		glfw.PollEvents()

	})
}

// LoadCamera sets the camera to be used by the display.
func (r *renderer) LoadCamera(camera components.Camera) {
	r.camera = camera
}

type renderable struct {
	Entity    entity.Entity
	Mesh      components.Mesh
	Transform components.Transform
}
<commit_msg>moved location of add and remove entity<commit_after>package systems

import (
	"fmt"
	"sync"

	"time"

	am "github.com/Ariemeth/quantum-pulse/assets"
	"github.com/Ariemeth/quantum-pulse/components"
	"github.com/Ariemeth/quantum-pulse/entity"
	"github.com/go-gl/gl/v4.1-core/gl"
	"github.com/go-gl/glfw/v3.2/glfw"
)

const (
	// SystemTypeRenderer is the name of the renderer system.
	SystemTypeRenderer = "renderer"
)

// Renderer provides the interface needed to process the rendering of Entities.  Each time Process is called all Entities will be rendered.
type Renderer interface {
	System
	// Process renders all renderable entities.
	Process()
	// LoadCamera sets the camera to be used by the display.
	LoadCamera(camera components.Camera)
	Stop()
}

type renderer struct {
	entities     map[string]renderable
	assets       *am.AssetManager
	camera       components.Camera
	mainFunc     func(f func())
	window       *glfw.Window
	remove       chan entity.Entity
	add          chan entity.Entity
	quit         chan interface{}
	runningLock  sync.Mutex
	isRunning    bool
	requirements []string
}

// NewRenderer creates a new renderer system.  The renderer system handles rendering all renderable Entities to the screen.
func NewRenderer(assetManager *am.AssetManager, mainFunc func(f func()), window *glfw.Window) Renderer {
	r := renderer{
		entities:     make(map[string]renderable),
		assets:       assetManager,
		camera:       components.NewCamera(),
		mainFunc:     mainFunc,
		window:       window,
		remove:       make(chan entity.Entity, 0),
		add:          make(chan entity.Entity, 0),
		quit:         make(chan interface{}),
		requirements: []string{""}, //TODO eventually add the actual requirements
	}

	r.Start()

	return &r
}

// SystemType retrieves the type of system such as renderer, mover, etc.
func (r *renderer) SystemType() string {
	return SystemTypeRenderer
}

// AddEntity adds an entity to the renderer to be capable of being rendered to the screen.
func (r *renderer) AddEntity(e entity.Entity) {
	r.add <- e
}

// RemoveEntity removes an entity from the renderer.  Once removed the Entity will no longer be rendered to the screen unless it is added back to the renderer.
func (r *renderer) RemoveEntity(e entity.Entity) {
	r.remove <- e
}

// Start will begin attempting to Render Entities that have been added.  This routine is expected to wait for various channel inputs and act accordingly. The renderer needs to run on the main thread.
func (r *renderer) Start() {
	defer r.runningLock.Unlock()
	r.runningLock.Lock()

	if r.isRunning {
		return
	}
	r.isRunning = true

	go func() {
		for {
			select {
			case ent := <-r.add:
				fmt.Printf("Adding %s to the renderer.\n", ent.ID())
				r.addEntity(ent)
			case ent := <-r.remove:
				fmt.Printf("Removing %s to the renderer.\n", ent.ID())
				r.removeEntity(ent)
			case <-r.quit:
				r.isRunning = false
				return
			case <-time.After(15 * time.Millisecond):
				r.Process()
			}
		}
	}()
}

// Stop Will stop the renderer from rendering any of its Entities.
func (r *renderer) Stop() {
	defer r.runningLock.Unlock()
	r.runningLock.Lock()
	if r.isRunning {
		r.quit <- true
	}
}

// IsRunning is useful to check if the renderer is running its process routine.
func (r *renderer) IsRunning() bool {
	defer r.runningLock.Unlock()
	r.runningLock.Lock()
	return r.isRunning
}

func (r *renderer) Process() {

	r.mainFunc(func() {
		r.runningLock.Lock()
		defer r.runningLock.Unlock()

		gl.Clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT)

		for _, ent := range r.entities {
			md := ent.Mesh.Data()
			shader, status := r.assets.Shaders().GetShaderProgram(md.ProgramID)
			if !status {
				// if the shader is not loaded there is no point in trying to render it to the screen.
				continue
			}
			gl.UseProgram(md.ProgramID)

			gl.BindVertexArray(md.VAO)
			// set the camera uniform.  TODO this only needs to be done once per shader
			projection := r.camera.Projection()
			gl.UniformMatrix4fv(shader.GetUniformLoc(am.ProjectionUniform), 1, false, &projection[0])
			// set the view uniform. TODO this only needs to be done once per shader
			view := r.camera.View()
			gl.UniformMatrix4fv(shader.GetUniformLoc(am.CameraUniform), 1, false, &view[0])

			td := ent.Transform.Data()
			gl.UniformMatrix4fv(shader.GetUniformLoc(am.ModelUniform), 1, false, &td[0])
			gl.ActiveTexture(gl.TEXTURE0)
			gl.Uniform1i(shader.GetUniformLoc(am.TextureUniform), 0)

			texture, isLoaded := r.assets.Textures().GetTexture(md.TextureFile)
			if isLoaded {
				gl.BindTexture(gl.TEXTURE_2D, texture)
			}

			if md.Indexed {
				gl.DrawElements(gl.TRIANGLE_FAN, int32(len(md.Indices)), gl.UNSIGNED_INT, gl.PtrOffset(0))
			} else {
				gl.DrawArrays(gl.TRIANGLES, 0, int32(len(md.Verts))/md.VertSize)
			}

			gl.BindVertexArray(0)
		}

		// Maintenance
		r.window.SwapBuffers()
		glfw.PollEvents()

	})
}

// LoadCamera sets the camera to be used by the display.
func (r *renderer) LoadCamera(camera components.Camera) {
	r.camera = camera
}

// addEntity adds an Entity to the system.  Each system will have a component requirement that must be met before the Entity can be added.
func (r *renderer) addEntity(e entity.Entity) {
	mesh, isMesh := e.Component(components.ComponentTypeMesh).(components.Mesh)
	transform, isTransform := e.Component(components.ComponentTypeTransform).(components.Transform)

	if isMesh && isTransform {
		rend := renderable{
			Mesh:      mesh,
			Transform: transform,
		}
		// Set up the shader
		md := mesh.Data()

		var shader am.Shader
		var err error
		r.mainFunc(func() {
			shader, err = r.assets.Shaders().LoadProgramFromFile(md.VertShaderFile, md.FragShaderFile, false)
		})
		if err != nil {
			fmt.Printf("Unable to load shaders %s,%s", md.VertShaderFile, md.FragShaderFile)
			return
		}

		r.mainFunc(func() {
			md.ProgramID = shader.ProgramID()
			md.VAO = shader.CreateVAO(mesh)
		})

		// Load and set the texture if it exists.
		var texture uint32
		r.mainFunc(func() {
			texture, err = r.assets.Textures().LoadTexture(md.TextureFile, md.TextureFile)
		})
		if err != nil {
			fmt.Printf("Unable to load texture:%s", md.TextureFile)
			return
		}

		md.TextureID = texture
		// Add the shader and texture ids to the mesh component.
		mesh.Set(md)

		r.entities[e.ID()] = rend
	}
}

// removeEntity removes an Entity from the system.
func (r *renderer) removeEntity(e entity.Entity) {
	delete(r.entities, e.ID())
}

type renderable struct {
	Entity    entity.Entity
	Mesh      components.Mesh
	Transform components.Transform
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package provider

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

type domainFilterTest struct {
	domainFilter []string
	exclusions   []string
	domains      []string
	expected     bool
}

var domainFilterTests = []domainFilterTest{
	{
		[]string{"google.com.", "exaring.de", "inovex.de"},
		[]string{},
		[]string{"google.com", "exaring.de", "inovex.de"},
		true,
	},
	{
		[]string{"google.com.", "exaring.de", "inovex.de"},
		[]string{},
		[]string{"google.com", "exaring.de", "inovex.de"},
		true,
	},
	{
		[]string{"google.com.", "exaring.de.", "inovex.de"},
		[]string{},
		[]string{"google.com", "exaring.de", "inovex.de"},
		true,
	},
	{
		[]string{"foo.org.      "},
		[]string{},
		[]string{"foo.org"},
		true,
	},
	{
		[]string{"   foo.org"},
		[]string{},
		[]string{"foo.org"},
		true,
	},
	{
		[]string{"foo.org."},
		[]string{},
		[]string{"foo.org"},
		true,
	},
	{
		[]string{"foo.org."},
		[]string{},
		[]string{"baz.org"},
		false,
	},
	{
		[]string{"baz.foo.org."},
		[]string{},
		[]string{"foo.org"},
		false,
	},
	{
		[]string{"", "foo.org."},
		[]string{},
		[]string{"foo.org"},
		true,
	},
	{
		[]string{"", "foo.org."},
		[]string{},
		[]string{},
		true,
	},
	{
		[]string{""},
		[]string{},
		[]string{"foo.org"},
		true,
	},
	{
		[]string{""},
		[]string{},
		[]string{},
		true,
	},
	{
		[]string{" "},
		[]string{},
		[]string{},
		true,
	},
	{
		[]string{"bar.sub.example.org"},
		[]string{},
		[]string{"foo.bar.sub.example.org"},
		true,
	},
	{
		[]string{"example.org"},
		[]string{},
		[]string{"anexample.org", "test.anexample.org"},
		false,
	},
	{
		[]string{".example.org"},
		[]string{},
		[]string{"anexample.org", "test.anexample.org"},
		false,
	},
	{
		[]string{".example.org"},
		[]string{},
		[]string{"example.org"},
		false,
	},
	{
		[]string{".example.org"},
		[]string{},
		[]string{"test.example.org"},
		true,
	},
	{
		[]string{"anexample.org"},
		[]string{},
		[]string{"example.org", "test.example.org"},
		false,
	},
	{
		[]string{".org"},
		[]string{},
		[]string{"example.org", "test.example.org", "foo.test.example.org"},
		true,
	},
	{
		[]string{"example.org"},
		[]string{"api.example.org"},
		[]string{"example.org", "test.example.org", "foo.test.example.org"},
		true,
	},
	{
		[]string{"example.org"},
		[]string{"api.example.org"},
		[]string{"foo.api.example.org", "api.example.org"},
		false,
	},
	{
		[]string{"   example.org. "},
		[]string{"   .api.example.org    "},
		[]string{"foo.api.example.org", "bar.baz.api.example.org."},
		false,
	},
	{
		[]string{"example.org."},
		[]string{"api.example.org"},
		[]string{"dev-api.example.org", "qa-api.example.org"},
		true,
	},
	{
		[]string{"example.org."},
		[]string{"api.example.org"},
		[]string{"dev.api.example.org", "qa.api.example.org"},
		false,
	},
	{
		[]string{"example.org", "api.example.org"},
		[]string{"internal.api.example.org"},
		[]string{"foo.api.example.org"},
		true,
	},
	{
		[]string{"example.org", "api.example.org"},
		[]string{"internal.api.example.org"},
		[]string{"foo.internal.api.example.org"},
		false,
	},
	{
		[]string{"eXaMPle.ORG", "API.example.ORG"},
		[]string{"Foo-Bar.Example.Org"},
		[]string{"FoOoo.Api.Example.Org"},
		true,
	},
	{
		[]string{"eXaMPle.ORG", "API.example.ORG"},
		[]string{"api.example.org"},
		[]string{"foobar.Example.Org"},
		true,
	},
	{
		[]string{"eXaMPle.ORG", "API.example.ORG"},
		[]string{"api.example.org"},
		[]string{"foobar.API.Example.Org"},
		false,
	},
}

func TestDomainFilterMatch(t *testing.T) {
	for i, tt := range domainFilterTests {
		if len(tt.exclusions) > 0 {
			t.Skip("NewDomainFilter() doesn't support exclusions")
		}
		domainFilter := NewDomainFilter(tt.domainFilter)
		for _, domain := range tt.domains {
			assert.Equal(t, tt.expected, domainFilter.Match(domain), "should not fail: %v in test-case #%v", domain, i)
			assert.Equal(t, tt.expected, domainFilter.Match(domain+"."), "should not fail: %v in test-case #%v", domain+".", i)
		}
	}
}

func TestDomainFilterWithExclusions(t *testing.T) {
	for i, tt := range domainFilterTests {
		domainFilter := NewDomainFilterWithExclusions(tt.domainFilter, tt.exclusions)
		for _, domain := range tt.domains {
			assert.Equal(t, tt.expected, domainFilter.Match(domain), "should not fail: %v in test-case #%v", domain, i)
			assert.Equal(t, tt.expected, domainFilter.Match(domain+"."), "should not fail: %v in test-case #%v", domain+".", i)
		}
	}
}

func TestDomainFilterMatchWithEmptyFilter(t *testing.T) {
	for _, tt := range domainFilterTests {
		domainFilter := DomainFilter{}
		for i, domain := range tt.domains {
			assert.True(t, domainFilter.Match(domain), "should not fail: %v in test-case #%v", domain, i)
			assert.True(t, domainFilter.Match(domain+"."), "should not fail: %v in test-case #%v", domain+".", i)
		}
	}
}

func TestPrepareFiltersStripsWhitespaceAndDotSuffix(t *testing.T) {
	for _, tt := range []struct {
		input  []string
		output []string
	}{
		{
			[]string{" ", "   ", ""},
			[]string{"", "", ""},
		},
		{
			[]string{"  foo   ", "  bar. ", "baz."},
			[]string{"foo", "bar", "baz"},
		},
		{
			[]string{"foo.bar", "  foo.bar.  ", " foo.bar.baz ", " foo.bar.baz.  "},
			[]string{"foo.bar", "foo.bar", "foo.bar.baz", "foo.bar.baz"},
		},
	} {
		t.Run("test string", func(t *testing.T) {
			assert.Equal(t, tt.output, prepareFilters(tt.input))
		})
	}
}

func TestMatchFilterReturnsProperEmptyVal(t *testing.T) {
	emptyFilters := []string{}
	assert.Equal(t, true, matchFilter(emptyFilters, "somedomain.com", true))
	assert.Equal(t, false, matchFilter(emptyFilters, "somedomain.com", false))
}

func TestDomainFilterIsConfigured(t *testing.T) {
	for _, tt := range []struct {
		filters  []string
		exclude  []string
		expected bool
	}{
		{
			[]string{""},
			[]string{""},
			false,
		},
		{
			[]string{"    "},
			[]string{"    "},
			false,
		},
		{
			[]string{"", ""},
			[]string{""},
			true,
		},
		{
			[]string{" . "},
			[]string{" . "},
			false,
		},
		{
			[]string{" notempty.com "},
			[]string{"  "},
			true,
		},
		{
			[]string{" notempty.com "},
			[]string{"  thisdoesntmatter.com "},
			true,
		},
	} {
		t.Run("test IsConfigured", func(t *testing.T) {
			df := NewDomainFilterWithExclusions(tt.filters, tt.exclude)
			assert.Equal(t, tt.expected, df.IsConfigured())
		})
	}
}
<commit_msg>Fix test case for domain exclusions<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package provider

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

type domainFilterTest struct {
	domainFilter []string
	exclusions   []string
	domains      []string
	expected     bool
}

var domainFilterTests = []domainFilterTest{
	{
		[]string{"google.com.", "exaring.de", "inovex.de"},
		[]string{""},
		[]string{"google.com", "exaring.de", "inovex.de"},
		true,
	},
	{
		[]string{"google.com.", "exaring.de", "inovex.de"},
		[]string{""},
		[]string{"google.com", "exaring.de", "inovex.de"},
		true,
	},
	{
		[]string{"google.com.", "exaring.de.", "inovex.de"},
		[]string{""},
		[]string{"google.com", "exaring.de", "inovex.de"},
		true,
	},
	{
		[]string{"foo.org.      "},
		[]string{""},
		[]string{"foo.org"},
		true,
	},
	{
		[]string{"   foo.org"},
		[]string{""},
		[]string{"foo.org"},
		true,
	},
	{
		[]string{"foo.org."},
		[]string{""},
		[]string{"foo.org"},
		true,
	},
	{
		[]string{"foo.org."},
		[]string{""},
		[]string{"baz.org"},
		false,
	},
	{
		[]string{"baz.foo.org."},
		[]string{""},
		[]string{"foo.org"},
		false,
	},
	{
		[]string{"", "foo.org."},
		[]string{""},
		[]string{"foo.org"},
		true,
	},
	{
		[]string{"", "foo.org."},
		[]string{""},
		[]string{},
		true,
	},
	{
		[]string{""},
		[]string{""},
		[]string{"foo.org"},
		true,
	},
	{
		[]string{""},
		[]string{""},
		[]string{},
		true,
	},
	{
		[]string{" "},
		[]string{""},
		[]string{},
		true,
	},
	{
		[]string{"bar.sub.example.org"},
		[]string{""},
		[]string{"foo.bar.sub.example.org"},
		true,
	},
	{
		[]string{"example.org"},
		[]string{""},
		[]string{"anexample.org", "test.anexample.org"},
		false,
	},
	{
		[]string{".example.org"},
		[]string{""},
		[]string{"anexample.org", "test.anexample.org"},
		false,
	},
	{
		[]string{".example.org"},
		[]string{""},
		[]string{"example.org"},
		false,
	},
	{
		[]string{".example.org"},
		[]string{""},
		[]string{"test.example.org"},
		true,
	},
	{
		[]string{"anexample.org"},
		[]string{""},
		[]string{"example.org", "test.example.org"},
		false,
	},
	{
		[]string{".org"},
		[]string{""},
		[]string{"example.org", "test.example.org", "foo.test.example.org"},
		true,
	},
	{
		[]string{"example.org"},
		[]string{"api.example.org"},
		[]string{"example.org", "test.example.org", "foo.test.example.org"},
		true,
	},
	{
		[]string{"example.org"},
		[]string{"api.example.org"},
		[]string{"foo.api.example.org", "api.example.org"},
		false,
	},
	{
		[]string{"   example.org. "},
		[]string{"   .api.example.org    "},
		[]string{"foo.api.example.org", "bar.baz.api.example.org."},
		false,
	},
	{
		[]string{"example.org."},
		[]string{"api.example.org"},
		[]string{"dev-api.example.org", "qa-api.example.org"},
		true,
	},
	{
		[]string{"example.org."},
		[]string{"api.example.org"},
		[]string{"dev.api.example.org", "qa.api.example.org"},
		false,
	},
	{
		[]string{"example.org", "api.example.org"},
		[]string{"internal.api.example.org"},
		[]string{"foo.api.example.org"},
		true,
	},
	{
		[]string{"example.org", "api.example.org"},
		[]string{"internal.api.example.org"},
		[]string{"foo.internal.api.example.org"},
		false,
	},
	{
		[]string{"eXaMPle.ORG", "API.example.ORG"},
		[]string{"Foo-Bar.Example.Org"},
		[]string{"FoOoo.Api.Example.Org"},
		true,
	},
	{
		[]string{"eXaMPle.ORG", "API.example.ORG"},
		[]string{"api.example.org"},
		[]string{"foobar.Example.Org"},
		true,
	},
	{
		[]string{"eXaMPle.ORG", "API.example.ORG"},
		[]string{"api.example.org"},
		[]string{"foobar.API.Example.Org"},
		false,
	},
}

func TestDomainFilterMatch(t *testing.T) {
	for i, tt := range domainFilterTests {
		if len(tt.exclusions) > 0 {
			t.Skip("NewDomainFilter() doesn't support exclusions")
		}
		domainFilter := NewDomainFilter(tt.domainFilter)
		for _, domain := range tt.domains {
			assert.Equal(t, tt.expected, domainFilter.Match(domain), "should not fail: %v in test-case #%v", domain, i)
			assert.Equal(t, tt.expected, domainFilter.Match(domain+"."), "should not fail: %v in test-case #%v", domain+".", i)
		}
	}
}

func TestDomainFilterWithExclusions(t *testing.T) {
	for i, tt := range domainFilterTests {
		domainFilter := NewDomainFilterWithExclusions(tt.domainFilter, tt.exclusions)
		for _, domain := range tt.domains {
			assert.Equal(t, tt.expected, domainFilter.Match(domain), "should not fail: %v in test-case #%v", domain, i)
			assert.Equal(t, tt.expected, domainFilter.Match(domain+"."), "should not fail: %v in test-case #%v", domain+".", i)
		}
	}
}

func TestDomainFilterMatchWithEmptyFilter(t *testing.T) {
	for _, tt := range domainFilterTests {
		domainFilter := DomainFilter{}
		for i, domain := range tt.domains {
			assert.True(t, domainFilter.Match(domain), "should not fail: %v in test-case #%v", domain, i)
			assert.True(t, domainFilter.Match(domain+"."), "should not fail: %v in test-case #%v", domain+".", i)
		}
	}
}

func TestPrepareFiltersStripsWhitespaceAndDotSuffix(t *testing.T) {
	for _, tt := range []struct {
		input  []string
		output []string
	}{
		{
			[]string{""},
			[]string{""},
		},
		{
			[]string{" ", "   ", ""},
			[]string{"", "", ""},
		},
		{
			[]string{"  foo   ", "  bar. ", "baz."},
			[]string{"foo", "bar", "baz"},
		},
		{
			[]string{"foo.bar", "  foo.bar.  ", " foo.bar.baz ", " foo.bar.baz.  "},
			[]string{"foo.bar", "foo.bar", "foo.bar.baz", "foo.bar.baz"},
		},
	} {
		t.Run("test string", func(t *testing.T) {
			assert.Equal(t, tt.output, prepareFilters(tt.input))
		})
	}
}

func TestMatchFilterReturnsProperEmptyVal(t *testing.T) {
	emptyFilters := []string{}
	assert.Equal(t, true, matchFilter(emptyFilters, "somedomain.com", true))
	assert.Equal(t, false, matchFilter(emptyFilters, "somedomain.com", false))
}

func TestDomainFilterIsConfigured(t *testing.T) {
	for _, tt := range []struct {
		filters  []string
		exclude  []string
		expected bool
	}{
		{
			[]string{""},
			[]string{""},
			false,
		},
		{
			[]string{"    "},
			[]string{"    "},
			false,
		},
		{
			[]string{"", ""},
			[]string{""},
			true,
		},
		{
			[]string{" . "},
			[]string{" . "},
			false,
		},
		{
			[]string{" notempty.com "},
			[]string{"  "},
			true,
		},
		{
			[]string{" notempty.com "},
			[]string{"  thisdoesntmatter.com "},
			true,
		},
	} {
		t.Run("test IsConfigured", func(t *testing.T) {
			df := NewDomainFilterWithExclusions(tt.filters, tt.exclude)
			assert.Equal(t, tt.expected, df.IsConfigured())
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"time"

	"encoding/json"
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/mopsalarm/go-pr0gramm"
	"github.com/rcrowley/go-metrics"
	"io/ioutil"
	"net/http"
	"strings"
	"unicode/utf8"
)

type tagWithItemId struct {
	pr0gramm.Tag
	ItemId pr0gramm.Id
}

func UpdateAll(db *sql.DB, request pr0gramm.ItemsRequest) (pr0gramm.Id, error) {
	var lastProcessedId pr0gramm.Id
	err := pr0gramm.StreamPaged(request, func(items []pr0gramm.Item) (bool, error) {
		if len(items) > 0 {
			lastProcessedId = items[len(items)-1].Id
		}

		writeItems(db, items)
		time.Sleep(5 * time.Second)
		return true, nil
	})

	return lastProcessedId, err
}

func Update(db *sql.DB, maxItemAge time.Duration) {
	logrus.WithField("max-age", maxItemAge).Info("Updating items now")

	var items []pr0gramm.Item
	err := pr0gramm.Stream(pr0gramm.NewItemsRequest(), pr0gramm.ConsumeIf(
		func(item pr0gramm.Item) bool {
			return time.Since(item.Created.Time).Seconds() < maxItemAge.Seconds()
		},
		func(item pr0gramm.Item) error {
			items = append(items, item)
			return nil
		}))

	if err != nil {
		logrus.WithError(err).Warn("Could not fetch all items")
	}

	if len(items) == 0 {
		return
	}

	writeItems(db, items)
}

func writeItems(db *sql.DB, items []pr0gramm.Item) {
	tx, err := db.Begin()
	if err != nil {
		logrus.WithError(err).Warn("Could not open transction")
		return
	}

	defer tx.Commit()

	itemStmt, err := tx.Prepare(`INSERT INTO items
		(id, promoted, up, down, created, image, thumb, fullsize, source, flags, username, mark, width, height, audio)
		VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15)
		ON CONFLICT (id) DO UPDATE SET up=EXCLUDED.up, down=EXCLUDED.down, promoted=EXCLUDED.promoted, mark=EXCLUDED.mark`)

	if err != nil {
		logrus.WithError(err).Warn("Could not prepare insert statement")
		return
	}

	defer itemStmt.Close()

	start := time.Now()
	logrus.WithField("count", len(items)).Info("Writing items to database")
	for _, item := range items {
		_, err := itemStmt.Exec(
			uint64(item.Id), uint64(item.Promoted),
			item.Up, item.Down,
			item.Created.Time.Unix(),
			item.Image, item.Thumbnail, item.Fullsize, item.Source,
			item.Flags, item.User, item.Mark, item.Width, item.Height, item.Audio)

		if err != nil {
			logrus.WithError(err).Warn("Could not insert item into database, skipping.")
		} else {
			metrics.GetOrRegisterMeter("pr0gramm.meta.items.inserted", nil).Mark(1)
		}
	}

	logrus.
		WithField("duration", time.Since(start)).
		WithField("count", len(items)).
		Info("Finished writing items")
}

func UpdateTags(db *sql.DB) (tagCount int) {
	start := time.Now()

	tx, err := db.Begin()
	if err != nil {
		logrus.WithError(err).Error("Could not create transaction to update tags")
		return
	}

	defer tx.Commit()

	var largestTagId uint64

	row := tx.QueryRow("SELECT COALESCE(MAX(id), 0) FROM tags")
	if err := row.Scan(&largestTagId); err != nil {
		logrus.WithError(err).Error("Could not get the value of the largest known tag-id")
		return
	}

	url := fmt.Sprintf("http://pr0gramm.com/api/tags/latest?id=%d", largestTagId)
	logrus.WithField("url", url).Info("Fetching tags from remote api now")

	response, err := http.Get(url)
	if err != nil {
		logrus.WithError(err).Error("Request to get the latest tags failed.")
		return
	}

	defer func() {
		ioutil.ReadAll(response.Body)
		response.Body.Close()
	}()

	var decoded struct {
		Tags []struct {
			Id         uint64
			Up         int32
			Down       int32
			Confidence float32
			ItemId     uint64
			Tag        string
		}
	}

	if err := json.NewDecoder(response.Body).Decode(&decoded); err != nil {
		logrus.WithError(err).Error("Could not decode response")
		return
	}

	if len(decoded.Tags) > 0 {
		tagStmt, err := tx.Prepare(`INSERT INTO tags
		(id, item_id, up, down, confidence, tag)
		VALUES ($1, $2, $3, $4, $5, $6)
		ON CONFLICT (id) DO UPDATE
		SET up=EXCLUDED.up, down=EXCLUDED.down, confidence=EXCLUDED.confidence`)

		if err != nil {
			logrus.WithError(err).Error("Could not prepare statement")
			return
		}

		defer tagStmt.Close()

		logrus.WithField("count", len(decoded.Tags)).Info("Will dump tags to database.")
		for _, tag := range decoded.Tags {
			if utf8.ValidString(tag.Tag) && !strings.ContainsRune(tag.Tag, 0) {
				_, err = tagStmt.Exec(tag.Id, tag.ItemId, tag.Up, tag.Down, tag.Confidence, tag.Tag)

				if err != nil {
					logrus.WithError(err).Warn("Could not insert tag into database, skipping")
				} else {
					metrics.GetOrRegisterMeter("pr0gramm.meta.tags.inserted", nil).Mark(1)
				}
			}
		}
	}

	logrus.
		WithField("duration", time.Since(start)).
		WithField("count", len(decoded.Tags)).
		Info("Finished writing tags")

	return len(decoded.Tags)
}
<commit_msg>Remove deletet items from database.<commit_after>package main

import (
	"database/sql"
	"time"

	"encoding/json"
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/mopsalarm/go-pr0gramm"
	"github.com/rcrowley/go-metrics"
	"io/ioutil"
	"net/http"
	"strings"
	"unicode/utf8"
)

type tagWithItemId struct {
	pr0gramm.Tag
	ItemId pr0gramm.Id
}

func UpdateAll(db *sql.DB, request pr0gramm.ItemsRequest) (pr0gramm.Id, error) {
	var lastProcessedId pr0gramm.Id
	err := pr0gramm.StreamPaged(request, func(items []pr0gramm.Item) (bool, error) {

		for _, item := range items {
			if lastProcessedId > 0 {
				for id := lastProcessedId - 1; id > item.Id; id-- {
					deleteItem(db, id)
				}
			}

			lastProcessedId = item.Id
		}

		writeItems(db, items)
		time.Sleep(5 * time.Second)
		return true, nil
	})

	return lastProcessedId, err
}

func Update(db *sql.DB, maxItemAge time.Duration) {
	logrus.WithField("max-age", maxItemAge).Info("Updating items now")

	var lastItemId pr0gramm.Id

	var deleteQueue []pr0gramm.Id
	var updateQueue []pr0gramm.Item

	err := pr0gramm.Stream(pr0gramm.NewItemsRequest(), pr0gramm.ConsumeIf(
		func(item pr0gramm.Item) bool {
			if lastItemId > 0 {
				for id := lastItemId - 1; id > item.Id; id-- {
					deleteQueue = append(deleteQueue, item.Id)
				}
			}

			lastItemId = item.Id

			return time.Since(item.Created.Time).Seconds() < maxItemAge.Seconds()
		},
		func(item pr0gramm.Item) error {
			updateQueue = append(updateQueue, item)
			return nil
		}))

	if err != nil {
		logrus.WithError(err).Warn("Could not fetch all items")
	}

	if len(deleteQueue) > 0 {
		logrus.Infof("Ensure that %d out of %d items are absent", len(deleteQueue), len(updateQueue))
		for _, itemId := range deleteQueue {
			deleteItem(db, itemId)
		}
	}

	if len(updateQueue) == 0 {
		return
	}

	writeItems(db, updateQueue)
}

func deleteItem(db *sql.DB, itemId pr0gramm.Id) {
	tx, err := db.Begin()
	if err != nil {
		logrus.WithError(err).Warn("Could not open transction")
		return
	}

	defer tx.Commit()

	if res, err := tx.Exec("DELETE FROM items WHERE id=$1", int(itemId)); err != nil {
		logrus.Warnf("Could not delete item %d from database", itemId)
		return
	} else {
		if affected, _ := res.RowsAffected(); affected > 0 {
			logrus.Infof("Item %d was deleted from database", int(itemId))
		}
	}
}

func writeItems(db *sql.DB, items []pr0gramm.Item) {
	tx, err := db.Begin()
	if err != nil {
		logrus.WithError(err).Warn("Could not open transction")
		return
	}

	defer tx.Commit()

	itemStmt, err := tx.Prepare(`INSERT INTO items
		(id, promoted, up, down, created, image, thumb, fullsize, source, flags, username, mark, width, height, audio)
		VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15)
		ON CONFLICT (id) DO UPDATE SET up=EXCLUDED.up, down=EXCLUDED.down, promoted=EXCLUDED.promoted, mark=EXCLUDED.mark
		WHERE items.up!=EXCLUDED.up OR items.down!=EXCLUDED.down OR items.promoted!=EXCLUDED.promoted OR items.mark!= EXCLUDED.mark`)

	if err != nil {
		logrus.WithError(err).Warn("Could not prepare insert statement")
		return
	}

	defer itemStmt.Close()

	start := time.Now()
	logrus.WithField("count", len(items)).Info("Writing items to database")
	for _, item := range items {
		_, err := itemStmt.Exec(
			uint64(item.Id), uint64(item.Promoted),
			item.Up, item.Down,
			item.Created.Time.Unix(),
			item.Image, item.Thumbnail, item.Fullsize, item.Source,
			item.Flags, item.User, item.Mark, item.Width, item.Height, item.Audio)

		if err != nil {
			logrus.WithError(err).Warn("Could not insert item into database, skipping.")
		} else {
			metrics.GetOrRegisterMeter("pr0gramm.meta.items.inserted", nil).Mark(1)
		}
	}

	logrus.
		WithField("duration", time.Since(start)).
		WithField("count", len(items)).
		Info("Finished writing items")
}

func UpdateTags(db *sql.DB) (tagCount int) {
	start := time.Now()

	tx, err := db.Begin()
	if err != nil {
		logrus.WithError(err).Error("Could not create transaction to update tags")
		return
	}

	defer tx.Commit()

	var largestTagId uint64

	row := tx.QueryRow("SELECT COALESCE(MAX(id), 0) FROM tags")
	if err := row.Scan(&largestTagId); err != nil {
		logrus.WithError(err).Error("Could not get the value of the largest known tag-id")
		return
	}

	url := fmt.Sprintf("http://pr0gramm.com/api/tags/latest?id=%d", largestTagId)
	logrus.WithField("url", url).Info("Fetching tags from remote api now")

	response, err := http.Get(url)
	if err != nil {
		logrus.WithError(err).Error("Request to get the latest tags failed.")
		return
	}

	defer func() {
		ioutil.ReadAll(response.Body)
		response.Body.Close()
	}()

	var decoded struct {
		Tags []struct {
			Id         uint64
			Up         int32
			Down       int32
			Confidence float32
			ItemId     uint64
			Tag        string
		}
	}

	if err := json.NewDecoder(response.Body).Decode(&decoded); err != nil {
		logrus.WithError(err).Error("Could not decode response")
		return
	}

	if len(decoded.Tags) > 0 {
		tagStmt, err := tx.Prepare(`INSERT INTO tags
		(id, item_id, up, down, confidence, tag)
		VALUES ($1, $2, $3, $4, $5, $6)
		ON CONFLICT (id) DO UPDATE
		SET up=EXCLUDED.up, down=EXCLUDED.down, confidence=EXCLUDED.confidence`)

		if err != nil {
			logrus.WithError(err).Error("Could not prepare statement")
			return
		}

		defer tagStmt.Close()

		logrus.WithField("count", len(decoded.Tags)).Info("Will dump tags to database.")
		for _, tag := range decoded.Tags {
			if utf8.ValidString(tag.Tag) && !strings.ContainsRune(tag.Tag, 0) {
				_, err = tagStmt.Exec(tag.Id, tag.ItemId, tag.Up, tag.Down, tag.Confidence, tag.Tag)

				if err != nil {
					logrus.WithError(err).Warn("Could not insert tag into database, skipping")
				} else {
					metrics.GetOrRegisterMeter("pr0gramm.meta.tags.inserted", nil).Mark(1)
				}
			}
		}
	}

	logrus.
		WithField("duration", time.Since(start)).
		WithField("count", len(decoded.Tags)).
		Info("Finished writing tags")

	return len(decoded.Tags)
}
<|endoftext|>


<commit_before>package kcp

import (
	"container/heap"
	"sync"
	"time"
)

var updater updateHeap

func init() {
	updater.init()
	go updater.updateTask()
}

// entry contains a session update info
type entry struct {
	ts time.Time
	s  *UDPSession
}

// a global heap managed kcp.flush() caller
type updateHeap struct {
	entries  []entry
	mu       sync.Mutex
	chWakeUp chan struct{}
}

func (h *updateHeap) Len() int           { return len(h.entries) }
func (h *updateHeap) Less(i, j int) bool { return h.entries[i].ts.Before(h.entries[j].ts) }
func (h *updateHeap) Swap(i, j int) {
	h.entries[i], h.entries[j] = h.entries[j], h.entries[i]
	h.entries[i].s.updaterIdx = i
	h.entries[j].s.updaterIdx = j
}

func (h *updateHeap) Push(x interface{}) {
	h.entries = append(h.entries, x.(entry))
	n := len(h.entries)
	h.entries[n-1].s.updaterIdx = n - 1
}

func (h *updateHeap) Pop() interface{} {
	n := len(h.entries)
	x := h.entries[n-1]
	h.entries[n-1].s.updaterIdx = -1
	h.entries[n-1] = entry{} // manual set nil for GC
	h.entries = h.entries[0 : n-1]
	return x
}

func (h *updateHeap) init() {
	h.chWakeUp = make(chan struct{}, 1)
}

func (h *updateHeap) addSession(s *UDPSession) {
	h.mu.Lock()
	heap.Push(h, entry{time.Now(), s})
	h.mu.Unlock()
	h.wakeup()
}

func (h *updateHeap) removeSession(s *UDPSession) {
	h.mu.Lock()
	if s.updaterIdx != -1 {
		heap.Remove(h, s.updaterIdx)
	}
	h.mu.Unlock()
}

func (h *updateHeap) wakeup() {
	select {
	case h.chWakeUp <- struct{}{}:
	default:
	}
}

func (h *updateHeap) updateTask() {
	var timer <-chan time.Time
	for {
		select {
		case <-timer:
		case <-h.chWakeUp:
		}

		h.mu.Lock()
		hlen := h.Len()
		for i := 0; i < hlen; i++ {
			entry := &h.entries[0]
			if time.Now().After(entry.ts) {
				interval := entry.s.update()
				entry.ts = time.Now().Add(interval)
				heap.Fix(h, 0)
			} else {
				break
			}
		}

		if hlen > 0 {
			timer = time.After(h.entries[0].ts.Sub(time.Now()))
		}
		h.mu.Unlock()
	}
}
<commit_msg>pin updater to thread for better localiaty while sending<commit_after>package kcp

import (
	"container/heap"
	"runtime"
	"sync"
	"time"
)

var updater updateHeap

func init() {
	updater.init()
	go updater.updateTask()
}

// entry contains a session update info
type entry struct {
	ts time.Time
	s  *UDPSession
}

// a global heap managed kcp.flush() caller
type updateHeap struct {
	entries  []entry
	mu       sync.Mutex
	chWakeUp chan struct{}
}

func (h *updateHeap) Len() int           { return len(h.entries) }
func (h *updateHeap) Less(i, j int) bool { return h.entries[i].ts.Before(h.entries[j].ts) }
func (h *updateHeap) Swap(i, j int) {
	h.entries[i], h.entries[j] = h.entries[j], h.entries[i]
	h.entries[i].s.updaterIdx = i
	h.entries[j].s.updaterIdx = j
}

func (h *updateHeap) Push(x interface{}) {
	h.entries = append(h.entries, x.(entry))
	n := len(h.entries)
	h.entries[n-1].s.updaterIdx = n - 1
}

func (h *updateHeap) Pop() interface{} {
	n := len(h.entries)
	x := h.entries[n-1]
	h.entries[n-1].s.updaterIdx = -1
	h.entries[n-1] = entry{} // manual set nil for GC
	h.entries = h.entries[0 : n-1]
	return x
}

func (h *updateHeap) init() {
	h.chWakeUp = make(chan struct{}, 1)
}

func (h *updateHeap) addSession(s *UDPSession) {
	h.mu.Lock()
	heap.Push(h, entry{time.Now(), s})
	h.mu.Unlock()
	h.wakeup()
}

func (h *updateHeap) removeSession(s *UDPSession) {
	h.mu.Lock()
	if s.updaterIdx != -1 {
		heap.Remove(h, s.updaterIdx)
	}
	h.mu.Unlock()
}

func (h *updateHeap) wakeup() {
	select {
	case h.chWakeUp <- struct{}{}:
	default:
	}
}

func (h *updateHeap) updateTask() {
	runtime.LockOSThread()
	var timer <-chan time.Time
	for {
		select {
		case <-timer:
		case <-h.chWakeUp:
		}

		h.mu.Lock()
		hlen := h.Len()
		for i := 0; i < hlen; i++ {
			entry := &h.entries[0]
			if time.Now().After(entry.ts) {
				interval := entry.s.update()
				entry.ts = time.Now().Add(interval)
				heap.Fix(h, 0)
			} else {
				break
			}
		}

		if hlen > 0 {
			timer = time.After(h.entries[0].ts.Sub(time.Now()))
		}
		h.mu.Unlock()
	}
}
<|endoftext|>