<commit_before>/**
 * This file is a bit funny. The goal here is to use setns() to manipulate
 * files inside the container, so we don't have to reason about the paths to
 * make sure they don't escape (we can simply rely on the kernel for
 * correctness). Unfortunately, you can't setns() to a mount namespace with a
 * multi-threaded program, which every golang binary is. However, by declaring
 * our init as an initializer, we can capture process control before it is
 * transferred to the golang runtime, so we can then setns() as we'd like
 * before golang has a chance to set up any threads. So, we implement two new
 * lxd fork* commands which are captured here, and take a file on the host fs
 * and copy it into the container ns.
 *
 * An alternative to this would be to move this code into a separate binary,
 * which of course has problems of its own when it comes to packaging (how do
 * we find the binary, what do we do if someone does file push and it is
 * missing, etc.). After some discussion, even though the embedded method is
 * somewhat convoluted, it was preferred.
 */
package main

import (
	// Used by cgo
	_ "github.com/lxc/lxd/lxd/include"
)

/*
#ifndef _GNU_SOURCE
#define _GNU_SOURCE 1
#endif
#include <errno.h>
#include <fcntl.h>
#include <grp.h>
#include <linux/limits.h>
#include <sched.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>

#include "include/memory_utils.h"
#include "include/process_utils.h"
#include "include/syscall_numbers.h"

// External functions
extern void checkfeature();
extern void forkexec();
extern void forkfile();
extern void forksyscall();
extern void forkmount();
extern void forknet();
extern void forkproxy();
extern void forkuevent();

// Command line parsing and tracking
char *cmdline_buf = NULL;
char *cmdline_cur = NULL;
ssize_t cmdline_size = -1;

int wait_for_pid(pid_t pid)
{
	int status, ret;

again:
	ret = waitpid(pid, &status, 0);
	if (ret == -1) {
		if (errno == EINTR)
			goto again;
		return -1;
	}
	if (ret != pid)
		goto again;
	if (!WIFEXITED(status) || WEXITSTATUS(status) != 0)
		return -1;
	return 0;
}

char* advance_arg(bool required) {
	while (*cmdline_cur != 0)
		cmdline_cur++;

	cmdline_cur++;
	if (cmdline_size <= cmdline_cur - cmdline_buf) {
		if (!required)
			return NULL;

		fprintf(stderr, "not enough arguments\n");
		_exit(1);
	}

	return cmdline_cur;
}

void error(char *msg)
{
	int old_errno = errno;

	if (old_errno == 0) {
		fprintf(stderr, "%s\n", msg);
		fprintf(stderr, "errno: 0\n");
		return;
	}

	perror(msg);
	fprintf(stderr, "errno: %d\n", old_errno);
}

int dosetns(int pid, char *nstype) {
	__do_close int ns_fd = -EBADF;
	char buf[PATH_MAX];

	sprintf(buf, "/proc/%d/ns/%s", pid, nstype);
	ns_fd = open(buf, O_RDONLY);
	if (ns_fd < 0) {
		error("error: open namespace");
		return -1;
	}

	if (setns(ns_fd, 0) < 0) {
		error("error: setns");
		return -1;
	}

	return 0;
}

int dosetns_file(char *file, char *nstype) {
	__do_close int ns_fd = -EBADF;

	ns_fd = open(file, O_RDONLY);
	if (ns_fd < 0) {
		error("error: open namespace");
		return -1;
	}

	if (setns(ns_fd, 0) < 0) {
		error("error: setns");
		return -1;
	}

	return 0;
}

int preserve_ns(pid_t pid, int ns_fd, const char *ns)
{
	int ret;
	if (ns_fd >= 0)
		return openat(ns_fd, ns, O_RDONLY | O_CLOEXEC);

// 5 /proc + 21 /int_as_str + 3 /ns + 20 /NS_NAME + 1 \0
#define __NS_PATH_LEN 50
	char path[__NS_PATH_LEN];

	// This way we can use this function to also check whether namespaces
	// are supported by the kernel by passing in the NULL or the empty
	// string.
	ret = snprintf(path, __NS_PATH_LEN, "/proc/%d/ns%s%s", pid,
		       !ns || strcmp(ns, "") == 0 ? "" : "/",
		       !ns || strcmp(ns, "") == 0 ? "" : ns);
	errno = EFBIG;
	if (ret < 0 || (size_t)ret >= __NS_PATH_LEN)
		return -EFBIG;

	return open(path, O_RDONLY | O_CLOEXEC);
}

// in_same_namespace - Check whether two processes are in the same namespace.
// @pid1	- PID of the first process.
// @pid2	- PID of the second process.
// @ns_fd2	- ns_fd @pid2.
// @ns   - Name of the namespace to check. Must correspond to one of the names
//         for the namespaces as shown in /proc/<pid/ns/
//
// If the two processes are not in the same namespace returns an fd to the
// namespace of the second process identified by @pid2. If the two processes are
// in the same namespace returns -EINVAL, -1 if an error occurred.
static int in_same_namespace(pid_t pid1, pid_t pid2, int ns_fd_pid2, const char *ns)
{
	__do_close int ns_fd1 = -EBADF, ns_fd2 = -EBADF;
	int ret = -1;
	struct stat ns_st1, ns_st2;

	ns_fd1 = preserve_ns(pid1, -EBADF, ns);
	if (ns_fd1 < 0) {
		// The kernel does not support this namespace. This is not an
		// error.
		if (errno == ENOENT)
			return -EINVAL;

		return -1;
	}

	ns_fd2 = preserve_ns(pid2, ns_fd_pid2, ns);
	if (ns_fd2 < 0)
		return -1;

	ret = fstat(ns_fd1, &ns_st1);
	if (ret < 0)
		return -1;

	ret = fstat(ns_fd2, &ns_st2);
	if (ret < 0)
		return -1;

	// processes are in the same namespace
	if ((ns_st1.st_dev == ns_st2.st_dev ) && (ns_st1.st_ino == ns_st2.st_ino))
		return -EINVAL;

	// processes are in different namespaces
	return move_fd(ns_fd2);
}

static void __attach_userns(int pid, int ns_fd)
{
	__do_close int userns_fd = -EBADF;
	int ret;

	userns_fd = in_same_namespace(getpid(), pid, ns_fd, "user");
	if (userns_fd < 0) {
		if (userns_fd == -EINVAL)
			return;

		_exit(EXIT_FAILURE);
	}

	ret = setns(userns_fd, CLONE_NEWUSER);
	if (ret < 0) {
		fprintf(stderr, "Failed setns to container user namespace: %s\n", strerror(errno));
		_exit(EXIT_FAILURE);
	}

	ret = setuid(0);
	if (ret < 0) {
		fprintf(stderr, "Failed setuid to container root user: %s\n", strerror(errno));
		_exit(1);
	}

	ret = setgid(0);
	if (ret < 0) {
		fprintf(stderr, "Failed setgid to container root group: %s\n", strerror(errno));
		_exit(1);
	}

	ret = setgroups(0, NULL);
	if (ret < 0) {
		fprintf(stderr, "Failed setgroups to container root groups: %s\n", strerror(errno));
		_exit(1);
	}
}

void attach_userns(int pid)
{
	return __attach_userns(pid, -EBADF);
}

void attach_userns_fd(int ns_fd)
{
	return __attach_userns(-1, ns_fd);
}

int pidfd_nsfd(int pidfd, pid_t pid)
{
	__do_close int ns_fd = -EBADF;
	int ret;
	char path[100];

	ret = snprintf(path, sizeof(path), "/proc/%d/ns", pid);
	if (ret < 0 || (size_t)ret >= sizeof(path))
		return -E2BIG;

	ns_fd = open(path, O_DIRECTORY | O_RDONLY | O_CLOEXEC);
	if (ns_fd < 0)
		return -errno;

	if (pidfd >= 0) {
		// Verify that the pid has not been recycled and our /proc/<pid> handle
		// is still valid.
		ret = pidfd_send_signal(pidfd, 0, NULL, 0);
		if (ret && errno != EPERM)
			return -errno;
	}

	return move_fd(ns_fd);
}

bool setnsat(int ns_fd, const char *ns)
{
	__do_close int fd = -EBADF;

	fd = openat(ns_fd, ns, O_RDONLY | O_CLOEXEC);
	if (fd < 0)
		return false;

	return setns(fd, 0) == 0;
}

static ssize_t lxc_read_nointr(int fd, void *buf, size_t count)
{
	ssize_t ret;
again:
	ret = read(fd, buf, count);
	if (ret < 0 && errno == EINTR)
		goto again;

	return ret;
}

static char *file_to_buf(char *path, ssize_t *length)
{
	__do_close int fd = -EBADF;
	__do_free char *copy = NULL;
	char buf[PATH_MAX];

	if (!length)
		return NULL;

	fd = open(path, O_RDONLY | O_CLOEXEC);
	if (fd < 0)
		return NULL;

	*length = 0;
	for (;;) {
		int n;
		char *old = copy;

		n = lxc_read_nointr(fd, buf, sizeof(buf));
		if (n < 0)
			return NULL;
		if (!n)
			break;

		copy = realloc(old, (*length + n) * sizeof(*old));
		if (!copy)
			return NULL;

		memcpy(copy + *length, buf, n);
		*length += n;
	}

	return move_ptr(copy);
}

__attribute__((constructor)) void init(void) {
	__do_free char *cmdline = NULL;
	int ret;

	cmdline_buf = file_to_buf("/proc/self/cmdline", &cmdline_size);
	if (!cmdline_buf)
		_exit(232);

	// Skip the first argument (but don't fail on missing second argument)
	cmdline = cmdline_cur = cmdline_buf;
	while (*cmdline_cur != 0)
		cmdline_cur++;
	cmdline_cur++;
	if (cmdline_size <= cmdline_cur - cmdline_buf) {
		checkfeature();
		return;
	}

	// Intercepts some subcommands
	if (strcmp(cmdline_cur, "forkexec") == 0)
		forkexec();
	if (strcmp(cmdline_cur, "forkfile") == 0)
		forkfile();
	else if (strcmp(cmdline_cur, "forksyscall") == 0)
		forksyscall();
	else if (strcmp(cmdline_cur, "forkmount") == 0)
		forkmount();
	else if (strcmp(cmdline_cur, "forknet") == 0)
		forknet();
	else if (strcmp(cmdline_cur, "forkproxy") == 0)
		forkproxy();
	else if (strcmp(cmdline_cur, "forkuevent") == 0)
		forkuevent();
	else if (strcmp(cmdline_cur, "forkzfs") == 0) {
		ret = unshare(CLONE_NEWNS);
		if (ret < 0) {
			fprintf(stderr, "Failed unshare of mount namespace: %s\n", strerror(errno));
			return;
		}
	} else if (strncmp(cmdline_cur, "-", 1) == 0 || strcmp(cmdline_cur, "daemon") == 0)
		checkfeature();
}
*/
import "C"
<commit_msg>nsexec: remove unused dosetns() function<commit_after>/**
 * This file is a bit funny. The goal here is to use setns() to manipulate
 * files inside the container, so we don't have to reason about the paths to
 * make sure they don't escape (we can simply rely on the kernel for
 * correctness). Unfortunately, you can't setns() to a mount namespace with a
 * multi-threaded program, which every golang binary is. However, by declaring
 * our init as an initializer, we can capture process control before it is
 * transferred to the golang runtime, so we can then setns() as we'd like
 * before golang has a chance to set up any threads. So, we implement two new
 * lxd fork* commands which are captured here, and take a file on the host fs
 * and copy it into the container ns.
 *
 * An alternative to this would be to move this code into a separate binary,
 * which of course has problems of its own when it comes to packaging (how do
 * we find the binary, what do we do if someone does file push and it is
 * missing, etc.). After some discussion, even though the embedded method is
 * somewhat convoluted, it was preferred.
 */
package main

import (
	// Used by cgo
	_ "github.com/lxc/lxd/lxd/include"
)

/*
#ifndef _GNU_SOURCE
#define _GNU_SOURCE 1
#endif
#include <errno.h>
#include <fcntl.h>
#include <grp.h>
#include <linux/limits.h>
#include <sched.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>

#include "include/memory_utils.h"
#include "include/process_utils.h"
#include "include/syscall_numbers.h"

// External functions
extern void checkfeature();
extern void forkexec();
extern void forkfile();
extern void forksyscall();
extern void forkmount();
extern void forknet();
extern void forkproxy();
extern void forkuevent();

// Command line parsing and tracking
char *cmdline_buf = NULL;
char *cmdline_cur = NULL;
ssize_t cmdline_size = -1;

int wait_for_pid(pid_t pid)
{
	int status, ret;

again:
	ret = waitpid(pid, &status, 0);
	if (ret == -1) {
		if (errno == EINTR)
			goto again;
		return -1;
	}
	if (ret != pid)
		goto again;
	if (!WIFEXITED(status) || WEXITSTATUS(status) != 0)
		return -1;
	return 0;
}

char* advance_arg(bool required) {
	while (*cmdline_cur != 0)
		cmdline_cur++;

	cmdline_cur++;
	if (cmdline_size <= cmdline_cur - cmdline_buf) {
		if (!required)
			return NULL;

		fprintf(stderr, "not enough arguments\n");
		_exit(1);
	}

	return cmdline_cur;
}

void error(char *msg)
{
	int old_errno = errno;

	if (old_errno == 0) {
		fprintf(stderr, "%s\n", msg);
		fprintf(stderr, "errno: 0\n");
		return;
	}

	perror(msg);
	fprintf(stderr, "errno: %d\n", old_errno);
}

int dosetns_file(char *file, char *nstype) {
	__do_close int ns_fd = -EBADF;

	ns_fd = open(file, O_RDONLY);
	if (ns_fd < 0) {
		error("error: open namespace");
		return -1;
	}

	if (setns(ns_fd, 0) < 0) {
		error("error: setns");
		return -1;
	}

	return 0;
}

int preserve_ns(pid_t pid, int ns_fd, const char *ns)
{
	int ret;
	if (ns_fd >= 0)
		return openat(ns_fd, ns, O_RDONLY | O_CLOEXEC);

// 5 /proc + 21 /int_as_str + 3 /ns + 20 /NS_NAME + 1 \0
#define __NS_PATH_LEN 50
	char path[__NS_PATH_LEN];

	// This way we can use this function to also check whether namespaces
	// are supported by the kernel by passing in the NULL or the empty
	// string.
	ret = snprintf(path, __NS_PATH_LEN, "/proc/%d/ns%s%s", pid,
		       !ns || strcmp(ns, "") == 0 ? "" : "/",
		       !ns || strcmp(ns, "") == 0 ? "" : ns);
	errno = EFBIG;
	if (ret < 0 || (size_t)ret >= __NS_PATH_LEN)
		return -EFBIG;

	return open(path, O_RDONLY | O_CLOEXEC);
}

// in_same_namespace - Check whether two processes are in the same namespace.
// @pid1	- PID of the first process.
// @pid2	- PID of the second process.
// @ns_fd2	- ns_fd @pid2.
// @ns   - Name of the namespace to check. Must correspond to one of the names
//         for the namespaces as shown in /proc/<pid/ns/
//
// If the two processes are not in the same namespace returns an fd to the
// namespace of the second process identified by @pid2. If the two processes are
// in the same namespace returns -EINVAL, -1 if an error occurred.
static int in_same_namespace(pid_t pid1, pid_t pid2, int ns_fd_pid2, const char *ns)
{
	__do_close int ns_fd1 = -EBADF, ns_fd2 = -EBADF;
	int ret = -1;
	struct stat ns_st1, ns_st2;

	ns_fd1 = preserve_ns(pid1, -EBADF, ns);
	if (ns_fd1 < 0) {
		// The kernel does not support this namespace. This is not an
		// error.
		if (errno == ENOENT)
			return -EINVAL;

		return -1;
	}

	ns_fd2 = preserve_ns(pid2, ns_fd_pid2, ns);
	if (ns_fd2 < 0)
		return -1;

	ret = fstat(ns_fd1, &ns_st1);
	if (ret < 0)
		return -1;

	ret = fstat(ns_fd2, &ns_st2);
	if (ret < 0)
		return -1;

	// processes are in the same namespace
	if ((ns_st1.st_dev == ns_st2.st_dev ) && (ns_st1.st_ino == ns_st2.st_ino))
		return -EINVAL;

	// processes are in different namespaces
	return move_fd(ns_fd2);
}

static void __attach_userns(int pid, int ns_fd)
{
	__do_close int userns_fd = -EBADF;
	int ret;

	userns_fd = in_same_namespace(getpid(), pid, ns_fd, "user");
	if (userns_fd < 0) {
		if (userns_fd == -EINVAL)
			return;

		_exit(EXIT_FAILURE);
	}

	ret = setns(userns_fd, CLONE_NEWUSER);
	if (ret < 0) {
		fprintf(stderr, "Failed setns to container user namespace: %s\n", strerror(errno));
		_exit(EXIT_FAILURE);
	}

	ret = setuid(0);
	if (ret < 0) {
		fprintf(stderr, "Failed setuid to container root user: %s\n", strerror(errno));
		_exit(1);
	}

	ret = setgid(0);
	if (ret < 0) {
		fprintf(stderr, "Failed setgid to container root group: %s\n", strerror(errno));
		_exit(1);
	}

	ret = setgroups(0, NULL);
	if (ret < 0) {
		fprintf(stderr, "Failed setgroups to container root groups: %s\n", strerror(errno));
		_exit(1);
	}
}

void attach_userns(int pid)
{
	return __attach_userns(pid, -EBADF);
}

void attach_userns_fd(int ns_fd)
{
	return __attach_userns(-1, ns_fd);
}

int pidfd_nsfd(int pidfd, pid_t pid)
{
	__do_close int ns_fd = -EBADF;
	int ret;
	char path[100];

	ret = snprintf(path, sizeof(path), "/proc/%d/ns", pid);
	if (ret < 0 || (size_t)ret >= sizeof(path))
		return -E2BIG;

	ns_fd = open(path, O_DIRECTORY | O_RDONLY | O_CLOEXEC);
	if (ns_fd < 0)
		return -errno;

	if (pidfd >= 0) {
		// Verify that the pid has not been recycled and our /proc/<pid> handle
		// is still valid.
		ret = pidfd_send_signal(pidfd, 0, NULL, 0);
		if (ret && errno != EPERM)
			return -errno;
	}

	return move_fd(ns_fd);
}

bool setnsat(int ns_fd, const char *ns)
{
	__do_close int fd = -EBADF;

	fd = openat(ns_fd, ns, O_RDONLY | O_CLOEXEC);
	if (fd < 0)
		return false;

	return setns(fd, 0) == 0;
}

static ssize_t lxc_read_nointr(int fd, void *buf, size_t count)
{
	ssize_t ret;
again:
	ret = read(fd, buf, count);
	if (ret < 0 && errno == EINTR)
		goto again;

	return ret;
}

static char *file_to_buf(char *path, ssize_t *length)
{
	__do_close int fd = -EBADF;
	__do_free char *copy = NULL;
	char buf[PATH_MAX];

	if (!length)
		return NULL;

	fd = open(path, O_RDONLY | O_CLOEXEC);
	if (fd < 0)
		return NULL;

	*length = 0;
	for (;;) {
		int n;
		char *old = copy;

		n = lxc_read_nointr(fd, buf, sizeof(buf));
		if (n < 0)
			return NULL;
		if (!n)
			break;

		copy = realloc(old, (*length + n) * sizeof(*old));
		if (!copy)
			return NULL;

		memcpy(copy + *length, buf, n);
		*length += n;
	}

	return move_ptr(copy);
}

__attribute__((constructor)) void init(void) {
	__do_free char *cmdline = NULL;
	int ret;

	cmdline_buf = file_to_buf("/proc/self/cmdline", &cmdline_size);
	if (!cmdline_buf)
		_exit(232);

	// Skip the first argument (but don't fail on missing second argument)
	cmdline = cmdline_cur = cmdline_buf;
	while (*cmdline_cur != 0)
		cmdline_cur++;
	cmdline_cur++;
	if (cmdline_size <= cmdline_cur - cmdline_buf) {
		checkfeature();
		return;
	}

	// Intercepts some subcommands
	if (strcmp(cmdline_cur, "forkexec") == 0)
		forkexec();
	if (strcmp(cmdline_cur, "forkfile") == 0)
		forkfile();
	else if (strcmp(cmdline_cur, "forksyscall") == 0)
		forksyscall();
	else if (strcmp(cmdline_cur, "forkmount") == 0)
		forkmount();
	else if (strcmp(cmdline_cur, "forknet") == 0)
		forknet();
	else if (strcmp(cmdline_cur, "forkproxy") == 0)
		forkproxy();
	else if (strcmp(cmdline_cur, "forkuevent") == 0)
		forkuevent();
	else if (strcmp(cmdline_cur, "forkzfs") == 0) {
		ret = unshare(CLONE_NEWNS);
		if (ret < 0) {
			fprintf(stderr, "Failed unshare of mount namespace: %s\n", strerror(errno));
			return;
		}
	} else if (strncmp(cmdline_cur, "-", 1) == 0 || strcmp(cmdline_cur, "daemon") == 0)
		checkfeature();
}
*/
import "C"
<|endoftext|>


<commit_before>package bot

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"

	"cloud.google.com/go/datastore"
	"github.com/nlopes/slack"
	"golang.org/x/net/context"
)

type (
	gerritCL struct {
		Project         string `json:"project"`
		ChangeID        string `json:"change_id"`
		Number          int    `json:"_number"`
		Subject         string `json:"subject"`
		Branch          string `json:"branch"`
		CurrentRevision string `json:"current_revision"`
		Revisions       map[string]struct {
			Commit struct {
				Subject string `json:"subject"`
				Message string `json:"message"`
			} `json:"commit"`
		} `json:"revisions"`
	}

	goCL struct {
		Tweeted   bool      `datastore:"Tweeted,noindex"`
		URL       string    `datastore:"URL,noindex"`
		Message   string    `datastore:"Message,noindex"`
		CrawledAt time.Time `datastore:"CrawledAt"`
	}
)

func (cl *gerritCL) link() string {
	return fmt.Sprintf("https://go-review.googlesource.com/c/%d/", cl.Number)
}

func (cl *gerritCL) message() string {
	subject := cl.Subject
	if cl.Project != "go" {
		subject = fmt.Sprintf("[%s] %s", cl.Project, subject)
	}

	return subject
}

func (b *Bot) datastoreClient() (context.Context, *datastore.Client) {
	ctx := context.Background()
	projectID := "gopher-slack-bot"
	dsClient, err := datastore.NewClient(ctx, projectID)
	if err != nil {
		b.logf("Failed to create client: %v", err)
		panic(err)
	}

	return ctx, dsClient
}

func (b *Bot) getCLFromDS(ctx context.Context, dsClient *datastore.Client, query *datastore.Query) (*datastore.Key, *goCL, error) {
	iter := dsClient.Run(ctx, query)

	dst := &goCL{}
	key, err := iter.Next(dst)
	if err != nil && err != datastore.Done {
		b.logf("error while fetching history: %v\n", err)
		return nil, nil, err
	}

	return key, dst, nil
}

func (b *Bot) getLastSeenCL(ctx context.Context, dsClient *datastore.Client) (int, error) {
	latestCLQuery := datastore.NewQuery("GoCL").
		Order("-CrawledAt").
		Limit(1).
		KeysOnly()

	key, _, err := b.getCLFromDS(ctx, dsClient, latestCLQuery)
	if err != nil {
		return -1, err
	}
	if key == nil {
		return -1, nil
	}
	return int(key.ID()), nil
}

func (b *Bot) wasShown(ctx context.Context, dsClient *datastore.Client, cl gerritCL) (bool, error) {
	key := datastore.NewKey(ctx, "GoCL", "", int64(cl.Number), nil)
	query := datastore.NewQuery("GoCL").Ancestor(key)
	key, _, err := b.getCLFromDS(ctx, dsClient, query)
	return key != nil, err
}

func (b *Bot) saveCL(ctx context.Context, dsClient *datastore.Client, cl gerritCL) error {
	taskKey := datastore.NewKey(ctx, "GoCL", "", int64(cl.Number), nil)
	gocl := &goCL{
		URL:       cl.link(),
		Message:   cl.message(),
		CrawledAt: time.Now(),
	}
	_, err := dsClient.Put(ctx, taskKey, gocl)
	return err
}

func (b *Bot) processCLList(lastID int) int {
	req, err := http.NewRequest("GET", b.gerritLink, nil)
	req.Header.Add("User-Agent", "Gophers Slack bot")
	resp, err := b.client.Do(req)
	if err != nil {
		b.logf("%s\n", err)
		return lastID
	}

	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		b.logf("got non-200 code: %d from gerrit api", resp.StatusCode)
		return lastID
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		b.logf("%s\n", err)
		return lastID
	}

	if len(body) < 4 {
		b.logf("got body: %s\n", string(body))
		return lastID
	}

	// Fix Gerrit adding a random prefix )]}'
	body = body[4:]
	cls := []gerritCL{}
	err = json.Unmarshal(body, &cls)
	if err != nil {
		b.logf("%s\n", err)
		return lastID
	}

	foundIdx := len(cls) - 1
	for idx := len(cls) - 1; idx >= 0; idx-- {
		if cls[idx].Number == lastID {
			foundIdx = idx
			break
		}
	}

	pubChannel := b.channels["golang-cls"].slackID
	if pubChannel[:1] == "#" {
		pubChannel = pubChannel[1:]
	}

	ctx, dsClient := b.datastoreClient()
	defer dsClient.Close()
	for idx := foundIdx - 1; idx >= 0; idx-- {
		cl := cls[idx]
		if cl.Branch != "master" {
			continue
		}

		if shown, err := b.wasShown(ctx, dsClient, cl); err == nil {
			if shown {
				continue
			}
		} else {
			b.logf("got error: %v\n", err)
			continue
		}

		msg := slack.Attachment{
			Title:     cl.Subject,
			TitleLink: cl.link(),
			Text:      cl.Revisions[cl.CurrentRevision].Commit.Message,
			Footer:    cl.ChangeID,
		}
		params := slack.PostMessageParameters{AsUser: true}
		params.Attachments = append(params.Attachments, msg)

		err = b.saveCL(ctx, dsClient, cl)
		if err != nil {
			b.logf("got error while saving CL to datastore: %v", err)
			return lastID
		}

		_, _, err = b.slackBotAPI.PostMessage(b.channels["golang_cls"].slackID, fmt.Sprintf("[%d] %s: %s", cl.Number, cl.message(), cl.link()), params)
		if err != nil {
			b.logf("%s\n", err)
			continue
		}

		lastID = cl.Number

		_, _, err = b.slackBotAPI.PostMessage(pubChannel, fmt.Sprintf("[%d] %s: %s", cl.Number, cl.message(), cl.link()), params)
		if err != nil {
			b.logf("%s\n", err)
			continue
		}
	}

	return lastID
}

func (b *Bot) shareCL(event *slack.MessageEvent, eventText string) {
	if !b.specialRestrictions("golang_cls", event) {
		b.logf("share attempt caught: %#v\n", event)

		params := slack.PostMessageParameters{AsUser: true}
		_, _, err := b.slackBotAPI.PostMessage(event.User, `You are not authorized to share CLs`, params)
		if err != nil {
			b.logf("%s\n", err)
		}
		return
	}

	eventText = strings.Replace(eventText, "share cl", "", -1)
	eventText = strings.Trim(eventText, " \n")

	clNumber, err := strconv.ParseInt(eventText, 10, 64)
	if err != nil {
		b.logf("could not convert string to int: %v from event: %#v\n", err, event)

		params := slack.PostMessageParameters{AsUser: true}
		_, _, err := b.slackBotAPI.PostMessage(event.User, `Could not share CL, please try again`, params)
		if err != nil {
			b.logf("%s\n", err)
		}
		return
	}

	ctx, dsClient := b.datastoreClient()
	defer dsClient.Close()

	key := datastore.NewKey(ctx, "GoCL", "", clNumber, nil)
	query := datastore.NewQuery("GoCL").Ancestor(key)
	key, cl, err := b.getCLFromDS(ctx, dsClient, query)
	if err != nil {
		b.logf("error while retriving CL from the DB: %v\n", err)

		params := slack.PostMessageParameters{AsUser: true}
		_, _, err := b.slackBotAPI.PostMessage(event.User, `Could not share CL, please try again`, params)
		if err != nil {
			b.logf("%s\n", err)
		}
		return
	}

	if cl.Tweeted {
		params := slack.PostMessageParameters{AsUser: true}
		_, _, err := b.slackBotAPI.PostMessage(event.User, `Already tweeted`, params)
		if err != nil {
			b.logf("%s\n", err)
		}
		return
	}

	message := cl.Message + " " + cl.URL
	_, err = b.twitterAPI.PostTweet(message, nil)
	if err != nil {
		b.logf("got error while tweeting CL: %d %#v\n", clNumber, err)

		params := slack.PostMessageParameters{AsUser: true}
		_, _, err := b.slackBotAPI.PostMessage(event.User, `Could not share CL, please try again`, params)
		if err != nil {
			b.logf("%s\n", err)
		}
		return
	}

	cl.Tweeted = true
	err = b.saveCL(ctx, dsClient, cl)
	if err != nil {
		b.logf("got error while updating CL to datastore: %v", err)

		params := slack.PostMessageParameters{AsUser: true}
		_, _, err := b.slackBotAPI.PostMessage(event.User, `Could not update tweet status in the DB`, params)
		if err != nil {
			b.logf("%s\n", err)
		}
	}
}

// MonitorGerrit handles the Gerrit changes
func (b *Bot) MonitorGerrit(duration time.Duration) {
	tk := time.NewTicker(duration)
	defer tk.Stop()

	ctx, dsClient := b.datastoreClient()

	lastID, err := b.getLastSeenCL(ctx, dsClient)
	if err != nil {
		b.logf("got error while loading last ID from the datastore: %v\n", err)
		dsClient.Close()
		return
	}
	dsClient.Close()

	lastID = b.processCLList(lastID)
	for range tk.C {
		lastID = b.processCLList(lastID)
	}
}
<commit_msg>Fix previous commit<commit_after>package bot

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"

	"cloud.google.com/go/datastore"
	"github.com/nlopes/slack"
	"golang.org/x/net/context"
)

type (
	gerritCL struct {
		Project         string `json:"project"`
		ChangeID        string `json:"change_id"`
		Number          int    `json:"_number"`
		Subject         string `json:"subject"`
		Branch          string `json:"branch"`
		CurrentRevision string `json:"current_revision"`
		Revisions       map[string]struct {
			Commit struct {
				Subject string `json:"subject"`
				Message string `json:"message"`
			} `json:"commit"`
		} `json:"revisions"`
	}

	storedCL struct {
		Tweeted   bool      `datastore:"Tweeted,noindex"`
		URL       string    `datastore:"URL,noindex"`
		Message   string    `datastore:"Message,noindex"`
		CrawledAt time.Time `datastore:"CrawledAt"`
	}
)

func (cl *gerritCL) link() string {
	return fmt.Sprintf("https://go-review.googlesource.com/c/%d/", cl.Number)
}

func (cl *gerritCL) message() string {
	subject := cl.Subject
	if cl.Project != "go" {
		subject = fmt.Sprintf("[%s] %s", cl.Project, subject)
	}

	return subject
}

func (b *Bot) datastoreClient() (context.Context, *datastore.Client) {
	ctx := context.Background()
	projectID := "gopher-slack-bot"
	dsClient, err := datastore.NewClient(ctx, projectID)
	if err != nil {
		b.logf("Failed to create client: %v", err)
		panic(err)
	}

	return ctx, dsClient
}

func (b *Bot) getCLFromDS(ctx context.Context, dsClient *datastore.Client, query *datastore.Query) (*datastore.Key, *storedCL, error) {
	iter := dsClient.Run(ctx, query)

	dst := &storedCL{}
	key, err := iter.Next(dst)
	if err != nil && err != datastore.Done {
		b.logf("error while fetching history: %v\n", err)
		return nil, nil, err
	}

	return key, dst, nil
}

func (b *Bot) getLastSeenCL(ctx context.Context, dsClient *datastore.Client) (int, error) {
	latestCLQuery := datastore.NewQuery("GoCL").
		Order("-CrawledAt").
		Limit(1).
		KeysOnly()

	key, _, err := b.getCLFromDS(ctx, dsClient, latestCLQuery)
	if err != nil {
		return -1, err
	}
	if key == nil {
		return -1, nil
	}
	return int(key.ID()), nil
}

func (b *Bot) wasShown(ctx context.Context, dsClient *datastore.Client, cl gerritCL) (bool, error) {
	key := datastore.NewKey(ctx, "GoCL", "", int64(cl.Number), nil)
	query := datastore.NewQuery("GoCL").Ancestor(key)
	key, _, err := b.getCLFromDS(ctx, dsClient, query)
	return key != nil, err
}

func (b *Bot) saveCL(ctx context.Context, dsClient *datastore.Client, cl gerritCL) error {
	taskKey := datastore.NewKey(ctx, "GoCL", "", int64(cl.Number), nil)
	gocl := &storedCL{
		URL:       cl.link(),
		Message:   cl.message(),
		CrawledAt: time.Now(),
	}
	_, err := dsClient.Put(ctx, taskKey, gocl)
	return err
}

func (b *Bot) updateCL(ctx context.Context, dsClient *datastore.Client, key *datastore.Key, cl *storedCL) error {
	taskKey := datastore.NewKey(ctx, "GoCL", "", key.ID(), nil)
	_, err := dsClient.Put(ctx, taskKey, cl)
	return err
}

func (b *Bot) processCLList(lastID int) int {
	req, err := http.NewRequest("GET", b.gerritLink, nil)
	req.Header.Add("User-Agent", "Gophers Slack bot")
	resp, err := b.client.Do(req)
	if err != nil {
		b.logf("%s\n", err)
		return lastID
	}

	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		b.logf("got non-200 code: %d from gerrit api", resp.StatusCode)
		return lastID
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		b.logf("%s\n", err)
		return lastID
	}

	if len(body) < 4 {
		b.logf("got body: %s\n", string(body))
		return lastID
	}

	// Fix Gerrit adding a random prefix )]}'
	body = body[4:]
	cls := []gerritCL{}
	err = json.Unmarshal(body, &cls)
	if err != nil {
		b.logf("%s\n", err)
		return lastID
	}

	foundIdx := len(cls) - 1
	for idx := len(cls) - 1; idx >= 0; idx-- {
		if cls[idx].Number == lastID {
			foundIdx = idx
			break
		}
	}

	pubChannel := b.channels["golang-cls"].slackID
	if pubChannel[:1] == "#" {
		pubChannel = pubChannel[1:]
	}

	ctx, dsClient := b.datastoreClient()
	defer dsClient.Close()
	for idx := foundIdx - 1; idx >= 0; idx-- {
		cl := cls[idx]
		if cl.Branch != "master" {
			continue
		}

		if shown, err := b.wasShown(ctx, dsClient, cl); err == nil {
			if shown {
				continue
			}
		} else {
			b.logf("got error: %v\n", err)
			continue
		}

		msg := slack.Attachment{
			Title:     cl.Subject,
			TitleLink: cl.link(),
			Text:      cl.Revisions[cl.CurrentRevision].Commit.Message,
			Footer:    cl.ChangeID,
		}
		params := slack.PostMessageParameters{AsUser: true}
		params.Attachments = append(params.Attachments, msg)

		err = b.saveCL(ctx, dsClient, cl)
		if err != nil {
			b.logf("got error while saving CL to datastore: %v", err)
			return lastID
		}

		_, _, err = b.slackBotAPI.PostMessage(b.channels["golang_cls"].slackID, fmt.Sprintf("[%d] %s: %s", cl.Number, cl.message(), cl.link()), params)
		if err != nil {
			b.logf("%s\n", err)
			continue
		}

		lastID = cl.Number

		_, _, err = b.slackBotAPI.PostMessage(pubChannel, fmt.Sprintf("[%d] %s: %s", cl.Number, cl.message(), cl.link()), params)
		if err != nil {
			b.logf("%s\n", err)
			continue
		}
	}

	return lastID
}

func (b *Bot) shareCL(event *slack.MessageEvent, eventText string) {
	if !b.specialRestrictions("golang_cls", event) {
		b.logf("share attempt caught: %#v\n", event)

		params := slack.PostMessageParameters{AsUser: true}
		_, _, err := b.slackBotAPI.PostMessage(event.User, `You are not authorized to share CLs`, params)
		if err != nil {
			b.logf("%s\n", err)
		}
		return
	}

	eventText = strings.Replace(eventText, "share cl", "", -1)
	eventText = strings.Trim(eventText, " \n")

	clNumber, err := strconv.ParseInt(eventText, 10, 64)
	if err != nil {
		b.logf("could not convert string to int: %v from event: %#v\n", err, event)

		params := slack.PostMessageParameters{AsUser: true}
		_, _, err := b.slackBotAPI.PostMessage(event.User, `Could not share CL, please try again`, params)
		if err != nil {
			b.logf("%s\n", err)
		}
		return
	}

	ctx, dsClient := b.datastoreClient()
	defer dsClient.Close()

	key := datastore.NewKey(ctx, "GoCL", "", clNumber, nil)
	query := datastore.NewQuery("GoCL").Ancestor(key)
	key, cl, err := b.getCLFromDS(ctx, dsClient, query)
	if err != nil {
		b.logf("error while retriving CL from the DB: %v\n", err)

		params := slack.PostMessageParameters{AsUser: true}
		_, _, err := b.slackBotAPI.PostMessage(event.User, `Could not share CL, please try again`, params)
		if err != nil {
			b.logf("%s\n", err)
		}
		return
	}

	if cl.Tweeted {
		params := slack.PostMessageParameters{AsUser: true}
		_, _, err := b.slackBotAPI.PostMessage(event.User, `Already tweeted`, params)
		if err != nil {
			b.logf("%s\n", err)
		}
		return
	}

	message := cl.Message + " " + cl.URL
	_, err = b.twitterAPI.PostTweet(message, nil)
	if err != nil {
		b.logf("got error while tweeting CL: %d %#v\n", clNumber, err)

		params := slack.PostMessageParameters{AsUser: true}
		_, _, err := b.slackBotAPI.PostMessage(event.User, `Could not share CL, please try again`, params)
		if err != nil {
			b.logf("%s\n", err)
		}
		return
	}

	cl.Tweeted = true
	err = b.updateCL(ctx, dsClient, key, cl)
	if err != nil {
		b.logf("got error while updating CL to datastore: %v", err)

		params := slack.PostMessageParameters{AsUser: true}
		_, _, err := b.slackBotAPI.PostMessage(event.User, `Could not update tweet status in the DB`, params)
		if err != nil {
			b.logf("%s\n", err)
		}
	}
}

// MonitorGerrit handles the Gerrit changes
func (b *Bot) MonitorGerrit(duration time.Duration) {
	tk := time.NewTicker(duration)
	defer tk.Stop()

	ctx, dsClient := b.datastoreClient()

	lastID, err := b.getLastSeenCL(ctx, dsClient)
	if err != nil {
		b.logf("got error while loading last ID from the datastore: %v\n", err)
		dsClient.Close()
		return
	}
	dsClient.Close()

	lastID = b.processCLList(lastID)
	for range tk.C {
		lastID = b.processCLList(lastID)
	}
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
	"os"
	"os/signal"
	"runtime"
	"errors"
//    "strconv"
    "encoding/hex"
    "encoding/binary"
    "github.com/fuzxxl/nfc/2.0/nfc"
    "github.com/fuzxxl/freefare/0.3/freefare"
    "code.google.com/p/go-sqlite/go1/sqlite3"
    "time"
    "github.com/davecheney/gpio"
    "./keydiversification"
    "./helpers"
)

func heartbeat() {
    for {
        time.Sleep(2000 * time.Millisecond)
        fmt.Println("Dunka-dunk")
    }
}

func pulse_gpio(pin gpio.Pin, ms int) {
    pin.Set()
    time.Sleep(time.Duration(ms) * time.Millisecond)
    pin.Clear()
}

func clear_and_close(pin gpio.Pin) {
    pin.Clear()
    pin.Close()
}

// Use structs to pass data around so I can refactor
type AppInfo struct {
    aid freefare.DESFireAid
    aidbytes []byte
    sysid []byte
    acl_read_base []byte
    acl_write_base []byte
    acl_file_id byte
}

type KeyChain struct {
    uid_read_key_id byte
    acl_read_key_id byte
    acl_write_key_id byte

    uid_read_key *freefare.DESFireKey
    acl_read_key *freefare.DESFireKey
    acl_write_key *freefare.DESFireKey
}

var (
    keychain = KeyChain{}
    appinfo = AppInfo{}
)

func init_appinfo() {
    keymap, err := helpers.LoadYAMLFile("keys.yaml")
    if err != nil {
        panic(err)
    }

    appmap, err := helpers.LoadYAMLFile("apps.yaml")
    if err != nil {
        panic(err)
    }

    // Application-id
    appinfo.aid, err = helpers.String2aid(appmap["hacklab_acl"].(map[interface{}]interface{})["aid"].(string))
    if err != nil {
        panic(err)
    }

    // Needed for diversification
    appinfo.aidbytes = helpers.Aid2bytes(appinfo.aid)
    appinfo.sysid, err = hex.DecodeString(appmap["hacklab_acl"].(map[interface{}]interface{})["sysid"].(string))
    if err != nil {
        panic(err)
    }

    appinfo.acl_file_id, err = helpers.String2byte(appmap["hacklab_acl"].(map[interface{}]interface{})["acl_file_id"].(string))
    if err != nil {
        panic(err)
    }

    // Key id numbers from config
    keychain.uid_read_key_id, err = helpers.String2byte(appmap["hacklab_acl"].(map[interface{}]interface{})["uid_read_key_id"].(string))
    if err != nil {
        panic(err)
    }
    keychain.acl_read_key_id, err = helpers.String2byte(appmap["hacklab_acl"].(map[interface{}]interface{})["acl_read_key_id"].(string))
    if err != nil {
        panic(err)
    }
    keychain.acl_write_key_id, err = helpers.String2byte(appmap["hacklab_acl"].(map[interface{}]interface{})["acl_write_key_id"].(string))
    if err != nil {
        panic(err)
    }

    // The static app key to read UID
    keychain.uid_read_key, err = helpers.String2aeskey(keymap["uid_read_key"].(string))
    if err != nil {
        panic(err)
    }

    // Bases for the diversified keys
    appinfo.acl_read_base, err = hex.DecodeString(keymap["acl_read_key"].(string))
    if err != nil {
        panic(err)
    }
    appinfo.acl_write_base, err = hex.DecodeString(keymap["acl_write_key"].(string))
    if err != nil {
        panic(err)
    }

}

func recalculate_diversified_keys(realuid []byte) error {
    acl_read_bytes, err := keydiversification.AES128(appinfo.acl_read_base[:], appinfo.aidbytes[:], realuid[:], appinfo.sysid[:])
    if err != nil {
        return err
    }
    acl_write_bytes, err := keydiversification.AES128(appinfo.acl_write_base[:], appinfo.aidbytes[:], realuid[:], appinfo.sysid[:])
    if err != nil {
        return err
    }
    keychain.acl_read_key = helpers.Bytes2aeskey(acl_read_bytes)
    keychain.acl_write_key = helpers.Bytes2aeskey(acl_write_bytes)
    return nil
}

func update_acl_file(desfiretag *freefare.DESFireTag, newdata *[]byte) error {
    fmt.Print("Re-auth with ACL write key, ")
    err := desfiretag.Authenticate(keychain.acl_write_key_id,*keychain.acl_write_key)
    if err != nil {
        return err
    }
    fmt.Println("Done")

    fmt.Print("Overwriting ACL data file, ")
    byteswritten, err := desfiretag.WriteData(appinfo.acl_file_id, 0, *newdata)
    if err != nil {
        return err
    }
    if (byteswritten < 8) {
        fmt.Println(fmt.Sprintf("WARNING: WriteData wrote %d bytes, 8 expected", byteswritten))
    }
    fmt.Println("Done")
    return nil
}

func check_revoked(desfiretag *freefare.DESFireTag, c *sqlite3.Conn, realuid_str string) (bool, error) {
    revoked_found := false
    sql := "SELECT rowid FROM revoked where uid=?"
    for s, err := c.Query(sql, realuid_str); err == nil; err = s.Next() {
        revoked_found = true
        var rowid int64
        s.Scan(&rowid)     // Assigns 1st column to rowid, the rest to row
        fmt.Println(fmt.Sprintf("WARNING: Found REVOKED key %s on row %d", realuid_str, rowid))

        // TODO: Publish a ZMQ message or something

        // Null the ACL file on card
        nullaclbytes := make([]byte, 8)
        err := update_acl_file(desfiretag, &nullaclbytes)
        if err != nil {
            return revoked_found, err
        }
    }
    return revoked_found, nil
}

func read_and_parse_acl_file(desfiretag *freefare.DESFireTag) (uint64, error) {
    fmt.Print("Re-auth with ACL read key, ")
    err := desfiretag.Authenticate(keychain.acl_read_key_id,*keychain.acl_read_key)
    if err != nil {
        return 0, err
    }
    fmt.Println("Done")

    aclbytes := make([]byte, 8)
    fmt.Print("Reading ACL data file, ")
    bytesread, err := desfiretag.ReadData(appinfo.acl_file_id, 0, aclbytes)
    if err != nil {
        return 0, err
    }
    if (bytesread < 8) {
        fmt.Println(fmt.Sprintf("WARNING: ReadData read %d bytes, 8 expected", bytesread))
    }
    acl, n := binary.Uvarint(aclbytes)
    if n <= 0 {
        return 0, errors.New(fmt.Sprintf("ERROR: binary.Uvarint returned %d, skipping tag", n))
    }
    fmt.Println("Done")
    return acl, nil
}

func get_db_acl(desfiretag *freefare.DESFireTag, c *sqlite3.Conn, realuid_str string) (uint64, error) {
    sql := "SELECT rowid,acl FROM keys where uid=?"
    for s, err := c.Query(sql, realuid_str); err == nil; err = s.Next() {
        var rowid int64
        row := make(sqlite3.RowMap)
        s.Scan(&rowid, row)     // Assigns 1st column to rowid, the rest to row
        return uint64(row["acl"].(int64)), nil
    }
    return 0, errors.New(fmt.Sprintf("UID not found"))
}

func check_tag(desfiretag *freefare.DESFireTag, c *sqlite3.Conn, required_acl uint64) (bool, error) {
    const errlimit = 3
    var err error = nil
    var realuid_str string
    var realuid []byte
    acl := uint64(0)
    db_acl := uint64(0)
    revoked_found := false
    errcnt := 0
    connected := false

RETRY:
    if err != nil {
        // TODO: Retry only on RF-errors
        errcnt++
        if errcnt < errlimit {
            fmt.Println(fmt.Sprintf("failed (%s), retrying", err))
        }
        fmt.Println(fmt.Sprintf("failed (%s), retry-count exceeded, skipping tag", err))
        goto FAIL
    }
    if connected {
        _ = desfiretag.Disconnect()
    }

    // Connect to this tag
    fmt.Print(fmt.Sprintf("Connecting to %s, ", desfiretag.UID()))
    err = desfiretag.Connect()
    if err != nil {
        goto RETRY
    }
    fmt.Println("done")
    connected = true

    fmt.Print(fmt.Sprintf("Selecting application %d, ", appinfo.aid.Aid()))
    err = desfiretag.SelectApplication(appinfo.aid);
    if err != nil {
        goto RETRY
    }
    fmt.Println("Done")

    fmt.Print("Authenticating, ")
    err = desfiretag.Authenticate(keychain.uid_read_key_id,*keychain.uid_read_key)
    if err != nil {
        goto RETRY
    }
    fmt.Println("Done")

    // Get card real UID
    realuid_str, err = desfiretag.CardUID()
    if err != nil {
        // TODO: Retry only on RF-errors
        goto RETRY
    }
    realuid, err = hex.DecodeString(realuid_str)
    if err != nil {
        fmt.Println(fmt.Sprintf("ERROR: Failed to parse real UID (%s), skipping tag", err))
        goto FAIL
    }
    fmt.Println("Got real UID:", hex.EncodeToString(realuid));

    // Calculate the diversified keys
    err = recalculate_diversified_keys(realuid[:])
    if err != nil {
        fmt.Println(fmt.Sprintf("ERROR: Failed to get diversified ACL keys (%s), skipping tag", err))
        goto FAIL
    }

    // Check for revoked key
    revoked_found, err = check_revoked(desfiretag, c, realuid_str)
    if err != nil {
        fmt.Println(fmt.Sprintf("check_revoked returned err (%s)", err))
        revoked_found = true
    }
    if revoked_found {
        goto FAIL
    }

    acl, err = read_and_parse_acl_file(desfiretag)
    if err != nil {
        goto RETRY
    }
    //fmt.Println("DEBUG: acl:", acl)

    // Get (possibly updated) ACL from DB, if returns error then UID is not known
    db_acl, err = get_db_acl(desfiretag, c, realuid_str)
    if err != nil {
        // No match
        fmt.Println(fmt.Sprintf("WARNING: key %s, not found in DB", realuid_str))
        // TODO: Should we null the ACL file just in case, because any key that is personalized but not either valid or revoked is in a weird limbo
        goto FAIL
    }

    // Check for ACL update
    if (acl != db_acl) {
        fmt.Println(fmt.Sprintf("NOTICE: card ACL (%x) does not match DB (%x), ", acl, db_acl))

        // Update the ACL file on card
        newaclbytes := make([]byte, 8)
        n := binary.PutUvarint(newaclbytes, db_acl)
        if (n < 0) {
            fmt.Println(fmt.Sprintf("binary.PutUvarint returned %d, skipping tag", n))
            goto FAIL
        }
        err := update_acl_file(desfiretag, &newaclbytes)
        if err != nil {
            goto RETRY
        }
    }

    // Now check the ACL match
    if (db_acl & required_acl) == 0 {
        fmt.Println(fmt.Sprintf("NOTICE: Found valid key %s, but ACL (%x) not granted", realuid_str, required_acl))
        // TODO: Publish a ZMQ message or something
        goto FAIL
    }

    fmt.Println(fmt.Sprintf("SUCCESS: Access granted to %s with ACL (%x)", realuid_str, db_acl))
    return true, nil
FAIL:
    if connected {
        _ = desfiretag.Disconnect()
    }
    return false, err
}

func main() {
    // TODO: configure this somewhere
    required_acl := uint64(1)

    init_appinfo()

    gpiomap, err := helpers.LoadYAMLFile("gpio.yaml")
    if err != nil {
        panic(err)
    }

    // Get cursor
    c, err := sqlite3.Open("keys.db")
    if err != nil {
        panic(err);
    }

    // Open NFC device
    d, err := nfc.Open("");
    if err != nil {
        panic(err);
    }

    // Start heartbeat goroutine
    go heartbeat()

    // Get open GPIO pins for our outputs
	green_led, err := gpio.OpenPin(gpiomap["green_led"].(map[interface{}]interface{})["pin"].(int), gpio.ModeOutput)
	if err != nil {
		fmt.Printf("err opening green_led! %s\n", err)
		return
	}
	red_led, err := gpio.OpenPin(gpiomap["red_led"].(map[interface{}]interface{})["pin"].(int), gpio.ModeOutput)
	if err != nil {
		fmt.Printf("err opening green_led! %s\n", err)
		return
	}
	relay, err := gpio.OpenPin(gpiomap["relay"].(map[interface{}]interface{})["pin"].(int), gpio.ModeOutput)
	if err != nil {
		fmt.Printf("err opening relay! %s\n", err)
		return
	}
	// turn the leds off on exit
	exit_ch := make(chan os.Signal, 1)
	signal.Notify(exit_ch, os.Interrupt)
	signal.Notify(exit_ch, os.Kill)
	go func() {
		for _ = range exit_ch {
			fmt.Printf("\nClearing and unexporting the pins.\n")
			go clear_and_close(green_led)
			go clear_and_close(red_led)
			go clear_and_close(relay)
			os.Exit(0)
		}
	}()

    fmt.Println("Starting mainloop")
    // mainloop
    for {
        // Poll for tags
        var tags []freefare.Tag
        for {
            tags, err = freefare.GetTags(d);
            if err != nil {
                // TODO: Probably should not panic here
                panic(err)
            }
            if len(tags) > 0 {
                break
            }
            time.Sleep(100 * time.Millisecond)
            //fmt.Println("...polling")
        }

        valid_found := false
        for i := 0; i < len(tags); i++ {
            tag := tags[i]
            if (tag.Type() != freefare.DESFire) {
                fmt.Println(fmt.Sprintf("Non-DESFire tag %s skipped", tag.UID()))
                continue
            }
            desfiretag := tag.(freefare.DESFireTag)

            tag_valid, err := check_tag(&desfiretag, c, required_acl)
            if err != nil {
                continue
            }
            if tag_valid {
                valid_found = true
            }
        }
        if !valid_found {
            fmt.Println("Access DENIED")
            go pulse_gpio(red_led, gpiomap["red_led"].(map[interface{}]interface{})["time"].(int))
        } else {
            go pulse_gpio(green_led, gpiomap["green_led"].(map[interface{}]interface{})["time"].(int))
            go pulse_gpio(relay, gpiomap["relay"].(map[interface{}]interface{})["time"].(int))
        }

        // Run GC at this time
        runtime.GC()
        // Wait a moment before continuing with fast polling
        time.Sleep(500 * time.Millisecond)

    }
}

<commit_msg>fix logic error<commit_after>package main

import (
    "fmt"
	"os"
	"os/signal"
	"runtime"
	"errors"
//    "strconv"
    "encoding/hex"
    "encoding/binary"
    "github.com/fuzxxl/nfc/2.0/nfc"
    "github.com/fuzxxl/freefare/0.3/freefare"
    "code.google.com/p/go-sqlite/go1/sqlite3"
    "time"
    "github.com/davecheney/gpio"
    "./keydiversification"
    "./helpers"
)

func heartbeat() {
    for {
        time.Sleep(2000 * time.Millisecond)
        fmt.Println("Dunka-dunk")
    }
}

func pulse_gpio(pin gpio.Pin, ms int) {
    pin.Set()
    time.Sleep(time.Duration(ms) * time.Millisecond)
    pin.Clear()
}

func clear_and_close(pin gpio.Pin) {
    pin.Clear()
    pin.Close()
}

// Use structs to pass data around so I can refactor
type AppInfo struct {
    aid freefare.DESFireAid
    aidbytes []byte
    sysid []byte
    acl_read_base []byte
    acl_write_base []byte
    acl_file_id byte
}

type KeyChain struct {
    uid_read_key_id byte
    acl_read_key_id byte
    acl_write_key_id byte

    uid_read_key *freefare.DESFireKey
    acl_read_key *freefare.DESFireKey
    acl_write_key *freefare.DESFireKey
}

var (
    keychain = KeyChain{}
    appinfo = AppInfo{}
)

func init_appinfo() {
    keymap, err := helpers.LoadYAMLFile("keys.yaml")
    if err != nil {
        panic(err)
    }

    appmap, err := helpers.LoadYAMLFile("apps.yaml")
    if err != nil {
        panic(err)
    }

    // Application-id
    appinfo.aid, err = helpers.String2aid(appmap["hacklab_acl"].(map[interface{}]interface{})["aid"].(string))
    if err != nil {
        panic(err)
    }

    // Needed for diversification
    appinfo.aidbytes = helpers.Aid2bytes(appinfo.aid)
    appinfo.sysid, err = hex.DecodeString(appmap["hacklab_acl"].(map[interface{}]interface{})["sysid"].(string))
    if err != nil {
        panic(err)
    }

    appinfo.acl_file_id, err = helpers.String2byte(appmap["hacklab_acl"].(map[interface{}]interface{})["acl_file_id"].(string))
    if err != nil {
        panic(err)
    }

    // Key id numbers from config
    keychain.uid_read_key_id, err = helpers.String2byte(appmap["hacklab_acl"].(map[interface{}]interface{})["uid_read_key_id"].(string))
    if err != nil {
        panic(err)
    }
    keychain.acl_read_key_id, err = helpers.String2byte(appmap["hacklab_acl"].(map[interface{}]interface{})["acl_read_key_id"].(string))
    if err != nil {
        panic(err)
    }
    keychain.acl_write_key_id, err = helpers.String2byte(appmap["hacklab_acl"].(map[interface{}]interface{})["acl_write_key_id"].(string))
    if err != nil {
        panic(err)
    }

    // The static app key to read UID
    keychain.uid_read_key, err = helpers.String2aeskey(keymap["uid_read_key"].(string))
    if err != nil {
        panic(err)
    }

    // Bases for the diversified keys
    appinfo.acl_read_base, err = hex.DecodeString(keymap["acl_read_key"].(string))
    if err != nil {
        panic(err)
    }
    appinfo.acl_write_base, err = hex.DecodeString(keymap["acl_write_key"].(string))
    if err != nil {
        panic(err)
    }

}

func recalculate_diversified_keys(realuid []byte) error {
    acl_read_bytes, err := keydiversification.AES128(appinfo.acl_read_base[:], appinfo.aidbytes[:], realuid[:], appinfo.sysid[:])
    if err != nil {
        return err
    }
    acl_write_bytes, err := keydiversification.AES128(appinfo.acl_write_base[:], appinfo.aidbytes[:], realuid[:], appinfo.sysid[:])
    if err != nil {
        return err
    }
    keychain.acl_read_key = helpers.Bytes2aeskey(acl_read_bytes)
    keychain.acl_write_key = helpers.Bytes2aeskey(acl_write_bytes)
    return nil
}

func update_acl_file(desfiretag *freefare.DESFireTag, newdata *[]byte) error {
    fmt.Print("Re-auth with ACL write key, ")
    err := desfiretag.Authenticate(keychain.acl_write_key_id,*keychain.acl_write_key)
    if err != nil {
        return err
    }
    fmt.Println("Done")

    fmt.Print("Overwriting ACL data file, ")
    byteswritten, err := desfiretag.WriteData(appinfo.acl_file_id, 0, *newdata)
    if err != nil {
        return err
    }
    if (byteswritten < 8) {
        fmt.Println(fmt.Sprintf("WARNING: WriteData wrote %d bytes, 8 expected", byteswritten))
    }
    fmt.Println("Done")
    return nil
}

func check_revoked(desfiretag *freefare.DESFireTag, c *sqlite3.Conn, realuid_str string) (bool, error) {
    revoked_found := false
    sql := "SELECT rowid FROM revoked where uid=?"
    for s, err := c.Query(sql, realuid_str); err == nil; err = s.Next() {
        revoked_found = true
        var rowid int64
        s.Scan(&rowid)     // Assigns 1st column to rowid, the rest to row
        fmt.Println(fmt.Sprintf("WARNING: Found REVOKED key %s on row %d", realuid_str, rowid))

        // TODO: Publish a ZMQ message or something

        // Null the ACL file on card
        nullaclbytes := make([]byte, 8)
        err := update_acl_file(desfiretag, &nullaclbytes)
        if err != nil {
            return revoked_found, err
        }
    }
    return revoked_found, nil
}

func read_and_parse_acl_file(desfiretag *freefare.DESFireTag) (uint64, error) {
    fmt.Print("Re-auth with ACL read key, ")
    err := desfiretag.Authenticate(keychain.acl_read_key_id,*keychain.acl_read_key)
    if err != nil {
        return 0, err
    }
    fmt.Println("Done")

    aclbytes := make([]byte, 8)
    fmt.Print("Reading ACL data file, ")
    bytesread, err := desfiretag.ReadData(appinfo.acl_file_id, 0, aclbytes)
    if err != nil {
        return 0, err
    }
    if (bytesread < 8) {
        fmt.Println(fmt.Sprintf("WARNING: ReadData read %d bytes, 8 expected", bytesread))
    }
    acl, n := binary.Uvarint(aclbytes)
    if n <= 0 {
        return 0, errors.New(fmt.Sprintf("ERROR: binary.Uvarint returned %d, skipping tag", n))
    }
    fmt.Println("Done")
    return acl, nil
}

func get_db_acl(desfiretag *freefare.DESFireTag, c *sqlite3.Conn, realuid_str string) (uint64, error) {
    sql := "SELECT rowid,acl FROM keys where uid=?"
    for s, err := c.Query(sql, realuid_str); err == nil; err = s.Next() {
        var rowid int64
        row := make(sqlite3.RowMap)
        s.Scan(&rowid, row)     // Assigns 1st column to rowid, the rest to row
        return uint64(row["acl"].(int64)), nil
    }
    return 0, errors.New(fmt.Sprintf("UID not found"))
}

func check_tag(desfiretag *freefare.DESFireTag, c *sqlite3.Conn, required_acl uint64) (bool, error) {
    const errlimit = 3
    var err error = nil
    var realuid_str string
    var realuid []byte
    acl := uint64(0)
    db_acl := uint64(0)
    revoked_found := false
    errcnt := 0
    connected := false

RETRY:
    if err != nil {
        // TODO: Retry only on RF-errors
        errcnt++
        if errcnt > errlimit {
            fmt.Println(fmt.Sprintf("failed (%s), retrying", err))
        }
        fmt.Println(fmt.Sprintf("failed (%s), retry-count exceeded, skipping tag", err))
        goto FAIL
    }
    if connected {
        _ = desfiretag.Disconnect()
    }

    // Connect to this tag
    fmt.Print(fmt.Sprintf("Connecting to %s, ", desfiretag.UID()))
    err = desfiretag.Connect()
    if err != nil {
        goto RETRY
    }
    fmt.Println("done")
    connected = true

    fmt.Print(fmt.Sprintf("Selecting application %d, ", appinfo.aid.Aid()))
    err = desfiretag.SelectApplication(appinfo.aid);
    if err != nil {
        goto RETRY
    }
    fmt.Println("Done")

    fmt.Print("Authenticating, ")
    err = desfiretag.Authenticate(keychain.uid_read_key_id,*keychain.uid_read_key)
    if err != nil {
        goto RETRY
    }
    fmt.Println("Done")

    // Get card real UID
    realuid_str, err = desfiretag.CardUID()
    if err != nil {
        // TODO: Retry only on RF-errors
        goto RETRY
    }
    realuid, err = hex.DecodeString(realuid_str)
    if err != nil {
        fmt.Println(fmt.Sprintf("ERROR: Failed to parse real UID (%s), skipping tag", err))
        goto FAIL
    }
    fmt.Println("Got real UID:", hex.EncodeToString(realuid));

    // Calculate the diversified keys
    err = recalculate_diversified_keys(realuid[:])
    if err != nil {
        fmt.Println(fmt.Sprintf("ERROR: Failed to get diversified ACL keys (%s), skipping tag", err))
        goto FAIL
    }

    // Check for revoked key
    revoked_found, err = check_revoked(desfiretag, c, realuid_str)
    if err != nil {
        fmt.Println(fmt.Sprintf("check_revoked returned err (%s)", err))
        revoked_found = true
    }
    if revoked_found {
        goto FAIL
    }

    acl, err = read_and_parse_acl_file(desfiretag)
    if err != nil {
        goto RETRY
    }
    //fmt.Println("DEBUG: acl:", acl)

    // Get (possibly updated) ACL from DB, if returns error then UID is not known
    db_acl, err = get_db_acl(desfiretag, c, realuid_str)
    if err != nil {
        // No match
        fmt.Println(fmt.Sprintf("WARNING: key %s, not found in DB", realuid_str))
        // TODO: Should we null the ACL file just in case, because any key that is personalized but not either valid or revoked is in a weird limbo
        goto FAIL
    }

    // Check for ACL update
    if (acl != db_acl) {
        fmt.Println(fmt.Sprintf("NOTICE: card ACL (%x) does not match DB (%x), ", acl, db_acl))

        // Update the ACL file on card
        newaclbytes := make([]byte, 8)
        n := binary.PutUvarint(newaclbytes, db_acl)
        if (n < 0) {
            fmt.Println(fmt.Sprintf("binary.PutUvarint returned %d, skipping tag", n))
            goto FAIL
        }
        err := update_acl_file(desfiretag, &newaclbytes)
        if err != nil {
            goto RETRY
        }
    }

    // Now check the ACL match
    if (db_acl & required_acl) == 0 {
        fmt.Println(fmt.Sprintf("NOTICE: Found valid key %s, but ACL (%x) not granted", realuid_str, required_acl))
        // TODO: Publish a ZMQ message or something
        goto FAIL
    }

    fmt.Println(fmt.Sprintf("SUCCESS: Access granted to %s with ACL (%x)", realuid_str, db_acl))
    return true, nil
FAIL:
    if connected {
        _ = desfiretag.Disconnect()
    }
    return false, err
}

func main() {
    // TODO: configure this somewhere
    required_acl := uint64(1)

    init_appinfo()

    gpiomap, err := helpers.LoadYAMLFile("gpio.yaml")
    if err != nil {
        panic(err)
    }

    // Get cursor
    c, err := sqlite3.Open("keys.db")
    if err != nil {
        panic(err);
    }

    // Open NFC device
    d, err := nfc.Open("");
    if err != nil {
        panic(err);
    }

    // Start heartbeat goroutine
    go heartbeat()

    // Get open GPIO pins for our outputs
	green_led, err := gpio.OpenPin(gpiomap["green_led"].(map[interface{}]interface{})["pin"].(int), gpio.ModeOutput)
	if err != nil {
		fmt.Printf("err opening green_led! %s\n", err)
		return
	}
	red_led, err := gpio.OpenPin(gpiomap["red_led"].(map[interface{}]interface{})["pin"].(int), gpio.ModeOutput)
	if err != nil {
		fmt.Printf("err opening green_led! %s\n", err)
		return
	}
	relay, err := gpio.OpenPin(gpiomap["relay"].(map[interface{}]interface{})["pin"].(int), gpio.ModeOutput)
	if err != nil {
		fmt.Printf("err opening relay! %s\n", err)
		return
	}
	// turn the leds off on exit
	exit_ch := make(chan os.Signal, 1)
	signal.Notify(exit_ch, os.Interrupt)
	signal.Notify(exit_ch, os.Kill)
	go func() {
		for _ = range exit_ch {
			fmt.Printf("\nClearing and unexporting the pins.\n")
			go clear_and_close(green_led)
			go clear_and_close(red_led)
			go clear_and_close(relay)
			os.Exit(0)
		}
	}()

    fmt.Println("Starting mainloop")
    // mainloop
    for {
        // Poll for tags
        var tags []freefare.Tag
        for {
            tags, err = freefare.GetTags(d);
            if err != nil {
                // TODO: Probably should not panic here
                panic(err)
            }
            if len(tags) > 0 {
                break
            }
            time.Sleep(100 * time.Millisecond)
            //fmt.Println("...polling")
        }

        valid_found := false
        for i := 0; i < len(tags); i++ {
            tag := tags[i]
            if (tag.Type() != freefare.DESFire) {
                fmt.Println(fmt.Sprintf("Non-DESFire tag %s skipped", tag.UID()))
                continue
            }
            desfiretag := tag.(freefare.DESFireTag)

            tag_valid, err := check_tag(&desfiretag, c, required_acl)
            if err != nil {
                continue
            }
            if tag_valid {
                valid_found = true
            }
        }
        if !valid_found {
            fmt.Println("Access DENIED")
            go pulse_gpio(red_led, gpiomap["red_led"].(map[interface{}]interface{})["time"].(int))
        } else {
            go pulse_gpio(green_led, gpiomap["green_led"].(map[interface{}]interface{})["time"].(int))
            go pulse_gpio(relay, gpiomap["relay"].(map[interface{}]interface{})["time"].(int))
        }

        // Run GC at this time
        runtime.GC()
        // Wait a moment before continuing with fast polling
        time.Sleep(500 * time.Millisecond)

    }
}

<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"github.com/gmacd/container/set"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

var (
	srcRoot string
)

func main() {
	fmt.Println("..chanzero..")

	// TODO Specfiy output folder
	flag.StringVar(&srcRoot, "src", "", "Path to root src file for site to build.")
	flag.Parse()

	if _, err := os.Stat(srcRoot); err != nil {
		fmt.Printf("Couldn't open file \"%v\" (%v)\n", srcRoot, err.Error())
		os.Exit(-1)
	}

	fmt.Println("Building site with root", srcRoot)

	exportSite(srcRoot)
}

type page struct {
	srcPath    string
	html       []byte
	linkedUrls []string
}

func NewPage(path string) *page {
	return &page{path, nil, make([]string, 0)}
}

func (page *page) AddLink(url string) {
	page.linkedUrls = append(page.linkedUrls, url)
}

func replaceExtension(path, newExtention string) string {
	basePath := strings.TrimSuffix(path, filepath.Ext(path))
	return basePath + "." + newExtention
}

// Wrapped HtmlRenderer which gathers all links in markdown
// TODO Write post about wrapping/oeverriding functionality in Go.
type LinkGatheringHtmlRenderer struct {
	*blackfriday.Html

	page *page
}

func NewLinkGatheringHtmlRenderer(renderer blackfriday.Renderer, page *page) *LinkGatheringHtmlRenderer {
	return &LinkGatheringHtmlRenderer{renderer.(*blackfriday.Html), page}
}

func (html *LinkGatheringHtmlRenderer) AutoLink(out *bytes.Buffer, link []byte, kind int) {
	html.page.AddLink(string(link))
	html.Html.AutoLink(out, link, kind)
}

func (html *LinkGatheringHtmlRenderer) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) {
	html.page.AddLink(string(link))
	html.Html.Link(out, link, title, content)
}

// Given a single root page, load the page and follow all local src links,
// loading each page recursively, linked to the root.
func importPage(path string) *page {
	page := NewPage(path)

	mdsrc, err := ioutil.ReadFile(path)
	if err != nil {
		fmt.Printf("Couldn't load \"%v\": %v\n", path, err.Error())
	}

	// Set up a 'common' converter
	htmlFlags := 0
	htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS
	htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS
	htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES
	renderer := blackfriday.HtmlRenderer(htmlFlags, "", "")

	linkGatheringRenderer := NewLinkGatheringHtmlRenderer(renderer, page)

	extensions := 0
	//extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS
	extensions |= blackfriday.EXTENSION_TABLES
	extensions |= blackfriday.EXTENSION_FENCED_CODE
	extensions |= blackfriday.EXTENSION_AUTOLINK
	extensions |= blackfriday.EXTENSION_STRIKETHROUGH
	extensions |= blackfriday.EXTENSION_SPACE_HEADERS
	extensions |= blackfriday.EXTENSION_HEADER_IDS

	page.html = blackfriday.Markdown(mdsrc, linkGatheringRenderer, extensions)

	return page
}

// Given a root page, export the entire site.
func exportSite(rootSrc string) {
	exportedPages := set.NewSetOfValues()
	exportPage(rootSrc, exportedPages)
}

func exportPage(srcPath string, previouslyExportedPaths *set.Set) {
	if !previouslyExportedPaths.Contains(srcPath) {
		fmt.Println(" Exporting page", srcPath)

		previouslyExportedPaths.Add(srcPath)

		page := importPage(srcRoot)

		destPath := replaceExtension(srcPath, "html")
		ioutil.WriteFile(destPath, page.html, 0644)

		// Export referenced pages
		for _, linkUrl := range page.linkedUrls {
			linkSrc := replaceExtension(linkUrl, "md")

			exportPage(linkSrc, previouslyExportedPaths)
		}
	}
}
<commit_msg>Exports!<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"github.com/gmacd/container/set"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

var (
	srcRoot string
)

func main() {
	fmt.Println("..chanzero..")

	// TODO Specfiy output folder
	flag.StringVar(&srcRoot, "src", "", "Path to root src file for site to build.")
	flag.Parse()

	if _, err := os.Stat(srcRoot); err != nil {
		fmt.Printf("Couldn't open file \"%v\" (%v)\n", srcRoot, err.Error())
		os.Exit(-1)
	}

	fmt.Println("Building site with root", srcRoot)

	exportSite(srcRoot)
}

type page struct {
	srcPath    string
	html       []byte
	linkedUrls []string
}

func NewPage(path string) *page {
	return &page{path, nil, make([]string, 0)}
}

func (page *page) AddLink(url string) {
	page.linkedUrls = append(page.linkedUrls, url)
}

func replaceExtension(path, newExtention string) string {
	basePath := strings.TrimSuffix(path, filepath.Ext(path))
	return basePath + "." + newExtention
}

// Wrapped HtmlRenderer which gathers all links in markdown
// TODO Write post about wrapping/oeverriding functionality in Go.
type LinkGatheringHtmlRenderer struct {
	*blackfriday.Html

	page *page
}

func NewLinkGatheringHtmlRenderer(renderer blackfriday.Renderer, page *page) *LinkGatheringHtmlRenderer {
	return &LinkGatheringHtmlRenderer{renderer.(*blackfriday.Html), page}
}

func (html *LinkGatheringHtmlRenderer) AutoLink(out *bytes.Buffer, link []byte, kind int) {
	html.page.AddLink(string(link))
	html.Html.AutoLink(out, link, kind)
}

func (html *LinkGatheringHtmlRenderer) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) {
	html.page.AddLink(string(link))
	html.Html.Link(out, link, title, content)
}

// Given a single root page, load the page and follow all local src links,
// loading each page recursively, linked to the root.
func importPage(path string) *page {
	page := NewPage(path)

	mdsrc, err := ioutil.ReadFile(path)
	if err != nil {
		fmt.Printf("Couldn't load \"%v\": %v\n", path, err.Error())
	}

	// Set up a 'common' converter
	htmlFlags := 0
	htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS
	htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS
	htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES
	renderer := blackfriday.HtmlRenderer(htmlFlags, "", "")

	linkGatheringRenderer := NewLinkGatheringHtmlRenderer(renderer, page)

	extensions := 0
	//extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS
	extensions |= blackfriday.EXTENSION_TABLES
	extensions |= blackfriday.EXTENSION_FENCED_CODE
	extensions |= blackfriday.EXTENSION_AUTOLINK
	extensions |= blackfriday.EXTENSION_STRIKETHROUGH
	extensions |= blackfriday.EXTENSION_SPACE_HEADERS
	extensions |= blackfriday.EXTENSION_HEADER_IDS

	page.html = blackfriday.Markdown(mdsrc, linkGatheringRenderer, extensions)

	return page
}

// Given a root page, export the entire site.
func exportSite(rootSrc string) {
	rootFile := filepath.Base(rootSrc)
	rootSrcPath := filepath.Dir(rootSrc)
	destSrcPath := filepath.Dir(rootSrcPath)

	exportedPages := set.NewSetOfValues()
	exportPage(rootFile, rootSrcPath, destSrcPath, exportedPages)
}

func exportPage(pageSrcPath, rootSrcPath, destSrcPath string, previouslyExportedPaths *set.Set) {
	if !previouslyExportedPaths.Contains(pageSrcPath) {
		previouslyExportedPaths.Add(pageSrcPath)

		fullSrcPath := rootSrcPath + "/" + pageSrcPath
		fullDestPath := destSrcPath + "/" + replaceExtension(pageSrcPath, "html")
		fmt.Println(" Exporting page  src:", fullSrcPath)
		fmt.Println("                dest:", fullDestPath)

		page := importPage(fullSrcPath)

		// Ensure destination exists
		os.MkdirAll(filepath.Dir(fullDestPath), 0755)

		ioutil.WriteFile(fullDestPath, page.html, 0644)

		// Export referenced pages
		for _, linkUrl := range page.linkedUrls {
			linkSrc := replaceExtension(linkUrl, "md")

			exportPage(linkSrc, rootSrcPath, destSrcPath, previouslyExportedPaths)
		}
	}
}
<|endoftext|>


<commit_before>// The goldigen binary
// See https://github.com/fgrosse/goldi#the-goldigen-binary
package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"gopkg.in/alecthomas/kingpin.v2"
)

const Version = "0.9.4"

var (
	app = kingpin.New("goldigen", "The goldi dependency injection container generator.\n\nSee https://github.com/fgrosse/goldi for further information.")

	inputFile     = app.Flag("in", "The input yaml file to generate type definitions from").Required().File()
	outputPath    = app.Flag("out", "The output file to save the generated go code").String()
	packageName   = app.Flag("package", "The name of the genarated package").String()
	functionName  = app.Flag("function", fmt.Sprintf("The name of the generated function that must be called to register your types (default %q)", DefaultFunctionName)).String()
	noInteraction = app.Flag("nointeraction", "Do not ask for any user input").Default("false").Bool()
	verbose       = app.Flag("verbose", "Print verbose output").Default("false").Bool()
	overwrite     = app.Flag("overwrite", "Overwrite any existing files").Default("false").Short('y').Bool()
	forceStdOut   = app.Flag("echo", "Echo the generated code to std out even if a output path is given").Default("false").Bool()
)

func main() {
	defer panicHandler()
	app.Version(Version)

	kingpin.MustParse(app.Parse(os.Args[1:]))

	inputPath, _ := filepath.Abs((*inputFile).Name())
	if *outputPath != "" {
		*outputPath, _ = filepath.Abs(*outputPath)
	}

	outputPackageName := determineOutputPackageName()
	config := NewConfig(outputPackageName, *functionName, inputPath, *outputPath)
	gen := New(config)
	output := &bytes.Buffer{}

	if *verbose {
		gen.Debug = true
	}

	logVerboseGeneratorConfig(inputPath, outputPackageName)
	err := gen.Generate(*inputFile, output)
	if err != nil {
		log(err.Error())
		os.Exit(1)
	}

	if *outputPath == "" || *forceStdOut {
		logVerbose("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
		fmt.Println(output.String())
		return
	}

	writeOutputFile(output)
}

func panicHandler() {
	if r := recover(); r != nil {
		log("FATAL ERROR: %s", r)
		os.Exit(1)
	}
}

func determineOutputPackageName() string {
	outputPackageName := *packageName
	if outputPackageName != "" {
		return outputPackageName
	}

	goPathChecker := NewGoPathChecker(*verbose)
	outputPackageName = goPathChecker.PackageName(*outputPath)
	logVerbose("Package name for output path %q is %q", *outputPath, outputPackageName)

	if outputPackageName != "" {
		return outputPackageName
	}

	if *outputPath != "" {
		log("Could not determine the output package name for %q", *outputPath)
	}

	return ask("Output package name: ")
}

func ask(question string) string {
	if *noInteraction {
		os.Exit(1)
	}

	log(question)
	var answer string
	_, err := fmt.Scan(&answer)
	if err != nil {
		panic(err)
	}

	return strings.TrimSpace(answer)
}

func logVerboseGeneratorConfig(inputPath, outputPackageName string) {
	logVerbose("Generating output from file %q", inputPath)
	if *outputPath != "" {
		logVerbose("Output will be saved to %q", *outputPath)
	}

	if outputPackageName == "" {
		logVerbose("Output package name is empty")
	} else {
		logVerbose("Output package name is %q", outputPackageName)
	}
}

func logVerbose(message string, args ...interface{}) {
	if *verbose == false {
		return
	}

	log(message, args...)
}

func log(message string, args ...interface{}) {
	writer := os.Stdout
	if *outputPath == "" {
		// since we already output the generated code on stdout we print messages on stderr
		writer = os.Stderr
	}

	fmt.Fprintf(writer, message+"\n", args...)
}

func writeOutputFile(output *bytes.Buffer) {
	if _, err := os.Stat(*outputPath); err == nil {
		checkUserWantsToOverwriteFile()
	}

	err := ioutil.WriteFile(*outputPath, output.Bytes(), 0644)
	if err != nil {
		log("Error while writing output file: %s", err)
		os.Exit(1)
	}
	log("Successfully wrote %d bytes to %q", output.Len(), *outputPath)
}

func checkUserWantsToOverwriteFile() {
	if *overwrite {
		return
	}

	log("Output file %q does already exist.", *outputPath)
	answer := ask("Do you want me to overwrite that file? [yN] ")
	answer = strings.ToLower(answer)
	if answer == "" || answer == "n" {
		log("Output has NOT been saved")
		os.Exit(1)
	}
}

// EnableVerboseLog can be used when debugging the code in the tests
func EnableVerboseLog() {
	*verbose = true
}
<commit_msg>Bump version<commit_after>// The goldigen binary
// See https://github.com/fgrosse/goldi#the-goldigen-binary
package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"gopkg.in/alecthomas/kingpin.v2"
)

const Version = "0.9.5"

var (
	app = kingpin.New("goldigen", "The goldi dependency injection container generator.\n\nSee https://github.com/fgrosse/goldi for further information.")

	inputFile     = app.Flag("in", "The input yaml file to generate type definitions from").Required().File()
	outputPath    = app.Flag("out", "The output file to save the generated go code").String()
	packageName   = app.Flag("package", "The name of the genarated package").String()
	functionName  = app.Flag("function", fmt.Sprintf("The name of the generated function that must be called to register your types (default %q)", DefaultFunctionName)).String()
	noInteraction = app.Flag("nointeraction", "Do not ask for any user input").Default("false").Bool()
	verbose       = app.Flag("verbose", "Print verbose output").Default("false").Bool()
	overwrite     = app.Flag("overwrite", "Overwrite any existing files").Default("false").Short('y').Bool()
	forceStdOut   = app.Flag("echo", "Echo the generated code to std out even if a output path is given").Default("false").Bool()
)

func main() {
	defer panicHandler()
	app.Version(Version)

	kingpin.MustParse(app.Parse(os.Args[1:]))

	inputPath, _ := filepath.Abs((*inputFile).Name())
	if *outputPath != "" {
		*outputPath, _ = filepath.Abs(*outputPath)
	}

	outputPackageName := determineOutputPackageName()
	config := NewConfig(outputPackageName, *functionName, inputPath, *outputPath)
	gen := New(config)
	output := &bytes.Buffer{}

	if *verbose {
		gen.Debug = true
	}

	logVerboseGeneratorConfig(inputPath, outputPackageName)
	err := gen.Generate(*inputFile, output)
	if err != nil {
		log(err.Error())
		os.Exit(1)
	}

	if *outputPath == "" || *forceStdOut {
		logVerbose("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
		fmt.Println(output.String())
		return
	}

	writeOutputFile(output)
}

func panicHandler() {
	if r := recover(); r != nil {
		log("FATAL ERROR: %s", r)
		os.Exit(1)
	}
}

func determineOutputPackageName() string {
	outputPackageName := *packageName
	if outputPackageName != "" {
		return outputPackageName
	}

	goPathChecker := NewGoPathChecker(*verbose)
	outputPackageName = goPathChecker.PackageName(*outputPath)
	logVerbose("Package name for output path %q is %q", *outputPath, outputPackageName)

	if outputPackageName != "" {
		return outputPackageName
	}

	if *outputPath != "" {
		log("Could not determine the output package name for %q", *outputPath)
	}

	return ask("Output package name: ")
}

func ask(question string) string {
	if *noInteraction {
		os.Exit(1)
	}

	log(question)
	var answer string
	_, err := fmt.Scan(&answer)
	if err != nil {
		panic(err)
	}

	return strings.TrimSpace(answer)
}

func logVerboseGeneratorConfig(inputPath, outputPackageName string) {
	logVerbose("Generating output from file %q", inputPath)
	if *outputPath != "" {
		logVerbose("Output will be saved to %q", *outputPath)
	}

	if outputPackageName == "" {
		logVerbose("Output package name is empty")
	} else {
		logVerbose("Output package name is %q", outputPackageName)
	}
}

func logVerbose(message string, args ...interface{}) {
	if *verbose == false {
		return
	}

	log(message, args...)
}

func log(message string, args ...interface{}) {
	writer := os.Stdout
	if *outputPath == "" {
		// since we already output the generated code on stdout we print messages on stderr
		writer = os.Stderr
	}

	fmt.Fprintf(writer, message+"\n", args...)
}

func writeOutputFile(output *bytes.Buffer) {
	if _, err := os.Stat(*outputPath); err == nil {
		checkUserWantsToOverwriteFile()
	}

	err := ioutil.WriteFile(*outputPath, output.Bytes(), 0644)
	if err != nil {
		log("Error while writing output file: %s", err)
		os.Exit(1)
	}
	log("Successfully wrote %d bytes to %q", output.Len(), *outputPath)
}

func checkUserWantsToOverwriteFile() {
	if *overwrite {
		return
	}

	log("Output file %q does already exist.", *outputPath)
	answer := ask("Do you want me to overwrite that file? [yN] ")
	answer = strings.ToLower(answer)
	if answer == "" || answer == "n" {
		log("Output has NOT been saved")
		os.Exit(1)
	}
}

// EnableVerboseLog can be used when debugging the code in the tests
func EnableVerboseLog() {
	*verbose = true
}
<|endoftext|>


<commit_before>package google

import (
	"context"
	"fmt"
	"log"
	"net/http"

	"github.com/hashicorp/terraform/helper/logging"
	"github.com/hashicorp/terraform/helper/pathorcontents"
	"github.com/hashicorp/terraform/httpclient"
	"github.com/terraform-providers/terraform-provider-google/version"

	"golang.org/x/oauth2"
	googleoauth "golang.org/x/oauth2/google"
	appengine "google.golang.org/api/appengine/v1"
	"google.golang.org/api/bigquery/v2"
	"google.golang.org/api/cloudbilling/v1"
	"google.golang.org/api/cloudbuild/v1"
	"google.golang.org/api/cloudfunctions/v1"
	"google.golang.org/api/cloudiot/v1"
	"google.golang.org/api/cloudkms/v1"
	"google.golang.org/api/cloudresourcemanager/v1"
	resourceManagerV2Beta1 "google.golang.org/api/cloudresourcemanager/v2beta1"
	composer "google.golang.org/api/composer/v1beta1"
	computeBeta "google.golang.org/api/compute/v0.beta"
	"google.golang.org/api/compute/v1"
	"google.golang.org/api/container/v1"
	containerBeta "google.golang.org/api/container/v1beta1"
	dataflow "google.golang.org/api/dataflow/v1b3"
	"google.golang.org/api/dataproc/v1"
	"google.golang.org/api/dns/v1"
	dnsBeta "google.golang.org/api/dns/v1beta2"
	file "google.golang.org/api/file/v1beta1"
	"google.golang.org/api/iam/v1"
	cloudlogging "google.golang.org/api/logging/v2"
	"google.golang.org/api/pubsub/v1"
	redis "google.golang.org/api/redis/v1beta1"
	runtimeconfig "google.golang.org/api/runtimeconfig/v1beta1"
	"google.golang.org/api/servicemanagement/v1"
	"google.golang.org/api/serviceusage/v1"
	"google.golang.org/api/sourcerepo/v1"
	"google.golang.org/api/spanner/v1"
	sqladmin "google.golang.org/api/sqladmin/v1beta4"
	"google.golang.org/api/storage/v1"
	"google.golang.org/api/storagetransfer/v1"
)

// Config is the configuration structure used to instantiate the Google
// provider.
type Config struct {
	Credentials string
	AccessToken string
	Project     string
	Region      string
	Zone        string
	Scopes      []string

	client    *http.Client
	userAgent string

	tokenSource oauth2.TokenSource

	clientBilling                *cloudbilling.APIService
	clientBuild                  *cloudbuild.Service
	clientComposer               *composer.Service
	clientCompute                *compute.Service
	clientComputeBeta            *computeBeta.Service
	clientContainer              *container.Service
	clientContainerBeta          *containerBeta.Service
	clientDataproc               *dataproc.Service
	clientDataflow               *dataflow.Service
	clientDns                    *dns.Service
	clientDnsBeta                *dnsBeta.Service
	clientFilestore              *file.Service
	clientKms                    *cloudkms.Service
	clientLogging                *cloudlogging.Service
	clientPubsub                 *pubsub.Service
	clientRedis                  *redis.Service
	clientResourceManager        *cloudresourcemanager.Service
	clientResourceManagerV2Beta1 *resourceManagerV2Beta1.Service
	clientRuntimeconfig          *runtimeconfig.Service
	clientSpanner                *spanner.Service
	clientSourceRepo             *sourcerepo.Service
	clientStorage                *storage.Service
	clientSqlAdmin               *sqladmin.Service
	clientIAM                    *iam.Service
	clientServiceMan             *servicemanagement.APIService
	clientServiceUsage           *serviceusage.Service
	clientBigQuery               *bigquery.Service
	clientCloudFunctions         *cloudfunctions.Service
	clientCloudIoT               *cloudiot.Service
	clientAppEngine              *appengine.APIService
	clientStorageTransfer        *storagetransfer.Service

	bigtableClientFactory *BigtableClientFactory
}

var defaultClientScopes = []string{
	"https://www.googleapis.com/auth/compute",
	"https://www.googleapis.com/auth/cloud-platform",
	"https://www.googleapis.com/auth/ndev.clouddns.readwrite",
	"https://www.googleapis.com/auth/devstorage.full_control",
}

func (c *Config) loadAndValidate() error {
	if len(c.Scopes) == 0 {
		c.Scopes = defaultClientScopes
	}

	tokenSource, err := c.getTokenSource(c.Scopes)
	if err != nil {
		return err
	}
	c.tokenSource = tokenSource

	client := oauth2.NewClient(context.Background(), tokenSource)
	client.Transport = logging.NewTransport("Google", client.Transport)

	terraformVersion := httpclient.UserAgentString()
	providerVersion := fmt.Sprintf("terraform-provider-google/%s", version.ProviderVersion)
	terraformWebsite := "(+https://www.terraform.io)"
	userAgent := fmt.Sprintf("%s %s %s", terraformVersion, terraformWebsite, providerVersion)

	c.client = client
	c.userAgent = userAgent

	log.Printf("[INFO] Instantiating GCE client...")
	c.clientCompute, err = compute.New(client)
	if err != nil {
		return err
	}
	c.clientCompute.UserAgent = userAgent

	log.Printf("[INFO] Instantiating GCE Beta client...")
	c.clientComputeBeta, err = computeBeta.New(client)
	if err != nil {
		return err
	}
	c.clientComputeBeta.UserAgent = userAgent

	log.Printf("[INFO] Instantiating GKE client...")
	c.clientContainer, err = container.New(client)
	if err != nil {
		return err
	}
	c.clientContainer.UserAgent = userAgent

	log.Printf("[INFO] Instantiating GKE Beta client...")
	c.clientContainerBeta, err = containerBeta.New(client)
	if err != nil {
		return err
	}
	c.clientContainerBeta.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud DNS client...")
	c.clientDns, err = dns.New(client)
	if err != nil {
		return err
	}
	c.clientDns.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud DNS Beta client...")
	c.clientDnsBeta, err = dnsBeta.New(client)
	if err != nil {
		return err
	}
	c.clientDnsBeta.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud KMS Client...")
	c.clientKms, err = cloudkms.New(client)
	if err != nil {
		return err
	}
	c.clientKms.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Stackdriver Logging client...")
	c.clientLogging, err = cloudlogging.New(client)
	if err != nil {
		return err
	}
	c.clientLogging.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Storage Client...")
	c.clientStorage, err = storage.New(client)
	if err != nil {
		return err
	}
	c.clientStorage.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google SqlAdmin Client...")
	c.clientSqlAdmin, err = sqladmin.New(client)
	if err != nil {
		return err
	}
	c.clientSqlAdmin.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Pubsub Client...")
	c.clientPubsub, err = pubsub.New(client)
	if err != nil {
		return err
	}
	c.clientPubsub.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Dataflow Client...")
	c.clientDataflow, err = dataflow.New(client)
	if err != nil {
		return err
	}
	c.clientDataflow.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud Redis Client...")
	c.clientRedis, err = redis.New(client)
	if err != nil {
		return err
	}
	c.clientRedis.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud ResourceManager Client...")
	c.clientResourceManager, err = cloudresourcemanager.New(client)
	if err != nil {
		return err
	}
	c.clientResourceManager.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud ResourceManager V Client...")
	c.clientResourceManagerV2Beta1, err = resourceManagerV2Beta1.New(client)
	if err != nil {
		return err
	}
	c.clientResourceManagerV2Beta1.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud Runtimeconfig Client...")
	c.clientRuntimeconfig, err = runtimeconfig.New(client)
	if err != nil {
		return err
	}
	c.clientRuntimeconfig.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud IAM Client...")
	c.clientIAM, err = iam.New(client)
	if err != nil {
		return err
	}
	c.clientIAM.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud Service Management Client...")
	c.clientServiceMan, err = servicemanagement.New(client)
	if err != nil {
		return err
	}
	c.clientServiceMan.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud Service Usage Client...")
	c.clientServiceUsage, err = serviceusage.New(client)
	if err != nil {
		return err
	}
	c.clientServiceUsage.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud Billing Client...")
	c.clientBilling, err = cloudbilling.New(client)
	if err != nil {
		return err
	}
	c.clientBilling.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud Build Client...")
	c.clientBuild, err = cloudbuild.New(client)
	if err != nil {
		return err
	}
	c.clientBuild.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud BigQuery Client...")
	c.clientBigQuery, err = bigquery.New(client)
	if err != nil {
		return err
	}
	c.clientBigQuery.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud CloudFunctions Client...")
	c.clientCloudFunctions, err = cloudfunctions.New(client)
	if err != nil {
		return err
	}
	c.clientCloudFunctions.UserAgent = userAgent

	c.bigtableClientFactory = &BigtableClientFactory{
		UserAgent:   userAgent,
		TokenSource: tokenSource,
	}

	log.Printf("[INFO] Instantiating Google Cloud Source Repo Client...")
	c.clientSourceRepo, err = sourcerepo.New(client)
	if err != nil {
		return err
	}
	c.clientSourceRepo.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud Spanner Client...")
	c.clientSpanner, err = spanner.New(client)
	if err != nil {
		return err
	}
	c.clientSpanner.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud Dataproc Client...")
	c.clientDataproc, err = dataproc.New(client)
	if err != nil {
		return err
	}
	c.clientDataproc.UserAgent = userAgent

	c.clientFilestore, err = file.New(client)
	if err != nil {
		return err
	}
	c.clientFilestore.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud IoT Core Client...")
	c.clientCloudIoT, err = cloudiot.New(client)
	if err != nil {
		return err
	}
	c.clientCloudIoT.UserAgent = userAgent

	log.Printf("[INFO] Instantiating App Engine Client...")
	c.clientAppEngine, err = appengine.New(client)
	if err != nil {
		return err
	}
	c.clientAppEngine.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Cloud Composer Client...")
	c.clientComposer, err = composer.New(client)
	if err != nil {
		return err
	}
	c.clientComposer.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud Storage Transfer Client...")
	c.clientStorageTransfer, err = storagetransfer.New(client)
	if err != nil {
		return err
	}
	c.clientStorageTransfer.UserAgent = userAgent

	return nil
}

func (c *Config) getTokenSource(clientScopes []string) (oauth2.TokenSource, error) {
	if c.AccessToken != "" {
		contents, _, err := pathorcontents.Read(c.AccessToken)
		if err != nil {
			return nil, fmt.Errorf("Error loading access token: %s", err)
		}

		log.Printf("[INFO] Authenticating using configured Google JSON 'access_token'...")
		log.Printf("[INFO]   -- Scopes: %s", clientScopes)
		token := &oauth2.Token{AccessToken: contents}
		return oauth2.StaticTokenSource(token), nil
	}

	if c.Credentials != "" {
		contents, _, err := pathorcontents.Read(c.Credentials)
		if err != nil {
			return nil, fmt.Errorf("Error loading credentials: %s", err)
		}

		creds, err := googleoauth.CredentialsFromJSON(context.Background(), []byte(contents), clientScopes...)
		if err != nil {
			return nil, fmt.Errorf("Unable to parse credentials from '%s': %s", contents, err)
		}

		log.Printf("[INFO] Authenticating using configured Google JSON 'credentials'...")
		log.Printf("[INFO]   -- Scopes: %s", clientScopes)
		return creds.TokenSource, nil
	}

	log.Printf("[INFO] Authenticating using DefaultClient...")
	log.Printf("[INFO]   -- Scopes: %s", clientScopes)
	return googleoauth.DefaultTokenSource(context.Background(), clientScopes...)
}
<commit_msg>Expose config's load and validate method.<commit_after>package google

import (
	"context"
	"fmt"
	"log"
	"net/http"

	"github.com/hashicorp/terraform/helper/logging"
	"github.com/hashicorp/terraform/helper/pathorcontents"
	"github.com/hashicorp/terraform/httpclient"
	"github.com/terraform-providers/terraform-provider-google/version"

	"golang.org/x/oauth2"
	googleoauth "golang.org/x/oauth2/google"
	appengine "google.golang.org/api/appengine/v1"
	"google.golang.org/api/bigquery/v2"
	"google.golang.org/api/cloudbilling/v1"
	"google.golang.org/api/cloudbuild/v1"
	"google.golang.org/api/cloudfunctions/v1"
	"google.golang.org/api/cloudiot/v1"
	"google.golang.org/api/cloudkms/v1"
	"google.golang.org/api/cloudresourcemanager/v1"
	resourceManagerV2Beta1 "google.golang.org/api/cloudresourcemanager/v2beta1"
	composer "google.golang.org/api/composer/v1beta1"
	computeBeta "google.golang.org/api/compute/v0.beta"
	"google.golang.org/api/compute/v1"
	"google.golang.org/api/container/v1"
	containerBeta "google.golang.org/api/container/v1beta1"
	dataflow "google.golang.org/api/dataflow/v1b3"
	"google.golang.org/api/dataproc/v1"
	"google.golang.org/api/dns/v1"
	dnsBeta "google.golang.org/api/dns/v1beta2"
	file "google.golang.org/api/file/v1beta1"
	"google.golang.org/api/iam/v1"
	cloudlogging "google.golang.org/api/logging/v2"
	"google.golang.org/api/pubsub/v1"
	redis "google.golang.org/api/redis/v1beta1"
	runtimeconfig "google.golang.org/api/runtimeconfig/v1beta1"
	"google.golang.org/api/servicemanagement/v1"
	"google.golang.org/api/serviceusage/v1"
	"google.golang.org/api/sourcerepo/v1"
	"google.golang.org/api/spanner/v1"
	sqladmin "google.golang.org/api/sqladmin/v1beta4"
	"google.golang.org/api/storage/v1"
	"google.golang.org/api/storagetransfer/v1"
)

// Config is the configuration structure used to instantiate the Google
// provider.
type Config struct {
	Credentials string
	AccessToken string
	Project     string
	Region      string
	Zone        string
	Scopes      []string

	client    *http.Client
	userAgent string

	tokenSource oauth2.TokenSource

	clientBilling                *cloudbilling.APIService
	clientBuild                  *cloudbuild.Service
	clientComposer               *composer.Service
	clientCompute                *compute.Service
	clientComputeBeta            *computeBeta.Service
	clientContainer              *container.Service
	clientContainerBeta          *containerBeta.Service
	clientDataproc               *dataproc.Service
	clientDataflow               *dataflow.Service
	clientDns                    *dns.Service
	clientDnsBeta                *dnsBeta.Service
	clientFilestore              *file.Service
	clientKms                    *cloudkms.Service
	clientLogging                *cloudlogging.Service
	clientPubsub                 *pubsub.Service
	clientRedis                  *redis.Service
	clientResourceManager        *cloudresourcemanager.Service
	clientResourceManagerV2Beta1 *resourceManagerV2Beta1.Service
	clientRuntimeconfig          *runtimeconfig.Service
	clientSpanner                *spanner.Service
	clientSourceRepo             *sourcerepo.Service
	clientStorage                *storage.Service
	clientSqlAdmin               *sqladmin.Service
	clientIAM                    *iam.Service
	clientServiceMan             *servicemanagement.APIService
	clientServiceUsage           *serviceusage.Service
	clientBigQuery               *bigquery.Service
	clientCloudFunctions         *cloudfunctions.Service
	clientCloudIoT               *cloudiot.Service
	clientAppEngine              *appengine.APIService
	clientStorageTransfer        *storagetransfer.Service

	bigtableClientFactory *BigtableClientFactory
}

var defaultClientScopes = []string{
	"https://www.googleapis.com/auth/compute",
	"https://www.googleapis.com/auth/cloud-platform",
	"https://www.googleapis.com/auth/ndev.clouddns.readwrite",
	"https://www.googleapis.com/auth/devstorage.full_control",
}

func (c *Config) LoadAndValidate() error {
	if len(c.Scopes) == 0 {
		c.Scopes = defaultClientScopes
	}

	tokenSource, err := c.getTokenSource(c.Scopes)
	if err != nil {
		return err
	}
	c.tokenSource = tokenSource

	client := oauth2.NewClient(context.Background(), tokenSource)
	client.Transport = logging.NewTransport("Google", client.Transport)

	terraformVersion := httpclient.UserAgentString()
	providerVersion := fmt.Sprintf("terraform-provider-google/%s", version.ProviderVersion)
	terraformWebsite := "(+https://www.terraform.io)"
	userAgent := fmt.Sprintf("%s %s %s", terraformVersion, terraformWebsite, providerVersion)

	c.client = client
	c.userAgent = userAgent

	log.Printf("[INFO] Instantiating GCE client...")
	c.clientCompute, err = compute.New(client)
	if err != nil {
		return err
	}
	c.clientCompute.UserAgent = userAgent

	log.Printf("[INFO] Instantiating GCE Beta client...")
	c.clientComputeBeta, err = computeBeta.New(client)
	if err != nil {
		return err
	}
	c.clientComputeBeta.UserAgent = userAgent

	log.Printf("[INFO] Instantiating GKE client...")
	c.clientContainer, err = container.New(client)
	if err != nil {
		return err
	}
	c.clientContainer.UserAgent = userAgent

	log.Printf("[INFO] Instantiating GKE Beta client...")
	c.clientContainerBeta, err = containerBeta.New(client)
	if err != nil {
		return err
	}
	c.clientContainerBeta.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud DNS client...")
	c.clientDns, err = dns.New(client)
	if err != nil {
		return err
	}
	c.clientDns.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud DNS Beta client...")
	c.clientDnsBeta, err = dnsBeta.New(client)
	if err != nil {
		return err
	}
	c.clientDnsBeta.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud KMS Client...")
	c.clientKms, err = cloudkms.New(client)
	if err != nil {
		return err
	}
	c.clientKms.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Stackdriver Logging client...")
	c.clientLogging, err = cloudlogging.New(client)
	if err != nil {
		return err
	}
	c.clientLogging.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Storage Client...")
	c.clientStorage, err = storage.New(client)
	if err != nil {
		return err
	}
	c.clientStorage.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google SqlAdmin Client...")
	c.clientSqlAdmin, err = sqladmin.New(client)
	if err != nil {
		return err
	}
	c.clientSqlAdmin.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Pubsub Client...")
	c.clientPubsub, err = pubsub.New(client)
	if err != nil {
		return err
	}
	c.clientPubsub.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Dataflow Client...")
	c.clientDataflow, err = dataflow.New(client)
	if err != nil {
		return err
	}
	c.clientDataflow.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud Redis Client...")
	c.clientRedis, err = redis.New(client)
	if err != nil {
		return err
	}
	c.clientRedis.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud ResourceManager Client...")
	c.clientResourceManager, err = cloudresourcemanager.New(client)
	if err != nil {
		return err
	}
	c.clientResourceManager.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud ResourceManager V Client...")
	c.clientResourceManagerV2Beta1, err = resourceManagerV2Beta1.New(client)
	if err != nil {
		return err
	}
	c.clientResourceManagerV2Beta1.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud Runtimeconfig Client...")
	c.clientRuntimeconfig, err = runtimeconfig.New(client)
	if err != nil {
		return err
	}
	c.clientRuntimeconfig.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud IAM Client...")
	c.clientIAM, err = iam.New(client)
	if err != nil {
		return err
	}
	c.clientIAM.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud Service Management Client...")
	c.clientServiceMan, err = servicemanagement.New(client)
	if err != nil {
		return err
	}
	c.clientServiceMan.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud Service Usage Client...")
	c.clientServiceUsage, err = serviceusage.New(client)
	if err != nil {
		return err
	}
	c.clientServiceUsage.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud Billing Client...")
	c.clientBilling, err = cloudbilling.New(client)
	if err != nil {
		return err
	}
	c.clientBilling.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud Build Client...")
	c.clientBuild, err = cloudbuild.New(client)
	if err != nil {
		return err
	}
	c.clientBuild.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud BigQuery Client...")
	c.clientBigQuery, err = bigquery.New(client)
	if err != nil {
		return err
	}
	c.clientBigQuery.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud CloudFunctions Client...")
	c.clientCloudFunctions, err = cloudfunctions.New(client)
	if err != nil {
		return err
	}
	c.clientCloudFunctions.UserAgent = userAgent

	c.bigtableClientFactory = &BigtableClientFactory{
		UserAgent:   userAgent,
		TokenSource: tokenSource,
	}

	log.Printf("[INFO] Instantiating Google Cloud Source Repo Client...")
	c.clientSourceRepo, err = sourcerepo.New(client)
	if err != nil {
		return err
	}
	c.clientSourceRepo.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud Spanner Client...")
	c.clientSpanner, err = spanner.New(client)
	if err != nil {
		return err
	}
	c.clientSpanner.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud Dataproc Client...")
	c.clientDataproc, err = dataproc.New(client)
	if err != nil {
		return err
	}
	c.clientDataproc.UserAgent = userAgent

	c.clientFilestore, err = file.New(client)
	if err != nil {
		return err
	}
	c.clientFilestore.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud IoT Core Client...")
	c.clientCloudIoT, err = cloudiot.New(client)
	if err != nil {
		return err
	}
	c.clientCloudIoT.UserAgent = userAgent

	log.Printf("[INFO] Instantiating App Engine Client...")
	c.clientAppEngine, err = appengine.New(client)
	if err != nil {
		return err
	}
	c.clientAppEngine.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Cloud Composer Client...")
	c.clientComposer, err = composer.New(client)
	if err != nil {
		return err
	}
	c.clientComposer.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud Storage Transfer Client...")
	c.clientStorageTransfer, err = storagetransfer.New(client)
	if err != nil {
		return err
	}
	c.clientStorageTransfer.UserAgent = userAgent

	return nil
}

func (c *Config) getTokenSource(clientScopes []string) (oauth2.TokenSource, error) {
	if c.AccessToken != "" {
		contents, _, err := pathorcontents.Read(c.AccessToken)
		if err != nil {
			return nil, fmt.Errorf("Error loading access token: %s", err)
		}

		log.Printf("[INFO] Authenticating using configured Google JSON 'access_token'...")
		log.Printf("[INFO]   -- Scopes: %s", clientScopes)
		token := &oauth2.Token{AccessToken: contents}
		return oauth2.StaticTokenSource(token), nil
	}

	if c.Credentials != "" {
		contents, _, err := pathorcontents.Read(c.Credentials)
		if err != nil {
			return nil, fmt.Errorf("Error loading credentials: %s", err)
		}

		creds, err := googleoauth.CredentialsFromJSON(context.Background(), []byte(contents), clientScopes...)
		if err != nil {
			return nil, fmt.Errorf("Unable to parse credentials from '%s': %s", contents, err)
		}

		log.Printf("[INFO] Authenticating using configured Google JSON 'credentials'...")
		log.Printf("[INFO]   -- Scopes: %s", clientScopes)
		return creds.TokenSource, nil
	}

	log.Printf("[INFO] Authenticating using DefaultClient...")
	log.Printf("[INFO]   -- Scopes: %s", clientScopes)
	return googleoauth.DefaultTokenSource(context.Background(), clientScopes...)
}
<|endoftext|>