<commit_before>package model

import (
	"github.com/dinever/dingo/app/model/sql_builder"
)

const schema = `
CREATE TABLE IF NOT EXISTS
posts (
  id                 integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  title              varchar(150) NOT NULL,
  slug               varchar(150) NOT NULL,
  markdown           text,
  html               text,
  image              text,
  featured           tinyint NOT NULL DEFAULT '0',
  page               tinyint NOT NULL DEFAULT '0',
  allow_comment      tinyint NOT NULL DEFAULT '0',
  comment_num        integer NOT NULL DEFAULT '0',
  status             varchar(150) NOT NULL DEFAULT 'draft',
  language           varchar(6) NOT NULL DEFAULT 'en_US',
  meta_title         varchar(150),
  meta_description   varchar(200),
  author_id          integer NOT NULL,
  created_at         datetime NOT NULL,
  created_by         integer NOT NULL,
  updated_at         datetime,
  updated_by         integer,
  published_at       datetime,
  published_by       integer
);

CREATE TABLE IF NOT EXISTS
tokens (
  value       varchar(40) NOT NULL PRIMARY KEY,
  user_id     integer UNIQUE,
  created_at  datetime,
  expired_at  datetime
);

CREATE TABLE IF NOT EXISTS
users (
  id               integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  name             varchar(150) NOT NULL,
  slug             varchar(150) NOT NULL,
  password         varchar(60) NOT NULL,
  email            varchar(254) NOT NULL,
  image            text,
  cover            text,
  bio              varchar(200),
  website          text,
  location         text,
  accessibility    text,
  status           varchar(150) NOT NULL DEFAULT 'active',
  language         varchar(6) NOT NULL DEFAULT 'en_US',
  meta_title       varchar(150),
  meta_description varchar(200),
  last_login       datetime,
  created_at       datetime NOT NULL,
  created_by       integer NOT NULL,
  updated_at       datetime,
  updated_by       integer
);

CREATE TABLE IF NOT EXISTS
categories (
  id                integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  name              varchar(150) NOT NULL,
  slug              varchar(150) NOT NULL,
  description       varchar(200),
  parent_id         integer,
  meta_title        varchar(150),
  meta_description  varchar(200),
  created_at        datetime NOT NULL,
  created_by        integer NOT NULL,
  updated_at        datetime,
  updated_by        integer
);

CREATE TABLE IF NOT EXISTS
tags (
  id                integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  name              varchar(150) NOT NULL,
  slug              varchar(150) NOT NULL,
  description       varchar(200),
  image             text,
  hidden            boolean NOT NULL DEFAULT 0,
  parent_id         integer,
  meta_title        varchar(150),
  meta_description  varchar(200),
  created_at        datetime NOT NULL,
  created_by        integer NOT NULL,
  updated_at        datetime,
  updated_by        integer
);

CREATE TABLE IF NOT EXISTS
comments (
  id            integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  post_id       varchar(150) NOT NULL,
  author        varchar(150) NOT NULL,
  author_email  varchar(150) NOT NULL,
  author_url    varchar(200) NOT NULL,
  author_ip     varchar(100) NOT NULL,
  created_at    datetime NOT NULL,
  content       text NOT NULL,
  approved      tinyint NOT NULL DEFAULT '0',
  agent         varchar(255) NOT NULL,
  type          varchar(20),
  parent        integer,
  user_id       integer
);

CREATE TABLE IF NOT EXISTS
posts_tags (
  id       integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  post_id  integer NOT NULL,
  tag_id   integer NOT NULL
);

CREATE TABLE IF NOT EXISTS
posts_categories (
  id           integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  post_id      integer NOT NULL,
  category_id  integer NOT NULL
);

CREATE TABLE IF NOT EXISTS
settings (
  id          integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  key         varchar(150) NOT NULL,
  value       varchar(20) NOT NULL,
  type        varchar(150) NOT NULL DEFAULT 'core',
  created_at  datetime NOT NULL,
  created_by  integer NOT NULL,
  updated_at  datetime,
  updated_by  integer
);

CREATE TABLE IF NOT EXISTS
roles (
  id           integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  name         varchar(150) NOT NULL,
  description  varchar(200),
  created_at   datetime NOT NULL,
  created_by   integer NOT NULL,
  updated_at   datetime,
  updated_by   integer
);

CREATE TABLE IF NOT EXISTS
messages (
  id           integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  type         varchar(20) NOT NULL,
  data         text NOT NULL,
  is_read      boolean NOT NULL default 0,
  created_at   datetime NOT NULL
);
`

// Posts
var postCountSelector = SQL.Select(`count(*)`).From(`posts`)
var stmtGetPublishedPostsCount = postCountSelector.Copy().Where(`status = "published"`).SQL()
var stmtGetAllPostsCount = postCountSelector.Copy().SQL()
var stmtGetPostsCountByUser = postCountSelector.Copy().Where(`author_id = ?`).SQL()
var stmtGetPostsCountByTag = postCountSelector.Copy().From(`posts, posts_tags`).Where(`posts_tags.post_id = posts.id`, `posts_tags.tag_id = ?`, `status = 'published'`).SQL()

var postSelector = SQL.Select(`id, title, slug, markdown, html, featured, page, allow_comment, comment_num, status, image, author_id, created_at, created_by, updated_at, updated_by, published_at, published_by`).From(`posts`)
var stmtGetPublishedPostList = postSelector.Copy().Where(`status = "published"`).OrderBy(`published_at DESC`).Limit(`?`).Offset(`?`).SQL()
var stmtGetAllPostList = postSelector.Copy().OrderBy(`published_at DESC`).Limit(`?`).Offset(`?`).SQL()
var stmtGetPostsByUser = postSelector.Copy().Where(`status = 'published'`, `author_id = ?`).OrderBy(`published_at DESC`).Limit(`?`).Offset(`?`).SQL()

var stmtGetPostById = postSelector.Copy().Where(`id = ?`).SQL()
var stmtGetPostBySlug = postSelector.Copy().Where(`slug = ?`).SQL()

var postsTagsSelector = SQL.Select(`posts.id, posts.title, posts.slug, posts.markdown, posts.html, posts.featured, posts.page, posts.allow_comment, posts.comment_num, posts.status, posts.image, posts.author_id, posts.created_at, posts.created_by, posts.updated_at, posts.updated_by, posts.published_at, posts.published_by`).From(`posts, posts_tags`)
var stmtGetPostsByTag = postsTagsSelector.Copy().Where(`status = 'published'`, `posts_tags.post_id = posts.id`, `posts_tags.tag_id = ?`).OrderBy(`published_at DESC`).Limit(`?`).Offset(`?`).SQL()
var stmtGetAllPostsByTag = postsTagsSelector.Copy().Where(`posts_tags.post_id = posts.id`, `posts_tags.tag_id = ?`).OrderBy(`published_at DESC`).SQL()

var pageCountSelector = SQL.Select(`count(*)`).From(`posts`).Where(`page = 1`)
var stmtGetPublishedPagesCount = pageCountSelector.Copy().Where(`status = "published"`).SQL()
var stmtGetAllPagesCount = pageCountSelector.Copy().SQL()
var stmtGetPagesCountByUser = pageCountSelector.Copy().Where(`author_id = ?`).SQL()
var stmtGetPagesCountByTag = pageCountSelector.Copy().Where(`posts_tags.post_id = posts.id`, `posts_tags.tag_id = ?`, `status = 'published'`).SQL()

var pageSelector = SQL.Select(`id, title, slug, markdown, html, featured, page, status, image, author_id, created_at, created_by, updated_at, updated_by, published_at, published_by`).From(`posts`).Where(`page = 1`)

// Comments

var commentCountSelector = SQL.Select(`count(*)`).From(`comments`)
var stmtGetAllCommentCount = commentCountSelector.SQL()
var commentSelector = SQL.Select(`id, post_id, author, author_email, author_url, created_at, content, approved, agent, parent, user_id`).From(`comments`)
var stmtGetAllCommentList = commentSelector.Copy().OrderBy(`created_at DESC`).Limit(`?`).Offset(`?`).SQL()
var stmtGetApprovedCommentList = commentSelector.Copy().Where(`approved = 1`).OrderBy(`created_at DESC`).Limit(`?`).Offset(`?`).SQL()
var stmtGetCommentById = commentSelector.Copy().Where(`id = ?`).SQL()
var stmtGetApprovedCommentListByPostId = commentSelector.Copy().Where(`post_id = ?`, `approved = 1`).OrderBy(`created_at DESC`).SQL()

const stmtInsertComment = `INSERT OR REPLACE INTO comments (id, post_id, author, author_email, author_url, author_ip, created_at, content, approved, agent, parent, user_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
const stmtDeleteCommentById = `DELETE FROM comments WHERE id = ?`

// Users
const stmtGetUserById = `SELECT id, name, slug, email, image, cover, bio, website, location FROM users WHERE id = ?`
const stmtGetUserBySlug = `SELECT id, name, slug, email, image, cover, bio, website, location FROM users WHERE slug = ?`
const stmtGetUserByName = `SELECT id, name, slug, email, image, cover, bio, website, location FROM users WHERE name = ?`
const stmtGetUserByEmail = `SELECT id, name, slug, email, image, cover, bio, website, location FROM users WHERE email = ?`
const stmtGetHashedPasswordByEmail = `SELECT password FROM users WHERE email = ?`
const stmtGetUsersCount = `SELECT count(*) FROM users`
const stmtGetUsersCountByEmail = `SELECT count(*) FROM users where email = ?`

// Tokens
const stmtGetTokenByValue = `SELECT value, user_id, created_at, expired_at FROM tokens WHERE value = ?`
const stmtUpdateToken = `INSERT OR REPLACE INTO tokens (value, user_id, created_at, expired_at) VALUES (?, ?, ?, ?)`

// Tags
const stmtGetAllTags = `SELECT id, name, slug FROM tags`
const stmtGetTags = `SELECT tag_id FROM posts_tags WHERE post_id = ?`
const stmtGetTagById = `SELECT id, name, slug FROM tags WHERE id = ?`
const stmtGetTagBySlug = `SELECT id, name, slug, hidden FROM tags WHERE slug = ?`

// Settings
const stmtGetBlog = `SELECT value FROM settings WHERE key = ?`
const stmtGetPostCreationDateById = `SELECT created_at FROM posts WHERE id = ?`

const stmtInsertPost = `INSERT INTO posts (id, title, slug, markdown, html, featured, page, allow_comment, status, image, author_id, created_at, created_by, updated_at, updated_by, published_at, published_by) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
const stmtInsertUser = `INSERT INTO users (id, name, slug, password, email, image, cover, created_at, created_by, updated_at, updated_by) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
const stmtInsertRoleUser = `INSERT INTO roles_users (id, role_id, user_id) VALUES (?, ?, ?)`
const stmtInsertTag = `INSERT INTO tags (id, name, slug, created_at, created_by, updated_at, updated_by, hidden) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
const stmtInsertPostTag = `INSERT INTO posts_tags (id, post_id, tag_id) VALUES (?, ?, ?)`
const stmtInsertSetting = `INSERT INTO settings (id, key, value, type, created_at, created_by, updated_at, updated_by) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`

const stmtUpdatePost = `UPDATE posts SET title = ?, slug = ?, markdown = ?, html = ?, featured = ?, page = ?, allow_comment = ?, status = ?, image = ?, updated_at = ?, updated_by = ? WHERE id = ?`
const stmtUpdatePostPublished = `UPDATE posts SET title = ?, slug = ?, markdown = ?, html = ?, featured = ?, page = ?, allow_comment = ?, status = ?, image = ?, updated_at = ?, updated_by = ?, published_at = ?, published_by = ? WHERE id = ?`
const stmtUpdateSettings = `UPDATE settings SET value = ?, updated_at = ?, updated_by = ? WHERE key = ?`
const stmtUpdateUser = `UPDATE users SET name = ?, slug = ?, email = ?, image = ?, cover = ?, bio = ?, website = ?, location = ?, updated_at = ?, updated_by = ? WHERE id = ?`
const stmtUpdateLastLogin = `UPDATE users SET last_login = ? WHERE id = ?`
const stmtUpdateUserPassword = `UPDATE users SET password = ?, updated_at = ?, updated_by = ? WHERE id = ?`
const stmtUpdateTag = `UPDATE tags SET name = ?, slug =?, updated_at = ?, updated_by = ?, hidden = ? WHERE id = ?`

const stmtDeletePostTagsByPostId = `DELETE FROM posts_tags WHERE post_id = ?`
const stmtDeletePostById = `DELETE FROM posts WHERE id = ?`
const stmtDeleteOldTags = `DELETE FROM tags WHERE id IN (SELECT id FROM tags EXCEPT SELECT tag_id FROM posts_tags)`

// Messages
var messageCountSelector = SQL.Select(`count(*)`).From(`messages`)
var sttmGetUnreadMessageCount = messageCountSelector.Copy().Where(`is_read = 0`).From(`messages`).SQL()

var messageSelector = SQL.Select(`id, type, data, is_read, created_at`).From(`messages`)
var stmtGetAllMessages = messageSelector.Copy().OrderBy(`created_at DESC`).SQL()
var stmtGetUnreadMessages = messageSelector.Copy().Where(`is_read = 0`).OrderBy(`created_at DESC`).Limit(`?`).Offset(`?`).SQL()
var stmtGetAllMessagesByPage = messageSelector.Copy().OrderBy(`created_at DESC`).Limit(`?`).Offset(`?`).SQL()

const stmtInsertMessage = `INSERT INTO messages (id, type, data, is_read, created_at) VALUES (?, ?, ?, ?, ?)`
<commit_msg>[refactor] Remove unused sql statements<commit_after>package model

import (
	"github.com/dinever/dingo/app/model/sql_builder"
)

const schema = `
CREATE TABLE IF NOT EXISTS
posts (
  id                 integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  title              varchar(150) NOT NULL,
  slug               varchar(150) NOT NULL,
  markdown           text,
  html               text,
  image              text,
  featured           tinyint NOT NULL DEFAULT '0',
  page               tinyint NOT NULL DEFAULT '0',
  allow_comment      tinyint NOT NULL DEFAULT '0',
  comment_num        integer NOT NULL DEFAULT '0',
  status             varchar(150) NOT NULL DEFAULT 'draft',
  language           varchar(6) NOT NULL DEFAULT 'en_US',
  meta_title         varchar(150),
  meta_description   varchar(200),
  author_id          integer NOT NULL,
  created_at         datetime NOT NULL,
  created_by         integer NOT NULL,
  updated_at         datetime,
  updated_by         integer,
  published_at       datetime,
  published_by       integer
);

CREATE TABLE IF NOT EXISTS
tokens (
  value       varchar(40) NOT NULL PRIMARY KEY,
  user_id     integer UNIQUE,
  created_at  datetime,
  expired_at  datetime
);

CREATE TABLE IF NOT EXISTS
users (
  id               integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  name             varchar(150) NOT NULL,
  slug             varchar(150) NOT NULL,
  password         varchar(60) NOT NULL,
  email            varchar(254) NOT NULL,
  image            text,
  cover            text,
  bio              varchar(200),
  website          text,
  location         text,
  accessibility    text,
  status           varchar(150) NOT NULL DEFAULT 'active',
  language         varchar(6) NOT NULL DEFAULT 'en_US',
  meta_title       varchar(150),
  meta_description varchar(200),
  last_login       datetime,
  created_at       datetime NOT NULL,
  created_by       integer NOT NULL,
  updated_at       datetime,
  updated_by       integer
);

CREATE TABLE IF NOT EXISTS
categories (
  id                integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  name              varchar(150) NOT NULL,
  slug              varchar(150) NOT NULL,
  description       varchar(200),
  parent_id         integer,
  meta_title        varchar(150),
  meta_description  varchar(200),
  created_at        datetime NOT NULL,
  created_by        integer NOT NULL,
  updated_at        datetime,
  updated_by        integer
);

CREATE TABLE IF NOT EXISTS
tags (
  id                integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  name              varchar(150) NOT NULL,
  slug              varchar(150) NOT NULL,
  description       varchar(200),
  image             text,
  hidden            boolean NOT NULL DEFAULT 0,
  parent_id         integer,
  meta_title        varchar(150),
  meta_description  varchar(200),
  created_at        datetime NOT NULL,
  created_by        integer NOT NULL,
  updated_at        datetime,
  updated_by        integer
);

CREATE TABLE IF NOT EXISTS
comments (
  id            integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  post_id       varchar(150) NOT NULL,
  author        varchar(150) NOT NULL,
  author_email  varchar(150) NOT NULL,
  author_url    varchar(200) NOT NULL,
  author_ip     varchar(100) NOT NULL,
  created_at    datetime NOT NULL,
  content       text NOT NULL,
  approved      tinyint NOT NULL DEFAULT '0',
  agent         varchar(255) NOT NULL,
  type          varchar(20),
  parent        integer,
  user_id       integer
);

CREATE TABLE IF NOT EXISTS
posts_tags (
  id       integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  post_id  integer NOT NULL,
  tag_id   integer NOT NULL
);

CREATE TABLE IF NOT EXISTS
posts_categories (
  id           integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  post_id      integer NOT NULL,
  category_id  integer NOT NULL
);

CREATE TABLE IF NOT EXISTS
settings (
  id          integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  key         varchar(150) NOT NULL,
  value       varchar(20) NOT NULL,
  type        varchar(150) NOT NULL DEFAULT 'core',
  created_at  datetime NOT NULL,
  created_by  integer NOT NULL,
  updated_at  datetime,
  updated_by  integer
);

CREATE TABLE IF NOT EXISTS
roles (
  id           integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  name         varchar(150) NOT NULL,
  description  varchar(200),
  created_at   datetime NOT NULL,
  created_by   integer NOT NULL,
  updated_at   datetime,
  updated_by   integer
);

CREATE TABLE IF NOT EXISTS
messages (
  id           integer NOT NULL PRIMARY KEY AUTOINCREMENT,
  type         varchar(20) NOT NULL,
  data         text NOT NULL,
  is_read      boolean NOT NULL default 0,
  created_at   datetime NOT NULL
);
`

// Posts
var postCountSelector = SQL.Select(`count(*)`).From(`posts`)
var stmtGetPostsCountByTag = postCountSelector.Copy().From(`posts, posts_tags`).Where(`posts_tags.post_id = posts.id`, `posts_tags.tag_id = ?`, `status = 'published'`).SQL()

var postSelector = SQL.Select(`id, title, slug, markdown, html, featured, page, allow_comment, comment_num, status, image, author_id, created_at, created_by, updated_at, updated_by, published_at, published_by`).From(`posts`)
var stmtGetPostById = postSelector.Copy().Where(`id = ?`).SQL()
var stmtGetPostBySlug = postSelector.Copy().Where(`slug = ?`).SQL()

var postsTagsSelector = SQL.Select(`posts.id, posts.title, posts.slug, posts.markdown, posts.html, posts.featured, posts.page, posts.allow_comment, posts.comment_num, posts.status, posts.image, posts.author_id, posts.created_at, posts.created_by, posts.updated_at, posts.updated_by, posts.published_at, posts.published_by`).From(`posts, posts_tags`)
var stmtGetPostsByTag = postsTagsSelector.Copy().Where(`status = 'published'`, `posts_tags.post_id = posts.id`, `posts_tags.tag_id = ?`).OrderBy(`published_at DESC`).Limit(`?`).Offset(`?`).SQL()
var stmtGetAllPostsByTag = postsTagsSelector.Copy().Where(`posts_tags.post_id = posts.id`, `posts_tags.tag_id = ?`).OrderBy(`published_at DESC`).SQL()

// Comments
var commentCountSelector = SQL.Select(`count(*)`).From(`comments`)
var stmtGetAllCommentCount = commentCountSelector.SQL()
var commentSelector = SQL.Select(`id, post_id, author, author_email, author_url, created_at, content, approved, agent, parent, user_id`).From(`comments`)
var stmtGetAllCommentList = commentSelector.Copy().OrderBy(`created_at DESC`).Limit(`?`).Offset(`?`).SQL()
var stmtGetApprovedCommentList = commentSelector.Copy().Where(`approved = 1`).OrderBy(`created_at DESC`).Limit(`?`).Offset(`?`).SQL()
var stmtGetCommentById = commentSelector.Copy().Where(`id = ?`).SQL()
var stmtGetApprovedCommentListByPostId = commentSelector.Copy().Where(`post_id = ?`, `approved = 1`).OrderBy(`created_at DESC`).SQL()

const stmtInsertComment = `INSERT OR REPLACE INTO comments (id, post_id, author, author_email, author_url, author_ip, created_at, content, approved, agent, parent, user_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
const stmtDeleteCommentById = `DELETE FROM comments WHERE id = ?`

// Users
const stmtGetUserById = `SELECT id, name, slug, email, image, cover, bio, website, location FROM users WHERE id = ?`
const stmtGetUserBySlug = `SELECT id, name, slug, email, image, cover, bio, website, location FROM users WHERE slug = ?`
const stmtGetUserByName = `SELECT id, name, slug, email, image, cover, bio, website, location FROM users WHERE name = ?`
const stmtGetUserByEmail = `SELECT id, name, slug, email, image, cover, bio, website, location FROM users WHERE email = ?`
const stmtGetHashedPasswordByEmail = `SELECT password FROM users WHERE email = ?`
const stmtGetUsersCountByEmail = `SELECT count(*) FROM users where email = ?`

// Tokens
const stmtGetTokenByValue = `SELECT value, user_id, created_at, expired_at FROM tokens WHERE value = ?`
const stmtUpdateToken = `INSERT OR REPLACE INTO tokens (value, user_id, created_at, expired_at) VALUES (?, ?, ?, ?)`

// Tags
const stmtGetAllTags = `SELECT id, name, slug FROM tags`
const stmtGetTags = `SELECT tag_id FROM posts_tags WHERE post_id = ?`
const stmtGetTagById = `SELECT id, name, slug FROM tags WHERE id = ?`
const stmtGetTagBySlug = `SELECT id, name, slug, hidden FROM tags WHERE slug = ?`

// Settings
const stmtInsertPost = `INSERT INTO posts (id, title, slug, markdown, html, featured, page, allow_comment, status, image, author_id, created_at, created_by, updated_at, updated_by, published_at, published_by) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
const stmtInsertUser = `INSERT INTO users (id, name, slug, password, email, image, cover, created_at, created_by, updated_at, updated_by) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
const stmtInsertRoleUser = `INSERT INTO roles_users (id, role_id, user_id) VALUES (?, ?, ?)`
const stmtInsertTag = `INSERT INTO tags (id, name, slug, created_at, created_by, updated_at, updated_by, hidden) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
const stmtInsertPostTag = `INSERT INTO posts_tags (id, post_id, tag_id) VALUES (?, ?, ?)`
const stmtInsertSetting = `INSERT INTO settings (id, key, value, type, created_at, created_by, updated_at, updated_by) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`

const stmtUpdatePost = `UPDATE posts SET title = ?, slug = ?, markdown = ?, html = ?, featured = ?, page = ?, allow_comment = ?, status = ?, image = ?, updated_at = ?, updated_by = ? WHERE id = ?`
const stmtUpdatePostPublished = `UPDATE posts SET title = ?, slug = ?, markdown = ?, html = ?, featured = ?, page = ?, allow_comment = ?, status = ?, image = ?, updated_at = ?, updated_by = ?, published_at = ?, published_by = ? WHERE id = ?`
const stmtUpdateUser = `UPDATE users SET name = ?, slug = ?, email = ?, image = ?, cover = ?, bio = ?, website = ?, location = ?, updated_at = ?, updated_by = ? WHERE id = ?`
const stmtUpdateTag = `UPDATE tags SET name = ?, slug =?, updated_at = ?, updated_by = ?, hidden = ? WHERE id = ?`

const stmtDeletePostTagsByPostId = `DELETE FROM posts_tags WHERE post_id = ?`
const stmtDeletePostById = `DELETE FROM posts WHERE id = ?`
const stmtDeleteOldTags = `DELETE FROM tags WHERE id IN (SELECT id FROM tags EXCEPT SELECT tag_id FROM posts_tags)`

// Messages
var messageSelector = SQL.Select(`id, type, data, is_read, created_at`).From(`messages`)
var stmtGetUnreadMessages = messageSelector.Copy().Where(`is_read = 0`).OrderBy(`created_at DESC`).Limit(`?`).Offset(`?`).SQL()

const stmtInsertMessage = `INSERT INTO messages (id, type, data, is_read, created_at) VALUES (?, ?, ?, ?, ?)`
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/NebulousLabs/Sia/api"
	"github.com/NebulousLabs/Sia/types"

	"github.com/spf13/cobra"
)

var (
	renterExportCmd = &cobra.Command{
		Use:   "export",
		Short: "export renter data to various formats",
		Long:  "Export renter data in various formats.",
		// Run field not provided; export requires a subcommand.
	}

	renterExportContractsCmd = &cobra.Command{
		Use:   "rankings-data [destination]",
		Short: "export the renter's contracts for import to `https://rankings.sia.tech/`",
		Long:  "Export the renter's current contract set in JSON format to the specified " +
			"file. Intended for upload to `https://rankings.sia.tech/`.",
		Run:   wrap(renterexportcontractscmd),
	}
)

// renterexportcontractscmd is the handler for the command `siac renter export contracts`.
// Exports the current contract set to JSON.
func renterexportcontractscmd(destination string) {
	var cs api.RenterContracts
	err := getAPI("/renter/contracts", &cs)
	if err != nil {
		die("Could not retrieve contracts:", err)
	}
	var contractTxns []types.Transaction
	for _, c := range cs.Contracts {
		contractTxns = append(contractTxns, c.LastTransaction)
	}
	destination = abs(destination)
	file, err := os.Create(destination)
	if err != nil {
		die("Could not export to file:", err)
	}
	err = json.NewEncoder(file).Encode(contractTxns)
	if err != nil {
		die("Could not export to file:", err)
	}
	fmt.Println("Exported contract data to", destination)
}
<commit_msg>Fix indentation.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/NebulousLabs/Sia/api"
	"github.com/NebulousLabs/Sia/types"

	"github.com/spf13/cobra"
)

var (
	renterExportCmd = &cobra.Command{
		Use:   "export",
		Short: "export renter data to various formats",
		Long:  "Export renter data in various formats.",
		// Run field not provided; export requires a subcommand.
	}

	renterExportContractsCmd = &cobra.Command{
		Use:   "rankings-data [destination]",
		Short: "export the renter's contracts for import to `https://rankings.sia.tech/`",
		Long: "Export the renter's current contract set in JSON format to the specified " +
			"file. Intended for upload to `https://rankings.sia.tech/`.",
		Run: wrap(renterexportcontractscmd),
	}
)

// renterexportcontractscmd is the handler for the command `siac renter export contracts`.
// Exports the current contract set to JSON.
func renterexportcontractscmd(destination string) {
	var cs api.RenterContracts
	err := getAPI("/renter/contracts", &cs)
	if err != nil {
		die("Could not retrieve contracts:", err)
	}
	var contractTxns []types.Transaction
	for _, c := range cs.Contracts {
		contractTxns = append(contractTxns, c.LastTransaction)
	}
	destination = abs(destination)
	file, err := os.Create(destination)
	if err != nil {
		die("Could not export to file:", err)
	}
	err = json.NewEncoder(file).Encode(contractTxns)
	if err != nil {
		die("Could not export to file:", err)
	}
	fmt.Println("Exported contract data to", destination)
}
<|endoftext|>


<commit_before>package platform

import (
	"reflect"
	"testing"
)

func TestIDFromString(t *testing.T) {
	tests := []struct {
		name    string
		idstr   string
		want    *ID
		wantErr bool
	}{
		{
			name:  "Is able to decode an id",
			idstr: "020f755c3c082000",
			want:  &ID{0x02, 0x0f, 0x75, 0x5c, 0x3c, 0x08, 0x20, 0x00},
		},
		{
			name:    "It should not be able to decode an id that's not hex",
			idstr:   "gggggggggggggg",
			wantErr: true,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got, err := IDFromString(tt.idstr)
			if (err != nil) != tt.wantErr {
				t.Errorf("IDFromString() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			if !reflect.DeepEqual(got, tt.want) {
				t.Errorf("IDFromString() = %v, want %v", got, tt.want)
			}
		})
	}
}
<commit_msg>Initial tests for platform.ID<commit_after>package platform

import (
	"bytes"
	"reflect"
	"testing"
)

func TestIDFromString(t *testing.T) {
	tests := []struct {
		name    string
		idstr   string
		want    *ID
		wantErr bool
	}{
		{
			name:  "Is able to decode an id",
			idstr: "020f755c3c082000",
			want:  &ID{0x02, 0x0f, 0x75, 0x5c, 0x3c, 0x08, 0x20, 0x00},
		},
		{
			name:    "It should not be able to decode an id that's not hex",
			idstr:   "gggggggggggggg",
			wantErr: true,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got, err := IDFromString(tt.idstr)
			if (err != nil) != tt.wantErr {
				t.Errorf("IDFromString() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			if !reflect.DeepEqual(got, tt.want) {
				t.Errorf("IDFromString() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestDecodeFromString(t *testing.T) {
	var id1 ID
	err := id1.DecodeFromString("020f755c3c082000")
	if err != nil {
		t.Errorf(err.Error())
	}
	want1 := []byte{48, 50, 48, 102, 55, 53, 53, 99, 51, 99, 48, 56, 50, 48, 48, 48}
	byte1 := id1.Encode()
	if !bytes.Equal(want1, byte1) {
		t.Errorf("encoding error")
	}
}

func TestDecodeFromShorterString(t *testing.T) {
	var id1 ID
	err := id1.DecodeFromString("020f75")
	if err != nil {
		t.Errorf(err.Error())
	}
	want1 := []byte{48, 50, 48, 102, 55, 53, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48}
	byte1 := id1.Encode()
	if !bytes.Equal(want1, byte1) {
		t.Errorf("encoding error")
	}

	if id1.String() != "020f750000000000" {
		t.Errorf("wrong id")
	}
}

func TestDecodeFromLongerString(t *testing.T) {
	var id1 ID
	err := id1.DecodeFromString("020f755c3c082000aaa")
	if err != nil {
		t.Errorf(err.Error())
	}
	want1 := []byte{48, 50, 48, 102, 55, 53, 53, 99, 51, 99, 48, 56, 50, 48, 48, 48}
	byte1 := id1.Encode()
	if !bytes.Equal(want1, byte1) {
		t.Errorf("encoding error")
	}
}

// todo
// func TestDecodeFromEmptyString(t *testing.T) {}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/ActiveState/tail"
	"io/ioutil"
	"os"
	"regexp"
	"strconv"
	"strings"
)

type inputTail struct {
	path     string
	format   string
	tag      string
	pos_file string

	offset int64
}

func (self *inputTail) new() interface{} {
	return &inputTail{offset: 0}
}

func (self *inputTail) configure(f map[string]interface{}) error {
	var value interface{}

	value = f["path"]
	if value != nil {
		self.path = value.(string)
	}

	value = f["format"]
	if value != nil {
		self.format = value.(string)
	}

	value = f["tag"]
	if value != nil {
		self.tag = value.(string)
	}

	value = f["pos_file"]
	if value != nil {
		self.pos_file = value.(string)

		str, err := ioutil.ReadFile(self.pos_file)
		if err != nil {
			Log(err)
		}

		f, err := os.Open(self.path)
		if err != nil {
			Log(err)
			return err
		}

		info, err := f.Stat()
		if err != nil {
			Log(err)
			return err
		}

		offset, _ := strconv.Atoi(string(str))
		if int64(offset) > info.Size() {
			self.offset = info.Size()
		} else {
			self.offset = int64(offset)
		}
	}

	return nil
}

func (self *inputTail) start(ctx chan Context) error {

	var seek int
	if self.offset > 0 {
		seek = os.SEEK_SET
	} else {
		seek = os.SEEK_END
	}

	t, err := tail.TailFile(self.path, tail.Config{Follow: true, Location: &tail.SeekInfo{int64(self.offset), seek}})
	if err != nil {
		return err
	}

	var re regexp.Regexp
	if string(self.format[0]) == string("/") || string(self.format[len(self.format)-1]) == string("/") {
		fmt.Println(strings.Trim(self.format, "/"))
		re = *regexp.MustCompile(self.format)
		self.format = "regexp"
	} else if self.format == "json" {

	}

	for line := range t.Lines {

		timeUnix := line.Time.Unix()
		data := make(map[string]string)

		if self.format == "regexp" {
			text := re.FindSubmatch([]byte(line.Text))
			if text == nil {
				continue
			}

			for i, name := range re.SubexpNames() {
				if i != 0 {
					Log(name, string(text[i]))
					data[name] = string(text[i])
				}
			}
		} else if self.format == "json" {
			err := json.Unmarshal([]byte(line.Text), &data)
			if err != nil {
				continue
			}
		}

		offset, err := t.Tell()
		if err != nil {
			fmt.Println("Tell return error: ", err)
		}

		str := strconv.Itoa(int(offset))

		f, err := os.OpenFile(self.pos_file, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
		if err != nil {
			Log(err)
		}

		_, err = f.WriteString(str)
		if err != nil {
			Log(err)
		}

		f.Close()

		//Log("offset:", offset, "raw_string:", line.Text)

		record := Record{timeUnix, data}
		ctx <- Context{self.tag, record}
	}

	err = t.Wait()
	if err != nil {
		return err
	}

	return err
}

func init() {
	RegisterInput("tail", &inputTail{})
}
<commit_msg>using poll to fix the bug when file is deleted<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/ActiveState/tail"
	"io/ioutil"
	"os"
	"regexp"
	"strconv"
	"strings"
)

type inputTail struct {
	path     string
	format   string
	tag      string
	pos_file string

	offset int64
}

func (self *inputTail) new() interface{} {
	return &inputTail{offset: 0}
}

func (self *inputTail) configure(f map[string]interface{}) error {
	var value interface{}

	value = f["path"]
	if value != nil {
		self.path = value.(string)
	}

	value = f["format"]
	if value != nil {
		self.format = value.(string)
	}

	value = f["tag"]
	if value != nil {
		self.tag = value.(string)
	}

	value = f["pos_file"]
	if value != nil {
		self.pos_file = value.(string)

		str, err := ioutil.ReadFile(self.pos_file)
		if err != nil {
			Log(err)
		}

		f, err := os.Open(self.path)
		if err != nil {
			Log(err)
			return err
		}

		info, err := f.Stat()
		if err != nil {
			Log(err)
			return err
		}

		offset, _ := strconv.Atoi(string(str))
		if int64(offset) > info.Size() {
			self.offset = info.Size()
		} else {
			self.offset = int64(offset)
		}
	}

	return nil
}

func (self *inputTail) start(ctx chan Context) error {

	var seek int
	if self.offset > 0 {
		seek = os.SEEK_SET
	} else {
		seek = os.SEEK_END
	}

	t, err := tail.TailFile(self.path, tail.Config{
		Poll:      true,
		ReOpen:    true,
		Follow:    true,
		MustExist: false,
		Location:  &tail.SeekInfo{int64(self.offset), seek}})
	if err != nil {
		return err
	}

	var re regexp.Regexp
	if string(self.format[0]) == string("/") || string(self.format[len(self.format)-1]) == string("/") {
		fmt.Println(strings.Trim(self.format, "/"))
		re = *regexp.MustCompile(self.format)
		self.format = "regexp"
	} else if self.format == "json" {

	}

	for line := range t.Lines {

		timeUnix := line.Time.Unix()
		data := make(map[string]string)

		if self.format == "regexp" {
			text := re.FindSubmatch([]byte(line.Text))
			if text == nil {
				continue
			}

			for i, name := range re.SubexpNames() {
				if i != 0 {
					Log(name, string(text[i]))
					data[name] = string(text[i])
				}
			}
		} else if self.format == "json" {
			err := json.Unmarshal([]byte(line.Text), &data)
			if err != nil {
				continue
			}
		}

		offset, err := t.Tell()
		if err != nil {
			fmt.Println("Tell return error: ", err)
		}

		str := strconv.Itoa(int(offset))

		f, err := os.OpenFile(self.pos_file, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
		if err != nil {
			Log(err)
		}

		_, err = f.WriteString(str)
		if err != nil {
			Log(err)
		}

		f.Close()

		//Log("offset:", offset, "raw_string:", line.Text)

		record := Record{timeUnix, data}
		ctx <- Context{self.tag, record}
	}

	err = t.Wait()
	if err != nil {
		return err
	}

	return err
}

func init() {
	RegisterInput("tail", &inputTail{})
}
<|endoftext|>


<commit_before>package mantle

import (
	"github.com/garyburd/redigo/redis"
	"github.com/vireshas/minimal_vitess_pool/pools"
	"strconv"
	"strings"
	"time"
)

//cant make these guys const as []string is not allowed in consts

//default pool size
var RedisPoolSize = 10

//default host:port to connect
var DefaultRedisConfig = []string{"localhost:6379"}

/*
 * This method creates a redis connection
   Connect is passed as a callback to pools
   Params:
     Instance: This is a reference to a struct redis instance
               Connect needs some params like db, hostAndPorts
               These params are read from this instance rederence
*/
func CreateRedisConnection(Instance interface{}) (pools.Resource, error) {
	//converting interface Redis struct type
	redisInstance := Instance.(*Redis)
	//this is a string of type "localhost:6379"
	hostNPorts := redisInstance.Settings.HostAndPorts
	//select db after dialing
	db := redisInstance.db

	//panic is more than 1 ip is given
	if len(hostNPorts) > 1 {
		panic("we can only connect to 1 server at the moment")
	}

	hostNPort := strings.Split(hostNPorts[0], ":")
	//dial host and port
	cli, err := redis.Dial("tcp", hostNPort[0]+":"+hostNPort[1])
	if err != nil {
		panic(err)
	}

	//select a redis db
	_, err = cli.Do("SELECT", db)
	if err != nil {
		panic(err)
	}

	//typecast to RedisConn
	return &RedisConn{cli}, nil
}

/*
 * Wrapping redigo redis connection
   Pool expects a Object which defines
   Close() and doesn't return anything, but
   redigo.Redis#Close() returns error, hence this wrapper
   around redis.Conn
*/
type RedisConn struct {
	redis.Conn
}

//Close a redis connection
func (r *RedisConn) Close() {
	_ = r.Conn.Close()
}

//Gets a connection from pool and converts to RedisConn type
//If all the connections are in use, timeout the present request after a minute
func (r *Redis) GetClient() (*RedisConn, error) {
	connection, err := r.pool.GetConn(r.Settings.Timeout)
	if err != nil {
		return nil, err
	}
	return connection.(*RedisConn), nil
}

//Puts a connection back to pool
func (r *Redis) PutClient(c *RedisConn) {
	r.pool.PutConn(c)
}

type Redis struct {
	Settings PoolSettings
	pool     *ResourcePool
	db       int
}

//Add default settings if they are missing
func (r *Redis) SetDefaults() {
	if len(r.Settings.HostAndPorts) == 0 {
		r.Settings.HostAndPorts = DefaultRedisConfig
	}
	//this is poolsize
	if r.Settings.Capacity == 0 {
		r.Settings.Capacity = RedisPoolSize
	}
	//maxcapacity of the pool
	if r.Settings.MaxCapacity == 0 {
		r.Settings.MaxCapacity = RedisPoolSize
	}
	//pool timeout
	r.Settings.Timeout = time.Minute

	//select a particular db in redis
	db, ok := r.Settings.Options["db"]
	if !ok {
		db = "0"
	}
	select_db, err := strconv.Atoi(db)
	if err != nil {
		panic("From Redis: select db is not a valid string")
	}
	r.db = select_db

	//create a pool finally
	r.pool = NewPool(CreateRedisConnection, r, r.Settings)
}

func (r *Redis) Configure(settings PoolSettings) {
	r.Settings = settings
	r.SetDefaults()
}

//Generic method to execute any redis call
//Gets a client from pool, executes a cmd, puts conn back in pool
func (r *Redis) Execute(cmd string, args ...interface{}) (interface{}, error) {
	client, err := r.GetClient()
	if err != nil {
		return nil, err
	}
	defer r.PutClient(client)
	return client.Do(cmd, args...)
}

func (r *Redis) Delete(keys ...interface{}) int {
	value, err := redis.Int(r.Execute("DEL", keys...))
	if err != nil {
		return -1
	}
	return value
}

func (r *Redis) Get(key string) string {
	value, err := redis.String(r.Execute("GET", key))
	if err != nil {
		return ""
	}
	return value
}

func (r *Redis) Set(key string, value interface{}) bool {
	_, err := r.Execute("SET", key, value)
	if err != nil {
		return false
	}
	return true
}

func (r *Redis) MGet(keys ...interface{}) []string {
	values, err := redis.Strings(r.Execute("MGET", keys...))
	if err != nil {
		return []string{}
	}
	return values
}

func (r *Redis) MSet(mapOfKeyVal map[string]interface{}) bool {
	_, err := r.Execute("MSET", redis.Args{}.AddFlat(mapOfKeyVal)...)
	if err != nil {
		return false
	}
	return true
}

func (r *Redis) Expire(key string, duration int) bool {
	_, err := r.Execute("EXPIRE", key, duration)
	if err != nil {
		return false
	}
	return true
}

func (r *Redis) Setex(key string, duration int, val interface{}) bool {
	_, err := r.Execute("SETEX", key, duration, val)
	if err != nil {
		return false
	}
	return true
}
<commit_msg>comments<commit_after>package mantle

import (
	"github.com/garyburd/redigo/redis"
	"github.com/vireshas/minimal_vitess_pool/pools"
	"strconv"
	"strings"
	"time"
)

//cant make these guys const as []string is not allowed in consts

//default pool size
var RedisPoolSize = 10

//default host:port to connect
var DefaultRedisConfig = []string{"localhost:6379"}

//This method creates a redis connection
//CreateRedisConnection is passed as a callback to pools
//Instance:
//  This is a reference to a struct redis instance
//  Connect needs some params like db, hostAndPorts
//  These params are read from this instance rederence
func CreateRedisConnection(Instance interface{}) (pools.Resource, error) {
	//converting interface Redis struct type
	redisInstance := Instance.(*Redis)
	//this is a string of type "localhost:6379"
	hostNPorts := redisInstance.Settings.HostAndPorts
	//select db after dialing
	db := redisInstance.db

	//panic is more than 1 ip is given
	if len(hostNPorts) > 1 {
		panic("we can only connect to 1 server at the moment")
	}

	hostNPort := strings.Split(hostNPorts[0], ":")
	//dial host and port
	cli, err := redis.Dial("tcp", hostNPort[0]+":"+hostNPort[1])
	if err != nil {
		panic(err)
	}

	//select a redis db
	_, err = cli.Do("SELECT", db)
	if err != nil {
		panic(err)
	}

	//typecast to RedisConn
	return &RedisConn{cli}, nil
}

// Wrapping redigo redis connection
//   Pool expects a Object which defines
//   Close() and doesn't return anything, but
//   redigo.Redis#Close() returns error, hence this wrapper
//   around redis.Conn
type RedisConn struct {
	redis.Conn
}

//Close a redis connection
func (r *RedisConn) Close() {
	_ = r.Conn.Close()
}

//Gets a connection from pool and converts to RedisConn type
//If all the connections are in use, timeout the present request after a minute
func (r *Redis) GetClient() (*RedisConn, error) {
	connection, err := r.pool.GetConn(r.Settings.Timeout)
	if err != nil {
		return nil, err
	}
	return connection.(*RedisConn), nil
}

//Puts a connection back to pool
func (r *Redis) PutClient(c *RedisConn) {
	r.pool.PutConn(c)
}

type Redis struct {
	Settings PoolSettings
	pool     *ResourcePool
	db       int
}

//Add default settings if they are missing
func (r *Redis) SetDefaults() {
	if len(r.Settings.HostAndPorts) == 0 {
		r.Settings.HostAndPorts = DefaultRedisConfig
	}
	//this is poolsize
	if r.Settings.Capacity == 0 {
		r.Settings.Capacity = RedisPoolSize
	}
	//maxcapacity of the pool
	if r.Settings.MaxCapacity == 0 {
		r.Settings.MaxCapacity = RedisPoolSize
	}
	//pool timeout
	r.Settings.Timeout = time.Minute

	//select a particular db in redis
	db, ok := r.Settings.Options["db"]
	if !ok {
		db = "0"
	}
	select_db, err := strconv.Atoi(db)
	if err != nil {
		panic("From Redis: select db is not a valid string")
	}
	r.db = select_db

	//create a pool finally
	r.pool = NewPool(CreateRedisConnection, r, r.Settings)
}

func (r *Redis) Configure(settings PoolSettings) {
	r.Settings = settings
	r.SetDefaults()
}

//Generic method to execute any redis call
//Gets a client from pool, executes a cmd, puts conn back in pool
func (r *Redis) Execute(cmd string, args ...interface{}) (interface{}, error) {
	client, err := r.GetClient()
	if err != nil {
		return nil, err
	}
	defer r.PutClient(client)
	return client.Do(cmd, args...)
}

func (r *Redis) Delete(keys ...interface{}) int {
	value, err := redis.Int(r.Execute("DEL", keys...))
	if err != nil {
		return -1
	}
	return value
}

func (r *Redis) Get(key string) string {
	value, err := redis.String(r.Execute("GET", key))
	if err != nil {
		return ""
	}
	return value
}

func (r *Redis) Set(key string, value interface{}) bool {
	_, err := r.Execute("SET", key, value)
	if err != nil {
		return false
	}
	return true
}

func (r *Redis) MGet(keys ...interface{}) []string {
	values, err := redis.Strings(r.Execute("MGET", keys...))
	if err != nil {
		return []string{}
	}
	return values
}

func (r *Redis) MSet(mapOfKeyVal map[string]interface{}) bool {
	_, err := r.Execute("MSET", redis.Args{}.AddFlat(mapOfKeyVal)...)
	if err != nil {
		return false
	}
	return true
}

func (r *Redis) Expire(key string, duration int) bool {
	_, err := r.Execute("EXPIRE", key, duration)
	if err != nil {
		return false
	}
	return true
}

func (r *Redis) Setex(key string, duration int, val interface{}) bool {
	_, err := r.Execute("SETEX", key, duration, val)
	if err != nil {
		return false
	}
	return true
}
<|endoftext|>


<commit_before><commit_msg>in sender, notice relevant failed messages send to engine<commit_after><|endoftext|>


<commit_before>package job

import (
	"context"
	"github.com/pkg/errors"
	"github.com/problame/go-streamrpc"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/zrepl/zrepl/config"
	"github.com/zrepl/zrepl/daemon/job/reset"
	"github.com/zrepl/zrepl/daemon/job/wakeup"
	"github.com/zrepl/zrepl/daemon/transport/connecter"
	"github.com/zrepl/zrepl/daemon/filters"
	"github.com/zrepl/zrepl/daemon/pruner"
	"github.com/zrepl/zrepl/endpoint"
	"github.com/zrepl/zrepl/replication"
	"github.com/zrepl/zrepl/zfs"
	"sync"
	"github.com/zrepl/zrepl/daemon/logging"
	"github.com/zrepl/zrepl/daemon/snapper"
	"time"
)

type ActiveSide struct {
	mode          activeMode
	name          string
	clientFactory *connecter.ClientFactory

	prunerFactory *pruner.PrunerFactory

	promRepStateSecs *prometheus.HistogramVec // labels: state
	promPruneSecs *prometheus.HistogramVec // labels: prune_side
	promBytesReplicated *prometheus.CounterVec // labels: filesystem

	tasksMtx sync.Mutex
	tasks    activeSideTasks
}

type activeSideTasks struct {
	replication *replication.Replication
	prunerSender, prunerReceiver *pruner.Pruner
}

func (a *ActiveSide) updateTasks(u func(*activeSideTasks)) activeSideTasks {
	a.tasksMtx.Lock()
	var copy activeSideTasks
	copy = a.tasks
	a.tasksMtx.Unlock()
	if u == nil {
		return copy
	}
	u(&copy)
	a.tasksMtx.Lock()
	a.tasks = copy
	a.tasksMtx.Unlock()
	return copy
}

type activeMode interface {
	SenderReceiver(client *streamrpc.Client) (replication.Sender, replication.Receiver, error)
	Type() Type
	RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{})
}

type modePush struct {
	fsfilter         endpoint.FSFilter
	snapper *snapper.PeriodicOrManual
}

func (m *modePush) SenderReceiver(client *streamrpc.Client) (replication.Sender, replication.Receiver, error) {
	sender := endpoint.NewSender(m.fsfilter)
	receiver := endpoint.NewRemote(client)
	return sender, receiver, nil
}

func (m *modePush) Type() Type { return TypePush }

func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan <- struct{}) {
	m.snapper.Run(ctx, wakeUpCommon)
}

func modePushFromConfig(g *config.Global, in *config.PushJob) (*modePush, error) {
	m := &modePush{}
	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
	if err != nil {
		return nil, errors.Wrap(err, "cannnot build filesystem filter")
	}
	m.fsfilter = fsf

	if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
		return nil, errors.Wrap(err, "cannot build snapper")
	}

	return m, nil
}

type modePull struct {
	rootFS   *zfs.DatasetPath
	interval time.Duration
}

func (m *modePull) SenderReceiver(client *streamrpc.Client) (replication.Sender, replication.Receiver, error) {
	sender := endpoint.NewRemote(client)
	receiver, err := endpoint.NewReceiver(m.rootFS)
	return sender, receiver, err
}

func (*modePull) Type() Type { return TypePull }

func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
	t := time.NewTicker(m.interval)
	defer t.Stop()
	for {
		select {
		case <-t.C:
			select {
			case wakeUpCommon <- struct{}{}:
			default:
				GetLogger(ctx).
					WithField("pull_interval", m.interval).
					Warn("pull job took longer than pull interval")
				wakeUpCommon <- struct{}{} // block anyways, to queue up the wakeup
			}
		case <-ctx.Done():
			return
		}
	}
}

func modePullFromConfig(g *config.Global, in *config.PullJob) (m *modePull, err error) {
	m = &modePull{}
	if in.Interval <= 0 {
		return nil, errors.New("interval must be positive")
	}
	m.interval = in.Interval

	m.rootFS, err = zfs.NewDatasetPath(in.RootFS)
	if err != nil {
		return nil, errors.New("RootFS is not a valid zfs filesystem path")
	}
	if m.rootFS.Length() <= 0 {
		return nil, errors.New("RootFS must not be empty") // duplicates error check of receiver
	}

	return m, nil
}

func activeSide(g *config.Global, in *config.ActiveJob, mode activeMode) (j *ActiveSide, err error) {

	j = &ActiveSide{mode: mode}
	j.name = in.Name
	j.promRepStateSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace:   "zrepl",
		Subsystem:   "replication",
		Name:        "state_time",
		Help:        "seconds spent during replication",
		ConstLabels: prometheus.Labels{"zrepl_job":j.name},
	}, []string{"state"})
	j.promBytesReplicated = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace:   "zrepl",
		Subsystem:   "replication",
		Name:        "bytes_replicated",
		Help:        "number of bytes replicated from sender to receiver per filesystem",
		ConstLabels: prometheus.Labels{"zrepl_job":j.name},
	}, []string{"filesystem"})

	j.clientFactory, err = connecter.FromConfig(g, in.Connect)
	if err != nil {
		return nil, errors.Wrap(err, "cannot build client")
	}

	j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace:   "zrepl",
		Subsystem:   "pruning",
		Name:        "time",
		Help:        "seconds spent in pruner",
		ConstLabels: prometheus.Labels{"zrepl_job":j.name},
	}, []string{"prune_side"})
	j.prunerFactory, err = pruner.NewPrunerFactory(in.Pruning, j.promPruneSecs)
	if err != nil {
		return nil, err
	}

	return j, nil
}

func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) {
	registerer.MustRegister(j.promRepStateSecs)
	registerer.MustRegister(j.promPruneSecs)
	registerer.MustRegister(j.promBytesReplicated)
}

func (j *ActiveSide) Name() string { return j.name }

type ActiveSideStatus struct {
	Replication *replication.Report
	PruningSender, PruningReceiver *pruner.Report
}

func (j *ActiveSide) Status() *Status {
	tasks := j.updateTasks(nil)

	s := &ActiveSideStatus{}
	t := j.mode.Type()
	if tasks.replication != nil {
		s.Replication = tasks.replication.Report()
	}
	if tasks.prunerSender != nil {
		s.PruningSender = tasks.prunerSender.Report()
	}
	if tasks.prunerReceiver != nil {
		s.PruningReceiver = tasks.prunerReceiver.Report()
	}
	return &Status{Type: t, JobSpecific: s}
}

func (j *ActiveSide) Run(ctx context.Context) {
	log := GetLogger(ctx)
	ctx = logging.WithSubsystemLoggers(ctx, log)

	defer log.Info("job exiting")

	periodicDone := make(chan struct{})
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	go j.mode.RunPeriodic(ctx, periodicDone)

	invocationCount := 0
outer:
	for {
		log.Info("wait for wakeups")
		select {
		case <-ctx.Done():
			log.WithError(ctx.Err()).Info("context")
			break outer

		case <-wakeup.Wait(ctx):
		case <-periodicDone:
		}
		invocationCount++
		invLog := log.WithField("invocation", invocationCount)
		j.do(WithLogger(ctx, invLog))
	}
}

func (j *ActiveSide) do(ctx context.Context) {

	log := GetLogger(ctx)
	ctx = logging.WithSubsystemLoggers(ctx, log)

	// allow cancellation of an invocation (this function)
	ctx, cancelThisRun := context.WithCancel(ctx)
	defer cancelThisRun()
	runDone := make(chan struct{})
	defer close(runDone)
	go func() {
		select {
		case <-runDone:
		case <-reset.Wait(ctx):
			log.Info("reset received, cancelling current invocation")
			cancelThisRun()
		case <-ctx.Done():
		}
	}()

	client, err := j.clientFactory.NewClient()
	if err != nil {
		log.WithError(err).Error("factory cannot instantiate streamrpc client")
	}
	defer client.Close(ctx)

	sender, receiver, err := j.mode.SenderReceiver(client)

	tasks := j.updateTasks(func(tasks *activeSideTasks) {
		// reset it
		*tasks = activeSideTasks{}
		tasks.replication = replication.NewReplication(j.promRepStateSecs, j.promBytesReplicated)
	})

	log.Info("start replication")
	tasks.replication.Drive(ctx, sender, receiver)

	tasks = j.updateTasks(func(tasks *activeSideTasks) {
		tasks.prunerSender = j.prunerFactory.BuildSenderPruner(ctx, sender, sender)
		tasks.prunerReceiver = j.prunerFactory.BuildReceiverPruner(ctx, receiver, sender)
	})
	log.Info("start pruning sender")
	tasks.prunerSender.Prune()
	log.Info("start pruning receiver")
	tasks.prunerReceiver.Prune()
}
<commit_msg>daemon/job/active: fix race in updateTasks<commit_after>package job

import (
	"context"
	"github.com/pkg/errors"
	"github.com/problame/go-streamrpc"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/zrepl/zrepl/config"
	"github.com/zrepl/zrepl/daemon/job/reset"
	"github.com/zrepl/zrepl/daemon/job/wakeup"
	"github.com/zrepl/zrepl/daemon/transport/connecter"
	"github.com/zrepl/zrepl/daemon/filters"
	"github.com/zrepl/zrepl/daemon/pruner"
	"github.com/zrepl/zrepl/endpoint"
	"github.com/zrepl/zrepl/replication"
	"github.com/zrepl/zrepl/zfs"
	"sync"
	"github.com/zrepl/zrepl/daemon/logging"
	"github.com/zrepl/zrepl/daemon/snapper"
	"time"
)

type ActiveSide struct {
	mode          activeMode
	name          string
	clientFactory *connecter.ClientFactory

	prunerFactory *pruner.PrunerFactory

	promRepStateSecs *prometheus.HistogramVec // labels: state
	promPruneSecs *prometheus.HistogramVec // labels: prune_side
	promBytesReplicated *prometheus.CounterVec // labels: filesystem

	tasksMtx sync.Mutex
	tasks    activeSideTasks
}

type activeSideTasks struct {
	replication *replication.Replication
	prunerSender, prunerReceiver *pruner.Pruner
}

func (a *ActiveSide) updateTasks(u func(*activeSideTasks)) activeSideTasks {
	a.tasksMtx.Lock()
	defer a.tasksMtx.Unlock()
	var copy activeSideTasks
	copy = a.tasks
	if u == nil {
		return copy
	}
	u(&copy)
	a.tasks = copy
	return copy
}

type activeMode interface {
	SenderReceiver(client *streamrpc.Client) (replication.Sender, replication.Receiver, error)
	Type() Type
	RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{})
}

type modePush struct {
	fsfilter         endpoint.FSFilter
	snapper *snapper.PeriodicOrManual
}

func (m *modePush) SenderReceiver(client *streamrpc.Client) (replication.Sender, replication.Receiver, error) {
	sender := endpoint.NewSender(m.fsfilter)
	receiver := endpoint.NewRemote(client)
	return sender, receiver, nil
}

func (m *modePush) Type() Type { return TypePush }

func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan <- struct{}) {
	m.snapper.Run(ctx, wakeUpCommon)
}

func modePushFromConfig(g *config.Global, in *config.PushJob) (*modePush, error) {
	m := &modePush{}
	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
	if err != nil {
		return nil, errors.Wrap(err, "cannnot build filesystem filter")
	}
	m.fsfilter = fsf

	if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
		return nil, errors.Wrap(err, "cannot build snapper")
	}

	return m, nil
}

type modePull struct {
	rootFS   *zfs.DatasetPath
	interval time.Duration
}

func (m *modePull) SenderReceiver(client *streamrpc.Client) (replication.Sender, replication.Receiver, error) {
	sender := endpoint.NewRemote(client)
	receiver, err := endpoint.NewReceiver(m.rootFS)
	return sender, receiver, err
}

func (*modePull) Type() Type { return TypePull }

func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
	t := time.NewTicker(m.interval)
	defer t.Stop()
	for {
		select {
		case <-t.C:
			select {
			case wakeUpCommon <- struct{}{}:
			default:
				GetLogger(ctx).
					WithField("pull_interval", m.interval).
					Warn("pull job took longer than pull interval")
				wakeUpCommon <- struct{}{} // block anyways, to queue up the wakeup
			}
		case <-ctx.Done():
			return
		}
	}
}

func modePullFromConfig(g *config.Global, in *config.PullJob) (m *modePull, err error) {
	m = &modePull{}
	if in.Interval <= 0 {
		return nil, errors.New("interval must be positive")
	}
	m.interval = in.Interval

	m.rootFS, err = zfs.NewDatasetPath(in.RootFS)
	if err != nil {
		return nil, errors.New("RootFS is not a valid zfs filesystem path")
	}
	if m.rootFS.Length() <= 0 {
		return nil, errors.New("RootFS must not be empty") // duplicates error check of receiver
	}

	return m, nil
}

func activeSide(g *config.Global, in *config.ActiveJob, mode activeMode) (j *ActiveSide, err error) {

	j = &ActiveSide{mode: mode}
	j.name = in.Name
	j.promRepStateSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace:   "zrepl",
		Subsystem:   "replication",
		Name:        "state_time",
		Help:        "seconds spent during replication",
		ConstLabels: prometheus.Labels{"zrepl_job":j.name},
	}, []string{"state"})
	j.promBytesReplicated = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace:   "zrepl",
		Subsystem:   "replication",
		Name:        "bytes_replicated",
		Help:        "number of bytes replicated from sender to receiver per filesystem",
		ConstLabels: prometheus.Labels{"zrepl_job":j.name},
	}, []string{"filesystem"})

	j.clientFactory, err = connecter.FromConfig(g, in.Connect)
	if err != nil {
		return nil, errors.Wrap(err, "cannot build client")
	}

	j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace:   "zrepl",
		Subsystem:   "pruning",
		Name:        "time",
		Help:        "seconds spent in pruner",
		ConstLabels: prometheus.Labels{"zrepl_job":j.name},
	}, []string{"prune_side"})
	j.prunerFactory, err = pruner.NewPrunerFactory(in.Pruning, j.promPruneSecs)
	if err != nil {
		return nil, err
	}

	return j, nil
}

func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) {
	registerer.MustRegister(j.promRepStateSecs)
	registerer.MustRegister(j.promPruneSecs)
	registerer.MustRegister(j.promBytesReplicated)
}

func (j *ActiveSide) Name() string { return j.name }

type ActiveSideStatus struct {
	Replication *replication.Report
	PruningSender, PruningReceiver *pruner.Report
}

func (j *ActiveSide) Status() *Status {
	tasks := j.updateTasks(nil)

	s := &ActiveSideStatus{}
	t := j.mode.Type()
	if tasks.replication != nil {
		s.Replication = tasks.replication.Report()
	}
	if tasks.prunerSender != nil {
		s.PruningSender = tasks.prunerSender.Report()
	}
	if tasks.prunerReceiver != nil {
		s.PruningReceiver = tasks.prunerReceiver.Report()
	}
	return &Status{Type: t, JobSpecific: s}
}

func (j *ActiveSide) Run(ctx context.Context) {
	log := GetLogger(ctx)
	ctx = logging.WithSubsystemLoggers(ctx, log)

	defer log.Info("job exiting")

	periodicDone := make(chan struct{})
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	go j.mode.RunPeriodic(ctx, periodicDone)

	invocationCount := 0
outer:
	for {
		log.Info("wait for wakeups")
		select {
		case <-ctx.Done():
			log.WithError(ctx.Err()).Info("context")
			break outer

		case <-wakeup.Wait(ctx):
		case <-periodicDone:
		}
		invocationCount++
		invLog := log.WithField("invocation", invocationCount)
		j.do(WithLogger(ctx, invLog))
	}
}

func (j *ActiveSide) do(ctx context.Context) {

	log := GetLogger(ctx)
	ctx = logging.WithSubsystemLoggers(ctx, log)

	// allow cancellation of an invocation (this function)
	ctx, cancelThisRun := context.WithCancel(ctx)
	defer cancelThisRun()
	runDone := make(chan struct{})
	defer close(runDone)
	go func() {
		select {
		case <-runDone:
		case <-reset.Wait(ctx):
			log.Info("reset received, cancelling current invocation")
			cancelThisRun()
		case <-ctx.Done():
		}
	}()

	client, err := j.clientFactory.NewClient()
	if err != nil {
		log.WithError(err).Error("factory cannot instantiate streamrpc client")
	}
	defer client.Close(ctx)

	sender, receiver, err := j.mode.SenderReceiver(client)

	tasks := j.updateTasks(func(tasks *activeSideTasks) {
		// reset it
		*tasks = activeSideTasks{}
		tasks.replication = replication.NewReplication(j.promRepStateSecs, j.promBytesReplicated)
	})

	log.Info("start replication")
	tasks.replication.Drive(ctx, sender, receiver)

	tasks = j.updateTasks(func(tasks *activeSideTasks) {
		tasks.prunerSender = j.prunerFactory.BuildSenderPruner(ctx, sender, sender)
		tasks.prunerReceiver = j.prunerFactory.BuildReceiverPruner(ctx, receiver, sender)
	})
	log.Info("start pruning sender")
	tasks.prunerSender.Prune()
	log.Info("start pruning receiver")
	tasks.prunerReceiver.Prune()
}
<|endoftext|>


<commit_before>package processor

import (
	"bytes"
	"errors"
	"fmt"
	"testing"
)

var successfullJobs = []Message{
	Message{Payload: []byte("message 1")},
	Message{Payload: []byte("message 2")},
	Message{Payload: []byte("message 3")},
	Message{Payload: []byte("message 4")},
	Message{Payload: []byte("message 5")},
	Message{Payload: []byte("message 6")},
}

type dummyWriter struct{}

func (dw *dummyWriter) Write(p []byte) (n int, err error) {
	return 1, nil
}

func createProcessorAdapterMock(t testing.TB, messages []Message) *processorAdapterMock {
	return &processorAdapterMock{tb: t, messages: messages}
}

var failingJobs = []Message{
	Message{Payload: []byte("message for failing job 1")},
}

type processorAdapterFactoryMock struct {
	t *testing.T
}

func (s *processorAdapterMock) New(config *Config) Adapter {
	return createProcessorAdapterMock(s.tb, successfullJobs)
}

type processorAdapterMock struct {
	tb       testing.TB
	messages []Message
}

func (s *processorAdapterMock) Open() error {
	return nil
}

func (s *processorAdapterMock) Close() error {
	return nil
}

func (s *processorAdapterMock) Messages() (<-chan Message, error) {
	msgChannel := make(chan Message)
	go func() {
		for _, message := range s.messages {
			msgChannel <- message
		}
	}()
	return msgChannel, nil
}

func (s *processorAdapterMock) ResultHandler(jobResult *JobResult, message Message) error {
	if jobResult.Status != JobStatusSuccess {
		s.tb.Errorf("Running should be successful  status %d output %s", jobResult.Status, jobResult.Output)
		return fmt.Errorf("Running should be successful  status %d output %s", jobResult.Status, jobResult.Output)
	}
	return nil
}

type failProcessorAdapterMock struct {
	*processorAdapterMock
}

type failAdapterOpenCloseMock struct {
	*processorAdapterMock
}

func (s failAdapterOpenCloseMock) Open() error {
	return errors.New("Could not open the connection")
}

func (s failAdapterOpenCloseMock) Close() error {
	return errors.New("There was an erro closing the connection")
}

func (s *failProcessorAdapterMock) ResultHandler(jobResult *JobResult, message Message) error {
	if jobResult.Status != JobStatusFailed {
		s.tb.Errorf("Running job should be unsuccesful %s", jobResult.Output)
	}
	return nil
}

func TestProcessorSuccessfulJobs(t *testing.T) {

	processorAdapterMock := &processorAdapterMock{tb: t, messages: successfullJobs}
	processorConfig := &Config{
		Adapter:     processorAdapterMock,
		Command:     `echo "Hello successful test"`,
		CommandPath: ".",
		Concurrency: 1,
		WaitTimeout: 200,
	}
	var w bytes.Buffer
	processor := New(processorConfig, &w, &w)
	processor.Start()
}

func BenchmarkProcessorSuccessfulJobs(b *testing.B) {
	processorAdapterMock := &processorAdapterMock{tb: b, messages: successfullJobs}
	processorConfig := &Config{
		Adapter:     processorAdapterMock,
		Command:     `echo "Hello successful test"`,
		CommandPath: ".",
		Concurrency: 1,
		WaitTimeout: 200,
	}

	var w bytes.Buffer
	processor := New(processorConfig, &w, &w)
	for n := 0; n < b.N; n++ {
		processor.Start()
	}
}

func TestProcessorFailingJob(t *testing.T) {
	adapterMock := createProcessorAdapterMock(t, failingJobs)
	failStreamAdapterMock := &failProcessorAdapterMock{adapterMock}

	processorConfig := &Config{
		Adapter:     failStreamAdapterMock,
		Command:     `cd nonexistingdir"`,
		CommandPath: ".",
		Concurrency: 1,
		WaitTimeout: 200,
	}
	w := &dummyWriter{}
	processor := New(processorConfig, w, w)
	processor.Start()
}

func BenchmarkProcessorFailedJobs(b *testing.B) {
	adapterMock := createProcessorAdapterMock(b, failingJobs)
	failStreamAdapterMock := &failProcessorAdapterMock{adapterMock}

	processorConfig := &Config{
		Adapter:     failStreamAdapterMock,
		Command:     `cd nonexistingdir"`,
		CommandPath: ".",
		Concurrency: 1,
		WaitTimeout: 200,
	}
	w := &dummyWriter{}
	processor := New(processorConfig, w, w)
	for n := 0; n < b.N; n++ {
		processor.Start()
	}
}

func TestNewMessage(t *testing.T) {
	messageStr := "hello world"
	m := Message{[]byte(messageStr), nil}

	if string(m.Payload) != messageStr {
		t.Fatalf("Message should be %s was %s", messageStr, string(m.Payload))
	}
}

func TestAdapterOpenError(t *testing.T) {
	adapterMock := createProcessorAdapterMock(t, successfullJobs)
	failAdapterOpenCloseMock := &failAdapterOpenCloseMock{adapterMock}
	processorConfig := &Config{
		Adapter: failAdapterOpenCloseMock,
	}
	sp := New(processorConfig, nil, nil)
	err := sp.Start()
	if err == nil {
		t.Error("Expected connection open to fail")
	}
}

func TestRegisterAdapterFactory(t *testing.T) {

	cs := &AdapterConfigurationSchema{
		Name: "test",
	}

	err := RegisterAdapterFactory(nil, cs)

	if err != nil {
		t.Errorf("The first factory should be registered correctly for %s", cs.Name)
	}

	err = RegisterAdapterFactory(nil, cs)

	if err == nil {
		t.Errorf("The registration should fail for %s", cs.Name)
	}
}

func TestGetConfigurationSchemas(t *testing.T) {
	RegisterAdapterFactory(nil, &AdapterConfigurationSchema{
		Name: "test",
	})

	schemas := AdapterSchemas()

	slen := len(schemas)
	if slen != 1 {
		t.Errorf("Expected schemas size %d was", slen)
	}
}

func TestGetProcessorAdapterSchema(t *testing.T) {
	RegisterAdapterFactory(nil, &AdapterConfigurationSchema{Name: "test"})

	schema, _ := AdapterSchema("test")

	if schema.Name != "test" {
		t.Errorf("expected schema name to be 'test' was %s", schema.Name)
	}
}
<commit_msg>#2 additional assertions to ResultHandler<commit_after>package processor

import (
	"bytes"
	"encoding/base64"
	"errors"
	"fmt"
	"strings"
	"testing"
)

var successfullJobs = []Message{
	Message{Payload: []byte("message 1")},
	Message{Payload: []byte("message 2")},
	Message{Payload: []byte("message 3")},
	Message{Payload: []byte("message 4")},
	Message{Payload: []byte("message 5")},
	Message{Payload: []byte("message 6")},
}

type dummyWriter struct{}

func (dw *dummyWriter) Write(p []byte) (n int, err error) {
	return 1, nil
}

func createProcessorAdapterMock(t testing.TB, messages []Message) *processorAdapterMock {
	return &processorAdapterMock{tb: t, messages: messages}
}

var failingJobs = []Message{
	Message{Payload: []byte("message for failing job 1")},
}

type processorAdapterFactoryMock struct {
	t *testing.T
}

func (s *processorAdapterMock) New(config *Config) Adapter {
	return createProcessorAdapterMock(s.tb, successfullJobs)
}

type processorAdapterMock struct {
	tb       testing.TB
	messages []Message
}

func (s *processorAdapterMock) Open() error {
	return nil
}

func (s *processorAdapterMock) Close() error {
	return nil
}

func (s *processorAdapterMock) Messages() (<-chan Message, error) {
	msgChannel := make(chan Message)
	go func() {
		for _, message := range s.messages {
			msgChannel <- message
		}
	}()
	return msgChannel, nil
}

func (s *processorAdapterMock) ResultHandler(jobResult *JobResult, message Message) error {
	if jobResult.Status != JobStatusSuccess {
		s.tb.Errorf("Running should be successful  status %d output %s", jobResult.Status, jobResult.Output)
		return fmt.Errorf("Running should be successful  status %d output %s", jobResult.Status, jobResult.Output)
	}
	originalMessage := base64.StdEncoding.EncodeToString(message.Payload)
	outputMessage := string(jobResult.Output)
	if !strings.Contains(outputMessage, originalMessage) {
		s.tb.Errorf("outputMessage => %s expected to contain %s", outputMessage, originalMessage)
	}
	return nil
}

type failProcessorAdapterMock struct {
	*processorAdapterMock
}

type failAdapterOpenCloseMock struct {
	*processorAdapterMock
}

func (s failAdapterOpenCloseMock) Open() error {
	return errors.New("Could not open the connection")
}

func (s failAdapterOpenCloseMock) Close() error {
	return errors.New("There was an erro closing the connection")
}

func (s *failProcessorAdapterMock) ResultHandler(jobResult *JobResult, message Message) error {
	if jobResult.Status != JobStatusFailed {
		s.tb.Errorf("Running job should be unsuccesful %s", jobResult.Output)
	}
	return nil
}

func TestProcessorSuccessfulJobs(t *testing.T) {

	processorAdapterMock := &processorAdapterMock{tb: t, messages: successfullJobs}
	processorConfig := &Config{
		Adapter:     processorAdapterMock,
		Command:     `echo "Hello successful test"`,
		CommandPath: ".",
		Concurrency: 1,
		WaitTimeout: 200,
	}
	var w bytes.Buffer
	processor := New(processorConfig, &w, &w)
	processor.Start()
}

func BenchmarkProcessorSuccessfulJobs(b *testing.B) {
	processorAdapterMock := &processorAdapterMock{tb: b, messages: successfullJobs}
	processorConfig := &Config{
		Adapter:     processorAdapterMock,
		Command:     `echo "Hello successful test"`,
		CommandPath: ".",
		Concurrency: 1,
		WaitTimeout: 200,
	}

	var w bytes.Buffer
	processor := New(processorConfig, &w, &w)
	for n := 0; n < b.N; n++ {
		processor.Start()
	}
}

func TestProcessorFailingJob(t *testing.T) {
	adapterMock := createProcessorAdapterMock(t, failingJobs)
	failStreamAdapterMock := &failProcessorAdapterMock{adapterMock}

	processorConfig := &Config{
		Adapter:     failStreamAdapterMock,
		Command:     `cd nonexistingdir"`,
		CommandPath: ".",
		Concurrency: 1,
		WaitTimeout: 200,
	}
	w := &dummyWriter{}
	processor := New(processorConfig, w, w)
	processor.Start()
}

func BenchmarkProcessorFailedJobs(b *testing.B) {
	adapterMock := createProcessorAdapterMock(b, failingJobs)
	failStreamAdapterMock := &failProcessorAdapterMock{adapterMock}

	processorConfig := &Config{
		Adapter:     failStreamAdapterMock,
		Command:     `cd nonexistingdir"`,
		CommandPath: ".",
		Concurrency: 1,
		WaitTimeout: 200,
	}
	w := &dummyWriter{}
	processor := New(processorConfig, w, w)
	for n := 0; n < b.N; n++ {
		processor.Start()
	}
}

func TestNewMessage(t *testing.T) {
	messageStr := "hello world"
	m := Message{[]byte(messageStr), nil}

	if string(m.Payload) != messageStr {
		t.Fatalf("Message should be %s was %s", messageStr, string(m.Payload))
	}
}

func TestAdapterOpenError(t *testing.T) {
	adapterMock := createProcessorAdapterMock(t, successfullJobs)
	failAdapterOpenCloseMock := &failAdapterOpenCloseMock{adapterMock}
	processorConfig := &Config{
		Adapter: failAdapterOpenCloseMock,
	}
	sp := New(processorConfig, nil, nil)
	err := sp.Start()
	if err == nil {
		t.Error("Expected connection open to fail")
	}
}

func TestRegisterAdapterFactory(t *testing.T) {

	cs := &AdapterConfigurationSchema{
		Name: "test",
	}

	err := RegisterAdapterFactory(nil, cs)

	if err != nil {
		t.Errorf("The first factory should be registered correctly for %s", cs.Name)
	}

	err = RegisterAdapterFactory(nil, cs)

	if err == nil {
		t.Errorf("The registration should fail for %s", cs.Name)
	}
}

func TestGetConfigurationSchemas(t *testing.T) {
	RegisterAdapterFactory(nil, &AdapterConfigurationSchema{
		Name: "test",
	})

	schemas := AdapterSchemas()

	slen := len(schemas)
	if slen != 1 {
		t.Errorf("Expected schemas size %d was", slen)
	}
}

func TestGetProcessorAdapterSchema(t *testing.T) {
	RegisterAdapterFactory(nil, &AdapterConfigurationSchema{Name: "test"})

	schema, _ := AdapterSchema("test")

	if schema.Name != "test" {
		t.Errorf("expected schema name to be 'test' was %s", schema.Name)
	}
}
<|endoftext|>


<commit_before>package gps

import (
	"io/ioutil"
	"net/url"
	"reflect"
	"sort"
	"testing"
)

func TestGitVersionFetching(t *testing.T) {
	// This test is slowish, skip it on -short
	if testing.Short() {
		t.Skip("Skipping git source version fetching test in short mode")
	}

	cpath, err := ioutil.TempDir("", "smcache")
	if err != nil {
		t.Errorf("Failed to create temp dir: %s", err)
	}
	rf := func() {
		err := removeAll(cpath)
		if err != nil {
			t.Errorf("removeAll failed: %s", err)
		}
	}

	n := "github.com/Masterminds/VCSTestRepo"
	u, err := url.Parse("https://" + n)
	if err != nil {
		t.Errorf("URL was bad, lolwut? errtext: %s", err)
		rf()
		t.FailNow()
	}
	mb := maybeGitSource{
		n:   n,
		url: u,
	}

	isrc, err := mb.try(cpath, naiveAnalyzer{})
	if err != nil {
		t.Errorf("Unexpected error while setting up gitSource for test repo: %s", err)
		rf()
		t.FailNow()
	}
	src, ok := isrc.(*gitSource)
	if !ok {
		t.Errorf("Expected a gitSource, got a %T", isrc)
		rf()
		t.FailNow()
	}

	vlist, err := src.listVersions()
	if err != nil {
		t.Errorf("Unexpected error getting version pairs from git repo: %s", err)
		rf()
		t.FailNow()
	}

	if src.ex.s&existsUpstream != existsUpstream {
		t.Errorf("gitSource.listVersions() should have set the upstream existence bit for search")
	}
	if src.ex.f&existsUpstream != existsUpstream {
		t.Errorf("gitSource.listVersions() should have set the upstream existence bit for found")
	}
	if src.ex.s&existsInCache != 0 {
		t.Errorf("gitSource.listVersions() should not have set the cache existence bit for search")
	}
	if src.ex.f&existsInCache != 0 {
		t.Errorf("gitSource.listVersions() should not have set the cache existence bit for found")
	}

	if len(vlist) != 3 {
		t.Errorf("git test repo should've produced three versions, got %v: vlist was %s", len(vlist), vlist)
	} else {
		sort.Sort(upgradeVersionSorter(vlist))
		evl := []Version{
			NewVersion("1.0.0").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e")),
			NewBranch("master").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e")),
			NewBranch("test").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e")),
		}
		if !reflect.DeepEqual(vlist, evl) {
			t.Errorf("Version list was not what we expected:\n\t(GOT): %s\n\t(WNT): %s", vlist, evl)
		}
	}
}
<commit_msg>Add basic tests for bzrSource<commit_after>package gps

import (
	"io/ioutil"
	"net/url"
	"reflect"
	"sort"
	"testing"
)

func TestGitVersionFetching(t *testing.T) {
	// This test is slowish, skip it on -short
	if testing.Short() {
		t.Skip("Skipping git source version fetching test in short mode")
	}

	cpath, err := ioutil.TempDir("", "smcache")
	if err != nil {
		t.Errorf("Failed to create temp dir: %s", err)
	}
	rf := func() {
		err := removeAll(cpath)
		if err != nil {
			t.Errorf("removeAll failed: %s", err)
		}
	}

	n := "github.com/Masterminds/VCSTestRepo"
	u, err := url.Parse("https://" + n)
	if err != nil {
		t.Errorf("URL was bad, lolwut? errtext: %s", err)
		rf()
		t.FailNow()
	}
	mb := maybeGitSource{
		n:   n,
		url: u,
	}

	isrc, err := mb.try(cpath, naiveAnalyzer{})
	if err != nil {
		t.Errorf("Unexpected error while setting up gitSource for test repo: %s", err)
		rf()
		t.FailNow()
	}
	src, ok := isrc.(*gitSource)
	if !ok {
		t.Errorf("Expected a gitSource, got a %T", isrc)
		rf()
		t.FailNow()
	}

	vlist, err := src.listVersions()
	if err != nil {
		t.Errorf("Unexpected error getting version pairs from git repo: %s", err)
		rf()
		t.FailNow()
	}

	if src.ex.s&existsUpstream != existsUpstream {
		t.Errorf("gitSource.listVersions() should have set the upstream existence bit for search")
	}
	if src.ex.f&existsUpstream != existsUpstream {
		t.Errorf("gitSource.listVersions() should have set the upstream existence bit for found")
	}
	if src.ex.s&existsInCache != 0 {
		t.Errorf("gitSource.listVersions() should not have set the cache existence bit for search")
	}
	if src.ex.f&existsInCache != 0 {
		t.Errorf("gitSource.listVersions() should not have set the cache existence bit for found")
	}

	if len(vlist) != 3 {
		t.Errorf("git test repo should've produced three versions, got %v: vlist was %s", len(vlist), vlist)
	} else {
		sort.Sort(upgradeVersionSorter(vlist))
		evl := []Version{
			NewVersion("1.0.0").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e")),
			NewBranch("master").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e")),
			NewBranch("test").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e")),
		}
		if !reflect.DeepEqual(vlist, evl) {
			t.Errorf("Version list was not what we expected:\n\t(GOT): %s\n\t(WNT): %s", vlist, evl)
		}
	}
}

func TestBzrVersionFetching(t *testing.T) {
	// This test is quite slow (ugh bzr), so skip it on -short
	if testing.Short() {
		t.Skip("Skipping bzr source version fetching test in short mode")
	}

	cpath, err := ioutil.TempDir("", "smcache")
	if err != nil {
		t.Errorf("Failed to create temp dir: %s", err)
	}
	rf := func() {
		err := removeAll(cpath)
		if err != nil {
			t.Errorf("removeAll failed: %s", err)
		}
	}

	n := "launchpad.net/govcstestbzrrepo"
	u, err := url.Parse("https://" + n)
	if err != nil {
		t.Errorf("URL was bad, lolwut? errtext: %s", err)
		rf()
		t.FailNow()
	}
	mb := maybeBzrSource{
		n:   n,
		url: u,
	}

	isrc, err := mb.try(cpath, naiveAnalyzer{})
	if err != nil {
		t.Errorf("Unexpected error while setting up bzrSource for test repo: %s", err)
		rf()
		t.FailNow()
	}
	src, ok := isrc.(*bzrSource)
	if !ok {
		t.Errorf("Expected a bzrSource, got a %T", isrc)
		rf()
		t.FailNow()
	}

	vlist, err := src.listVersions()
	if err != nil {
		t.Errorf("Unexpected error getting version pairs from bzr repo: %s", err)
	}

	if src.ex.s&existsUpstream|existsInCache != existsUpstream|existsInCache {
		t.Errorf("bzrSource.listVersions() should have set the upstream and cache existence bits for search")
	}
	if src.ex.f&existsUpstream|existsInCache != existsUpstream|existsInCache {
		t.Errorf("bzrSource.listVersions() should have set the upstream and cache existence bits for found")
	}

	if len(vlist) != 1 {
		t.Errorf("bzr test repo should've produced one version, got %v", len(vlist))
	} else {
		v := NewVersion("1.0.0").Is(Revision("matt@mattfarina.com-20150731135137-pbphasfppmygpl68"))
		if vlist[0] != v {
			t.Errorf("bzr pair fetch reported incorrect first version, got %s", vlist[0])
		}
	}

	// Run again, this time to ensure cache outputs correctly
	vlist, err = src.listVersions()
	if err != nil {
		t.Errorf("Unexpected error getting version pairs from bzr repo: %s", err)
	}

	if src.ex.s&existsUpstream|existsInCache != existsUpstream|existsInCache {
		t.Errorf("bzrSource.listVersions() should have set the upstream and cache existence bits for search")
	}
	if src.ex.f&existsUpstream|existsInCache != existsUpstream|existsInCache {
		t.Errorf("bzrSource.listVersions() should have set the upstream and cache existence bits for found")
	}

	if len(vlist) != 1 {
		t.Errorf("bzr test repo should've produced one version, got %v", len(vlist))
	} else {
		v := NewVersion("1.0.0").Is(Revision("matt@mattfarina.com-20150731135137-pbphasfppmygpl68"))
		if vlist[0] != v {
			t.Errorf("bzr pair fetch reported incorrect first version, got %s", vlist[0])
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 The mccdaq developers. All rights reserved.
// Project site: https://github.com/gotmc/mccdaq
// Use of this source code is governed by a MIT-style license that
// can be found in the LICENSE.txt file for the project.

package main

import (
	"fmt"
	"log"
	"time"

	"github.com/gotmc/libusb"
	"github.com/gotmc/mccdaq/usb1608fsplus"
)

const millisecondDelay = 100

func main() {
	ctx, err := libusb.Init()
	if err != nil {
		log.Fatal("Couldn't create USB context. Ending now.")
	}
	defer ctx.Exit()

	// Create the USB-1608FS-Plus DAQ device
	daq, err := usb1608fsplus.NewViaSN(ctx, "01ACD31D")
	if err != nil {
		log.Fatalf("Something bad getting S/N happened: %s", err)
	}
	// If you just want to grab the first USB-1608FS-Plus that's attached, you
	// can use:
	// daq, err := usb1608fsplus.GetFirstDevice(ctx)

	// Print some info about the device
	log.Printf("Vendor ID = 0x%x / Product ID = 0x%x\n", daq.DeviceDescriptor.VendorID,
		daq.DeviceDescriptor.ProductID)
	serialNumber, err := daq.SerialNumber()
	log.Printf("Serial number via control transfer = %s", serialNumber)
	log.Printf("USB ConfigurationIndex = %d\n", daq.ConfigDescriptor.ConfigurationIndex)
	log.Printf("Bulk endpoint address = 0x%x (%b)\n",
		daq.BulkEndpoint.EndpointAddress, daq.BulkEndpoint.EndpointAddress)

	// Test blinking the LED
	numBlinks := 5
	actualBlinks, err := daq.BlinkLED(numBlinks)
	if err != nil {
		fmt.Errorf("Error blinking LED %s", err)
	}
	log.Printf("Sent %d byte of data to blink LED %d times.", actualBlinks, numBlinks)

	// Get status
	status, err := daq.Status()
	log.Printf("Status = %v", status)

	// Read the calibration memory to setup the gain table
	gainTable, _ := daq.BuildGainTable()
	log.Printf("Slope = %v\n", gainTable.Slope)
	log.Printf("Intercept = %v\n", gainTable.Intercept)

	/**************************
	* Start the Analog Scan   *
	**************************/

	// Create new analog input and ensure the scan is stopped and buffer cleared
	var frequency float64 = 5000.0
	ai := daq.NewAnalogInput(frequency)
	ai.StopScan()
	time.Sleep(millisecondDelay * time.Millisecond)
	ai.ClearScanBuffer()

	// Setup the analog input scan
	ai.TransferMode = usb1608fsplus.BlockTransfer
	ai.DebugMode = true
	ai.ConfigureChannel(0, true, 5, "Vin1")
	ai.ConfigureChannel(1, true, 5, "Vin2")
	ai.ConfigureChannel(2, true, 10, "Vin3")
	ai.ConfigureChannel(3, true, 10, "Vin4")
	ai.ConfigureChannel(4, true, 1, "Iin1")
	ai.ConfigureChannel(5, true, 1, "Iin2")
	ai.ConfigureChannel(6, true, 2, "Iin3")
	ai.ConfigureChannel(7, true, 2, "Iin4")
	ai.SetScanRanges()

	// Read the scan ranges
	time.Sleep(millisecondDelay * time.Millisecond)
	scanRanges, err := ai.ScanRanges()
	log.Printf("Ranges = %v\n", scanRanges)

	// Read the totalScans using splitScansIn number of scans
	const (
		scansPerBuffer = 256
		totalBuffers   = 10
	)
	ai.StartScan(0)
	for j := 0; j < totalBuffers; j++ {
		time.Sleep(millisecondDelay * time.Millisecond)
		data, err := ai.ReadScan(scansPerBuffer)
		if err != nil {
			// Stop the analog scan and close the DAQ
			ai.StopScan()
			time.Sleep(millisecondDelay * time.Millisecond)
			daq.Close()
			log.Fatalf("Error reading scan: %s", err)
		}
		// Print the first 8 bytes and the last 8 bytes of each read
		bytesToShow := 8
		for i := 0; i < bytesToShow; i += 2 {
			log.Printf("data[%d:%d] = 0x%02x%02x\n", i, i+1, data[i+1], data[i])
		}
		for i := len(data) - bytesToShow; i < len(data); i += 2 {
			log.Printf("data[%d:%d] = 0x%02x%02x\n", i, i+1, data[i+1], data[i])
		}
		log.Printf("Length of data is %d bytes\n", len(data))
	}
	// Stop the analog scan and close the DAQ
	time.Sleep(millisecondDelay * time.Millisecond)
	ai.StopScan()
	time.Sleep(millisecondDelay * time.Millisecond)
	daq.Close()
}
<commit_msg>Add timing analysis to example<commit_after>// Copyright (c) 2016 The mccdaq developers. All rights reserved.
// Project site: https://github.com/gotmc/mccdaq
// Use of this source code is governed by a MIT-style license that
// can be found in the LICENSE.txt file for the project.

package main

import (
	"fmt"
	"log"
	"time"

	"github.com/gotmc/libusb"
	"github.com/gotmc/mccdaq/usb1608fsplus"
)

const millisecondDelay = 100

func main() {
	ctx, err := libusb.Init()
	if err != nil {
		log.Fatal("Couldn't create USB context. Ending now.")
	}
	defer ctx.Exit()

	// Create the USB-1608FS-Plus DAQ device
	daq, err := usb1608fsplus.NewViaSN(ctx, "01ACD31D")
	if err != nil {
		log.Fatalf("Something bad getting S/N happened: %s", err)
	}
	// If you just want to grab the first USB-1608FS-Plus that's attached, you
	// can use:
	// daq, err := usb1608fsplus.GetFirstDevice(ctx)

	// Print some info about the device
	log.Printf("Vendor ID = 0x%x / Product ID = 0x%x\n", daq.DeviceDescriptor.VendorID,
		daq.DeviceDescriptor.ProductID)
	serialNumber, err := daq.SerialNumber()
	log.Printf("Serial number via control transfer = %s", serialNumber)
	log.Printf("USB ConfigurationIndex = %d\n", daq.ConfigDescriptor.ConfigurationIndex)
	log.Printf("Bulk endpoint address = 0x%x (%b)\n",
		daq.BulkEndpoint.EndpointAddress, daq.BulkEndpoint.EndpointAddress)

	// Test blinking the LED
	numBlinks := 5
	actualBlinks, err := daq.BlinkLED(numBlinks)
	if err != nil {
		fmt.Errorf("Error blinking LED %s", err)
	}
	log.Printf("Sent %d byte of data to blink LED %d times.", actualBlinks, numBlinks)

	// Get status
	status, err := daq.Status()
	log.Printf("Status = %v", status)

	// Read the calibration memory to setup the gain table
	gainTable, _ := daq.BuildGainTable()
	log.Printf("Slope = %v\n", gainTable.Slope)
	log.Printf("Intercept = %v\n", gainTable.Intercept)

	/**************************
	* Start the Analog Scan   *
	**************************/

	// Create new analog input and ensure the scan is stopped and buffer cleared
	var frequency float64 = 500.0
	ai := daq.NewAnalogInput(frequency)
	ai.StopScan()
	time.Sleep(millisecondDelay * time.Millisecond)
	ai.ClearScanBuffer()

	// Setup the analog input scan
	ai.TransferMode = usb1608fsplus.BlockTransfer
	ai.DebugMode = true
	ai.ConfigureEnableChannel(0, "5V", "Vin1")
	ai.ConfigureEnableChannel(1, "5V", "Vin2")
	ai.ConfigureEnableChannel(2, "10V", "Vin3")
	ai.ConfigureEnableChannel(3, "10V", "Vin4")
	ai.ConfigureEnableChannel(4, "1V", "Iin1")
	ai.ConfigureEnableChannel(5, "1V", "Iin2")
	ai.ConfigureEnableChannel(6, "2V", "Iin3")
	ai.ConfigureEnableChannel(7, "2V", "Iin4")
	ai.SetScanRanges()

	// Read the scan ranges
	time.Sleep(millisecondDelay * time.Millisecond)
	scanRanges, err := ai.ScanRanges()
	log.Printf("Ranges = %v\n", scanRanges)

	// Read the totalScans using splitScansIn number of scans
	const (
		scansPerBuffer = 256
		totalBuffers   = 10
	)
	expectedDuration := (scansPerBuffer * totalBuffers) / frequency
	ai.StartScan(0)
	start := time.Now()
	totalBytesRead := 0
	for j := 0; j < totalBuffers; j++ {
		// time.Sleep(millisecondDelay * time.Millisecond)
		data, err := ai.ReadScan(scansPerBuffer)
		totalBytesRead += len(data)
		if err != nil {
			// Stop the analog scan and close the DAQ
			ai.StopScan()
			time.Sleep(millisecondDelay * time.Millisecond)
			daq.Close()
			log.Fatalf("Error reading scan: %s", err)
		}
		// Print the first 8 bytes and the last 8 bytes of each read
		bytesToShow := 8
		for i := 0; i < bytesToShow; i += 2 {
			log.Printf("data[%d:%d] = 0x%02x%02x\n", i, i+1, data[i+1], data[i])
		}
		for i := len(data) - bytesToShow; i < len(data); i += 2 {
			log.Printf("data[%d:%d] = 0x%02x%02x\n", i, i+1, data[i+1], data[i])
		}
		log.Printf("Length of data is %d bytes\n", len(data))
	}
	elapsed := time.Since(start)
	log.Printf("Reading %d bytes took %.2f s", totalBytesRead, elapsed.Seconds())
	log.Printf("Anticipated reading %d bytes to take %.2f s",
		scansPerBuffer*totalBuffers*ai.NumEnabledChannels()*2, expectedDuration)
	// Stop the analog scan and close the DAQ
	time.Sleep(millisecondDelay * time.Millisecond)
	ai.StopScan()
	time.Sleep(millisecondDelay * time.Millisecond)
	daq.Close()
}
<|endoftext|>


<commit_before>//+build !windows

package utils

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/api/types"
	"github.com/patrickmn/go-cache"
	"github.com/pkg/errors"
	"github.com/rancher/agent/utilities/constants"
	"github.com/vishvananda/netlink"
	"github.com/vishvananda/netns"
)

const (
	cniLabels       = "io.rancher.cni.network"
	linkName        = "eth0"
	cniStateBaseDir = "/var/lib/rancher/state/cni"
)

func getIP(inspect types.ContainerJSON, c *cache.Cache) (string, error) {
	if inspect.Config.Labels[cniLabels] != "" && c != nil {
		cacheIP, ok := c.Get(inspect.Config.Labels[constants.UUIDLabel])
		if ok && InterfaceToString(cacheIP) == "error" {
			c.Delete(inspect.Config.Labels[constants.UUIDLabel])
			return "", errors.New("Timeout getting IP address")
		} else if ok {
			c.Delete(inspect.Config.Labels[constants.UUIDLabel])
			return InterfaceToString(cacheIP), nil
		}
		ip, err := lookUpIP(inspect)
		if err != nil {
			c.Add(inspect.Config.Labels[constants.UUIDLabel], "error", cache.DefaultExpiration)
			return "", err
		}
		c.Add(inspect.Config.Labels[constants.UUIDLabel], ip, cache.DefaultExpiration)
		return ip, nil
	}
	return inspect.NetworkSettings.IPAddress, nil
}

func lookUpIP(inspect types.ContainerJSON) (string, error) {
	// if container is stopped just return empty ip
	if inspect.State.Pid == 0 {
		return "", nil
	}
	endTime := time.Now().Add(30 * time.Second)
	initTime := 250 * time.Millisecond
	maxTime := 2 * time.Second
	for {
		if ip, cniError := getIPFromStateFile(inspect); ip != "" || cniError != "" {
			var err error
			if cniError != "" {
				err = errors.New(cniError)
			}
			return ip, err
		}

		ip, err := getIPForPID(inspect.State.Pid)
		if err != nil || ip != "" {
			return ip, err
		}

		logrus.Debugf("Sleeping %v (%v remaining) waiting for IP on %s", initTime, endTime.Sub(time.Now()), inspect.ID)
		time.Sleep(initTime)
		initTime = initTime * 2
		if initTime.Seconds() > maxTime.Seconds() {
			initTime = maxTime
		}
		if time.Now().After(endTime) {
			return "", errors.New("Timeout getting IP address")
		}
	}
}

func getIPFromStateFile(inspect types.ContainerJSON) (string, string) {
	if inspect.ID == "" || inspect.State == nil || inspect.State.StartedAt == "" {
		return "", ""
	}
	filename := path.Join(cniStateBaseDir, inspect.ID, inspect.State.StartedAt)

	data, err := ioutil.ReadFile(filename)
	if err != nil {
		if !os.IsNotExist(err) {
			logrus.Warnf("Error reading cni state file %v: %v. Falling back to container inspection logic.", filename, err)
		}
		return "", ""
	}

	var state cniState
	if err := json.Unmarshal(data, &state); err != nil {
		logrus.Warnf("Error unmarshalling cni state data %s: %v. Falling back to container inspection logic.", data, err)
		return "", ""
	}

	return state.IP4.IP, state.Error
}

type cniState struct {
	Error string
	IP4   struct {
		IP string
	}
}

func getIPForPID(pid int) (string, error) {
	nsHandler, err := netns.GetFromPid(pid)
	if err != nil {
		return "", err
	}
	defer nsHandler.Close()
	handler, err := netlink.NewHandleAt(nsHandler)
	if err != nil {
		return "", err
	}
	defer handler.Delete()
	link, err := handler.LinkByName(linkName)
	if err != nil {
		// Don't return error, it's expected this may fail until iface is created
		return "", nil
	}
	addrs, err := handler.AddrList(link, netlink.FAMILY_V4)
	if err != nil {
		return "", err
	}
	if len(addrs) > 0 {
		return addrs[0].IP.String(), nil
	}
	return "", nil
}
<commit_msg>Properly parse IP from cni state file<commit_after>//+build !windows

package utils

import (
	"encoding/json"
	"io/ioutil"
	"net"
	"os"
	"path"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/api/types"
	"github.com/patrickmn/go-cache"
	"github.com/pkg/errors"
	"github.com/rancher/agent/utilities/constants"
	"github.com/vishvananda/netlink"
	"github.com/vishvananda/netns"
)

const (
	cniLabels       = "io.rancher.cni.network"
	linkName        = "eth0"
	cniStateBaseDir = "/var/lib/rancher/state/cni"
)

func getIP(inspect types.ContainerJSON, c *cache.Cache) (string, error) {
	if inspect.Config.Labels[cniLabels] != "" && c != nil {
		cacheIP, ok := c.Get(inspect.Config.Labels[constants.UUIDLabel])
		if ok && InterfaceToString(cacheIP) == "error" {
			c.Delete(inspect.Config.Labels[constants.UUIDLabel])
			return "", errors.New("Timeout getting IP address")
		} else if ok {
			c.Delete(inspect.Config.Labels[constants.UUIDLabel])
			return InterfaceToString(cacheIP), nil
		}
		ip, err := lookUpIP(inspect)
		if err != nil {
			c.Add(inspect.Config.Labels[constants.UUIDLabel], "error", cache.DefaultExpiration)
			return "", err
		}
		c.Add(inspect.Config.Labels[constants.UUIDLabel], ip, cache.DefaultExpiration)
		return ip, nil
	}
	return inspect.NetworkSettings.IPAddress, nil
}

func lookUpIP(inspect types.ContainerJSON) (string, error) {
	// if container is stopped just return empty ip
	if inspect.State.Pid == 0 {
		return "", nil
	}
	endTime := time.Now().Add(30 * time.Second)
	initTime := 250 * time.Millisecond
	maxTime := 2 * time.Second
	for {
		if ip, cniError := getIPFromStateFile(inspect); cniError != nil {
			return "", cniError
		} else if ip != "" {
			return ip, nil
		}

		ip, err := getIPForPID(inspect.State.Pid)
		if err != nil || ip != "" {
			return ip, err
		}

		logrus.Debugf("Sleeping %v (%v remaining) waiting for IP on %s", initTime, endTime.Sub(time.Now()), inspect.ID)
		time.Sleep(initTime)
		initTime = initTime * 2
		if initTime.Seconds() > maxTime.Seconds() {
			initTime = maxTime
		}
		if time.Now().After(endTime) {
			return "", errors.New("Timeout getting IP address")
		}
	}
}

func getIPFromStateFile(inspect types.ContainerJSON) (string, error) {
	if inspect.ID == "" || inspect.State == nil || inspect.State.StartedAt == "" {
		return "", nil
	}
	filename := path.Join(cniStateBaseDir, inspect.ID, inspect.State.StartedAt)

	data, err := ioutil.ReadFile(filename)
	if err != nil {
		if !os.IsNotExist(err) {
			logrus.Warnf("Error reading cni state file %v: %v. Falling back to container inspection logic.", filename, err)
		}
		return "", nil
	}

	var state cniState
	if err := json.Unmarshal(data, &state); err != nil {
		logrus.Warnf("Error unmarshalling cni state data %s: %v. Falling back to container inspection logic.", data, err)
		return "", nil
	}

	if state.Error != "" {
		return "", errors.New(state.Error)
	}

	if state.IP4.IP != "" {
		ip, _, err := net.ParseCIDR(state.IP4.IP)
		if err != nil {
			return "", errors.Wrapf(err, "Unable to parse recorded IP address %v", state.IP4.IP)
		}
		return ip.String(), nil
	}
	return "", nil
}

type cniState struct {
	Error string
	IP4   struct {
		IP string
	}
}

func getIPForPID(pid int) (string, error) {
	nsHandler, err := netns.GetFromPid(pid)
	if err != nil {
		return "", err
	}
	defer nsHandler.Close()
	handler, err := netlink.NewHandleAt(nsHandler)
	if err != nil {
		return "", err
	}
	defer handler.Delete()
	link, err := handler.LinkByName(linkName)
	if err != nil {
		// Don't return error, it's expected this may fail until iface is created
		return "", nil
	}
	addrs, err := handler.AddrList(link, netlink.FAMILY_V4)
	if err != nil {
		return "", err
	}
	if len(addrs) > 0 {
		return addrs[0].IP.String(), nil
	}
	return "", nil
}
<|endoftext|>