<commit_before>package libkb

import (
	"fmt"
)

// NOTE: This file is the source of truth for our version number, but we have a
//       script that reads it at packaging/version.sh. If you refactor this
//       file, update that script.

// Version as MAJOR.MINOR.PATCH
const Version = "1.0.0"

// Build number
const Build = "36"

// VersionString returns semantic version string.
func VersionString() string {
	return fmt.Sprintf("%s-%s", Version, Build)
}
<commit_msg>New version v1.0.0-38<commit_after>package libkb

import (
	"fmt"
)

// NOTE: This file is the source of truth for our version number, but we have a
//       script that reads it at packaging/version.sh. If you refactor this
//       file, update that script.

// Version as MAJOR.MINOR.PATCH
const Version = "1.0.0"

// Build number
const Build = "38"

// VersionString returns semantic version string.
func VersionString() string {
	return fmt.Sprintf("%s-%s", Version, Build)
}
<|endoftext|>


<commit_before>// Copyright 2016 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package pvl

var hardcodedPVLString = `
{
  "pvl_version": 1,
  "revision": 1,
  "services": {
    "coinbase": [
      [
        { "fill": { "into": "our_url", "with": "https://coinbase.com/%{username_service}/public-key" } },
        { "fetch": { "from": "our_url", "kind": "html" } },
        {
          "selector_css": {
            "error": ["FAILED_PARSE", "Couldn't find a div $(pre.statement)"],
            "into": "haystack",
            "selectors": ["pre.statement", 0]
          }
        },
        {
          "assert_find_base64": { "haystack": "haystack", "needle": "sig" },
          "error": ["TEXT_NOT_FOUND", "signature not found in body"]
        }
      ]
    ],
    "dns": [
      [
        {
          "assert_regex_match": {
            "error": ["CONTENT_FAILURE", "matching DNS entry not found"],
            "from": "txt",
            "pattern": "^keybase-site-verification=%{sig_id_medium}$"
          }
        }
      ]
    ],
    "facebook": [
      [
        {
          "regex_capture": {
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should start with 'https://m.facebook.com/%{username_service}/posts/', received '%{hint_url}'"
            ],
            "from": "hint_url",
            "into": ["username_from_url"],
            "pattern": "^https://m\\.facebook\\.com/([^/]*)/posts/.*$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_url",
            "b": "username_service",
            "cmp": "stripdots-then-cicmp",
            "error": [
              "BAD_API_URL",
              "Bad hint from server; username in URL match '%{username_service}', received '%{username_from_url}'"
            ]
          }
        },
        { "fetch": { "from": "hint_url", "kind": "html" } },
        {
          "selector_css": {
            "attr": "href",
            "error": ["FAILED_PARSE", "Couldn't find username href"],
            "into": "username_link",
            "selectors": ["#m_story_permalink_view > div:first-child > div:first-child > div:first-child h3", 0, "a", 0]
          }
        },
        {
          "parse_url": {
            "error": ["FAILED_PARSE", "Failed to parse username URL: %{username_link}"],
            "from": "username_link",
            "path": "path"
          }
        },
        {
          "regex_capture": {
            "error": ["FAILED_PARSE", "Username URL has no path"],
            "from": "path",
            "into": ["split_path_1"],
            "pattern": "^[^/]*/([^/]*)$"
          }
        },
        {
          "assert_compare": {
            "a": "split_path_1",
            "b": "username_service",
            "cmp": "stripdots-then-cicmp",
            "error": ["BAD_USERNAME", "Usernames don't match '%{split_path_1}' vs '%{username_service}'"]
          }
        },
        {
          "selector_css": {
            "error": ["FAILED_PARSE", "Couldn't find proof text header"],
            "into": "header",
            "selectors": ["#m_story_permalink_view > div:first-child > div:first-child > div:first-child h3", 1]
          }
        },
        { "whitespace_normalize": { "from": "header", "into": "header_nw" } },
        {
          "assert_regex_match": {
            "error": ["TEXT_NOT_FOUND", "Proof text not found: '' != ''"],
            "from": "header_nw",
            "pattern": "^Verifying myself: I am %{username_keybase} on Keybase\\.io\\. %{sig_id_medium}$"
          }
        }
      ]
    ],
    "generic_web_site": [
      [
        {
          "assert_regex_match": {
            "error": ["BAD_API_URL", "Bad hint from server; didn't recognize API url: \"%{active_string}\""],
            "from": "hint_url",
            "pattern": "^%{protocol}://%{hostname}/(?:\\.well-known/keybase\\.txt|keybase\\.txt)$"
          }
        },
        { "fetch": { "from": "hint_url", "into": "blob", "kind": "string" } },
        {
          "assert_find_base64": { "error": ["TEXT_NOT_FOUND", "signature not found in body"], "haystack": "blob", "needle": "sig" }
        }
      ]
    ],
    "github": [
      [
        {
          "regex_capture": {
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should start with either https://gist.github.com OR https://gist.githubusercontent.com"
            ],
            "from": "hint_url",
            "into": ["username_from_url"],
            "pattern": "^https://gist\\.github(?:usercontent)?\\.com/([^/]*)/.*$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_url",
            "b": "username_service",
            "cmp": "cicmp",
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should contain username matching %{username_service}; got %{username_from_url}"
            ]
          }
        },
        { "fetch": { "from": "hint_url", "into": "haystack", "kind": "string" } },
        {
          "assert_find_base64": { "haystack": "haystack", "needle": "sig" },
          "error": ["TEXT_NOT_FOUND", "signature not found in body"]
        }
      ]
    ],
    "hackernews": [
      [
        {
          "regex_capture": {
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should match https://hacker-news.firebaseio.com/v0/user/%{username_service}/about.json"
            ],
            "from": "hint_url",
            "into": ["username_from_url"],
            "pattern": "^https://hacker-news\\.firebaseio\\.com/v0/user/([^/]+)/about.json$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_url",
            "b": "username_service",
            "cmp": "cicmp",
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should contain username matching %{username_service}; got %{username_from_url}"
            ]
          }
        },
        { "fetch": { "from": "hint_url", "into": "profile", "kind": "string" } },
        {
          "assert_regex_match": {
            "error": ["TEXT_NOT_FOUND", "Posted text does not include signature '%{sig_id_medium}'"],
            "from": "profile",
            "pattern": "^.*%{sig_id_medium}.*$"
          }
        }
      ]
    ],
    "reddit": [
      [
        {
          "regex_capture": {
            "error": ["BAD_API_URL", "URL should start with 'https://www.reddit.com/r/keybaseproofs'"],
            "from": "hint_url",
            "into": ["subreddit_from_url", "path_remainder"],
            "pattern": "^https://www.reddit.com/r/([^/]+)/(.*)$"
          }
        },
        {
          "assert_regex_match": {
            "case_insensitive": true,
            "error": ["BAD_API_URL", "URL contained wrong subreddit '%{subreddit_from_url}' !+ 'keybaseproofs'"],
            "from": "subreddit_from_url",
            "pattern": "^keybaseproofs$"
          }
        },
        { "fetch": { "from": "hint_url", "kind": "json" } },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find 'kind' in json"],
            "into": "kind",
            "selectors": [0, "kind"]
          }
        },
        {
          "assert_regex_match": {
            "error": ["CONTENT_FAILURE", "Wanted a post of type 'Listing', but got %{kind}"],
            "from": "kind",
            "pattern": "^Listing$"
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find inner 'kind' in json"],
            "into": "inner_kind",
            "selectors": [0, "data", "children", 0, "kind"]
          }
        },
        {
          "assert_regex_match": {
            "error": ["CONTENT_FAILURE", "Wanted a child of type 't3' but got %{inner_kind}"],
            "from": "inner_kind",
            "pattern": "^t3$"
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find 'subreddit' in json"],
            "into": "subreddit_from_json",
            "selectors": [0, "data", "children", 0, "data", "subreddit"]
          }
        },
        {
          "assert_regex_match": {
            "case_insensitive": true,
            "error": ["CONTENT_FAILURE", "Wrong subreddti %{subreddit_from_json}"],
            "from": "subreddit_from_json",
            "pattern": "^keybaseproofs$"
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find author in json"],
            "into": "author",
            "selectors": [0, "data", "children", 0, "data", "author"]
          }
        },
        {
          "assert_compare": {
            "a": "author",
            "b": "username_service",
            "cmp": "cicmp",
            "error": ["BAD_USERNAME", "Bad post author; wanted '%{username_service} but got '%{author}'"]
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find title in json"],
            "into": "title",
            "selectors": [0, "data", "children", 0, "data", "title"]
          }
        },
        {
          "assert_regex_match": {
            "error": ["TITLE_NOT_FOUND", "Missing signature ID (%{sig_id_medium})) in post title '%{title}'"],
            "from": "title",
            "pattern": "^.*%{sig_id_medium}.*$"
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find selftext in json"],
            "into": "selftext",
            "selectors": [0, "data", "children", 0, "data", "selftext"]
          }
        },
        {
          "assert_find_base64": {
            "error": ["TEXT_NOT_FOUND", "signature not found in body"],
            "haystack": "selftext",
            "needle": "sig"
          }
        }
      ]
    ],
    "rooter": [
      [
        {
          "assert_regex_match": {
            "case_insensitive": true,
            "pattern": "^https?://[\\w:_\\-\\.]+/_/api/1\\.0/rooter/%{username_service}/.*$"
          }
        },
        { "fetch": { "kind": "json" } },
        { "selector_json": { "into": "name", "selectors": ["status", "name"] } },
        { "assert_regex_match": { "case_insensitive": true, "from": "name", "pattern": "^ok$" } },
        { "selector_json": { "into": "post", "selectors": ["toot", "post"] } },
        { "assert_regex_match": { "from": "post", "pattern": "^.*%{sig_id_medium}.*$" } }
      ]
    ],
    "twitter": [
      [
        {
          "regex_capture": {
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should start with 'https://twitter.com/%{username_service}/'"
            ],
            "from": "hint_url",
            "into": ["username_from_url"],
            "pattern": "^https://twitter\\.com/([^/]+)/.*$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_url",
            "b": "username_service",
            "cmp": "cicmp",
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should contain username matching %{username_service}; got %{username_from_url}"
            ]
          }
        },
        { "fetch": { "from": "hint_url", "kind": "html" } },
        {
          "selector_css": {
            "attr": "data-screen-name",
            "error": ["FAILED_PARSE", "Couldn't find a div $(div.permalink-tweet-container div.permalink-tweet).eq(0)"],
            "into": "data_screen_name",
            "selectors": ["div.permalink-tweet-container div.permalink-tweet", 0]
          }
        },
        {
          "assert_compare": {
            "a": "data_screen_name",
            "b": "username_service",
            "cmp": "cicmp",
            "error": ["BAD_USERNAME", "Bad post authored: wanted ${username_service} but got %{data_screen_name}"]
          }
        },
        {
          "selector_css": {
            "error": ["CONTENT_MISSING", "Missing <div class='tweet-text'> container for tweet"],
            "into": "tweet_contents",
            "selectors": ["div.permalink-tweet-container div.permalink-tweet", 0, "p.tweet-text", 0]
          }
        },
        { "whitespace_normalize": { "from": "tweet_contents", "into": "tweet_contents_nw" } },
        {
          "regex_capture": {
            "error": [
              "DELETED",
              "Could not find 'Verifying myself: I am %{username_keybase} on Keybase.io. %{sig_id_short}'"
            ],
            "from": "tweet_contents_nw",
            "into": ["username_from_tweet_contents", "sig_from_tweet_contents"],
            "pattern": "^ *(?:@[a-zA-Z0-9_-]+\\s*)* *Verifying myself: I am ([A-Za-z0-9_]+) on Keybase\\.io\\. (\\S+) */.*$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_tweet_contents",
            "b": "username_keybase",
            "cmp": "cicmp",
            "error": [
              "BAD_USERNAME",
              "Wrong username in tweet '%{username_from_tweet_contents}' should be '%{username_keybase}'"
            ]
          }
        },
        {
          "assert_regex_match": {
            "error": ["TEXT_NOT_FOUND", "Could not find sig '%{sig_from_tweet_contents}' != '%{sig_id_short}'"],
            "from": "sig_from_tweet_contents",
            "pattern": "^%{sig_id_short}$"
          }
        }
      ]
    ]
  }
}
`

// GetHardcodedPvlString returns the unparsed pvl
func GetHardcodedPvlString() string {
	return hardcodedPVLString
}
<commit_msg>better error for facebook<commit_after>// Copyright 2016 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package pvl

var hardcodedPVLString = `
{
  "pvl_version": 1,
  "revision": 1,
  "services": {
    "coinbase": [
      [
        { "fill": { "into": "our_url", "with": "https://coinbase.com/%{username_service}/public-key" } },
        { "fetch": { "from": "our_url", "kind": "html" } },
        {
          "selector_css": {
            "error": ["FAILED_PARSE", "Couldn't find a div $(pre.statement)"],
            "into": "haystack",
            "selectors": ["pre.statement", 0]
          }
        },
        {
          "assert_find_base64": { "haystack": "haystack", "needle": "sig" },
          "error": ["TEXT_NOT_FOUND", "signature not found in body"]
        }
      ]
    ],
    "dns": [
      [
        {
          "assert_regex_match": {
            "error": ["NOT_FOUND", "matching DNS entry not found"],
            "from": "txt",
            "pattern": "^keybase-site-verification=%{sig_id_medium}$"
          }
        }
      ]
    ],
    "facebook": [
      [
        {
          "regex_capture": {
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should start with 'https://m.facebook.com/%{username_service}/posts/', received '%{hint_url}'"
            ],
            "from": "hint_url",
            "into": ["username_from_url"],
            "pattern": "^https://m\\.facebook\\.com/([^/]*)/posts/.*$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_url",
            "b": "username_service",
            "cmp": "stripdots-then-cicmp",
            "error": [
              "BAD_API_URL",
              "Bad hint from server; username in URL match '%{username_service}', received '%{username_from_url}'"
            ]
          }
        },
        { "fetch": { "from": "hint_url", "kind": "html" } },
        {
          "selector_css": {
            "error": ["FAILED_PARSE", "Couldn't find facebook post %{hint_url}. Is it deleted or private?"],
            "into": "unused",
            "multi": true,
            "selectors": ["#m_story_permalink_view"]
          }
        },
        {
          "selector_css": {
            "attr": "href",
            "error": ["FAILED_PARSE", "Couldn't find username href"],
            "into": "username_link",
            "selectors": ["#m_story_permalink_view > div:first-child > div:first-child > div:first-child h3", 0, "a", 0]
          }
        },
        {
          "parse_url": {
            "error": ["FAILED_PARSE", "Failed to parse username URL: %{username_link}"],
            "from": "username_link",
            "path": "path"
          }
        },
        {
          "regex_capture": {
            "error": ["FAILED_PARSE", "Username URL has no path"],
            "from": "path",
            "into": ["split_path_1"],
            "pattern": "^[^/]*/([^/]*)$"
          }
        },
        {
          "assert_compare": {
            "a": "split_path_1",
            "b": "username_service",
            "cmp": "stripdots-then-cicmp",
            "error": ["BAD_USERNAME", "Usernames don't match '%{split_path_1}' vs '%{username_service}'"]
          }
        },
        {
          "selector_css": {
            "error": ["FAILED_PARSE", "Couldn't find proof text header"],
            "into": "header",
            "selectors": ["#m_story_permalink_view > div:first-child > div:first-child > div:first-child h3", 1]
          }
        },
        { "whitespace_normalize": { "from": "header", "into": "header_nw" } },
        {
          "assert_regex_match": {
            "error": ["TEXT_NOT_FOUND", "Proof text not found: '' != ''"],
            "from": "header_nw",
            "pattern": "^Verifying myself: I am %{username_keybase} on Keybase\\.io\\. %{sig_id_medium}$"
          }
        }
      ]
    ],
    "generic_web_site": [
      [
        {
          "assert_regex_match": {
            "error": ["BAD_API_URL", "Bad hint from server; didn't recognize API url: \"%{active_string}\""],
            "from": "hint_url",
            "pattern": "^%{protocol}://%{hostname}/(?:\\.well-known/keybase\\.txt|keybase\\.txt)$"
          }
        },
        { "fetch": { "from": "hint_url", "into": "blob", "kind": "string" } },
        {
          "assert_find_base64": { "error": ["TEXT_NOT_FOUND", "signature not found in body"], "haystack": "blob", "needle": "sig" }
        }
      ]
    ],
    "github": [
      [
        {
          "regex_capture": {
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should start with either https://gist.github.com OR https://gist.githubusercontent.com"
            ],
            "from": "hint_url",
            "into": ["username_from_url"],
            "pattern": "^https://gist\\.github(?:usercontent)?\\.com/([^/]*)/.*$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_url",
            "b": "username_service",
            "cmp": "cicmp",
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should contain username matching %{username_service}; got %{username_from_url}"
            ]
          }
        },
        { "fetch": { "from": "hint_url", "into": "haystack", "kind": "string" } },
        {
          "assert_find_base64": { "haystack": "haystack", "needle": "sig" },
          "error": ["TEXT_NOT_FOUND", "signature not found in body"]
        }
      ]
    ],
    "hackernews": [
      [
        {
          "regex_capture": {
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should match https://hacker-news.firebaseio.com/v0/user/%{username_service}/about.json"
            ],
            "from": "hint_url",
            "into": ["username_from_url"],
            "pattern": "^https://hacker-news\\.firebaseio\\.com/v0/user/([^/]+)/about.json$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_url",
            "b": "username_service",
            "cmp": "cicmp",
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should contain username matching %{username_service}; got %{username_from_url}"
            ]
          }
        },
        { "fetch": { "from": "hint_url", "into": "profile", "kind": "string" } },
        {
          "assert_regex_match": {
            "error": ["TEXT_NOT_FOUND", "Posted text does not include signature '%{sig_id_medium}'"],
            "from": "profile",
            "pattern": "^.*%{sig_id_medium}.*$"
          }
        }
      ]
    ],
    "reddit": [
      [
        {
          "regex_capture": {
            "error": ["BAD_API_URL", "URL should start with 'https://www.reddit.com/r/keybaseproofs'"],
            "from": "hint_url",
            "into": ["subreddit_from_url", "path_remainder"],
            "pattern": "^https://www.reddit.com/r/([^/]+)/(.*)$"
          }
        },
        {
          "assert_regex_match": {
            "case_insensitive": true,
            "error": ["BAD_API_URL", "URL contained wrong subreddit '%{subreddit_from_url}' !+ 'keybaseproofs'"],
            "from": "subreddit_from_url",
            "pattern": "^keybaseproofs$"
          }
        },
        { "fetch": { "from": "hint_url", "kind": "json" } },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find 'kind' in json"],
            "into": "kind",
            "selectors": [0, "kind"]
          }
        },
        {
          "assert_regex_match": {
            "error": ["CONTENT_FAILURE", "Wanted a post of type 'Listing', but got %{kind}"],
            "from": "kind",
            "pattern": "^Listing$"
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find inner 'kind' in json"],
            "into": "inner_kind",
            "selectors": [0, "data", "children", 0, "kind"]
          }
        },
        {
          "assert_regex_match": {
            "error": ["CONTENT_FAILURE", "Wanted a child of type 't3' but got %{inner_kind}"],
            "from": "inner_kind",
            "pattern": "^t3$"
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find 'subreddit' in json"],
            "into": "subreddit_from_json",
            "selectors": [0, "data", "children", 0, "data", "subreddit"]
          }
        },
        {
          "assert_regex_match": {
            "case_insensitive": true,
            "error": ["CONTENT_FAILURE", "Wrong subreddti %{subreddit_from_json}"],
            "from": "subreddit_from_json",
            "pattern": "^keybaseproofs$"
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find author in json"],
            "into": "author",
            "selectors": [0, "data", "children", 0, "data", "author"]
          }
        },
        {
          "assert_compare": {
            "a": "author",
            "b": "username_service",
            "cmp": "cicmp",
            "error": ["BAD_USERNAME", "Bad post author; wanted '%{username_service} but got '%{author}'"]
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find title in json"],
            "into": "title",
            "selectors": [0, "data", "children", 0, "data", "title"]
          }
        },
        {
          "assert_regex_match": {
            "error": ["TITLE_NOT_FOUND", "Missing signature ID (%{sig_id_medium})) in post title '%{title}'"],
            "from": "title",
            "pattern": "^.*%{sig_id_medium}.*$"
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find selftext in json"],
            "into": "selftext",
            "selectors": [0, "data", "children", 0, "data", "selftext"]
          }
        },
        {
          "assert_find_base64": {
            "error": ["TEXT_NOT_FOUND", "signature not found in body"],
            "haystack": "selftext",
            "needle": "sig"
          }
        }
      ]
    ],
    "rooter": [
      [
        {
          "assert_regex_match": {
            "case_insensitive": true,
            "pattern": "^https?://[\\w:_\\-\\.]+/_/api/1\\.0/rooter/%{username_service}/.*$"
          }
        },
        { "fetch": { "kind": "json" } },
        { "selector_json": { "into": "name", "selectors": ["status", "name"] } },
        { "assert_regex_match": { "case_insensitive": true, "from": "name", "pattern": "^ok$" } },
        { "selector_json": { "into": "post", "selectors": ["toot", "post"] } },
        { "assert_regex_match": { "from": "post", "pattern": "^.*%{sig_id_medium}.*$" } }
      ]
    ],
    "twitter": [
      [
        {
          "regex_capture": {
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should start with 'https://twitter.com/%{username_service}/'"
            ],
            "from": "hint_url",
            "into": ["username_from_url"],
            "pattern": "^https://twitter\\.com/([^/]+)/.*$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_url",
            "b": "username_service",
            "cmp": "cicmp",
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should contain username matching %{username_service}; got %{username_from_url}"
            ]
          }
        },
        { "fetch": { "from": "hint_url", "kind": "html" } },
        {
          "selector_css": {
            "attr": "data-screen-name",
            "error": ["FAILED_PARSE", "Couldn't find a div $(div.permalink-tweet-container div.permalink-tweet).eq(0)"],
            "into": "data_screen_name",
            "selectors": ["div.permalink-tweet-container div.permalink-tweet", 0]
          }
        },
        {
          "assert_compare": {
            "a": "data_screen_name",
            "b": "username_service",
            "cmp": "cicmp",
            "error": ["BAD_USERNAME", "Bad post authored: wanted ${username_service} but got %{data_screen_name}"]
          }
        },
        {
          "selector_css": {
            "error": ["CONTENT_MISSING", "Missing <div class='tweet-text'> container for tweet"],
            "into": "tweet_contents",
            "selectors": ["div.permalink-tweet-container div.permalink-tweet", 0, "p.tweet-text", 0]
          }
        },
        { "whitespace_normalize": { "from": "tweet_contents", "into": "tweet_contents_nw" } },
        {
          "regex_capture": {
            "error": [
              "DELETED",
              "Could not find 'Verifying myself: I am %{username_keybase} on Keybase.io. %{sig_id_short}'"
            ],
            "from": "tweet_contents_nw",
            "into": ["username_from_tweet_contents", "sig_from_tweet_contents"],
            "pattern": "^ *(?:@[a-zA-Z0-9_-]+\\s*)* *Verifying myself: I am ([A-Za-z0-9_]+) on Keybase\\.io\\. (\\S+) */.*$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_tweet_contents",
            "b": "username_keybase",
            "cmp": "cicmp",
            "error": [
              "BAD_USERNAME",
              "Wrong username in tweet '%{username_from_tweet_contents}' should be '%{username_keybase}'"
            ]
          }
        },
        {
          "assert_regex_match": {
            "error": ["TEXT_NOT_FOUND", "Could not find sig '%{sig_from_tweet_contents}' != '%{sig_id_short}'"],
            "from": "sig_from_tweet_contents",
            "pattern": "^%{sig_id_short}$"
          }
        }
      ]
    ]
  }
}
`

// GetHardcodedPvlString returns the unparsed pvl
func GetHardcodedPvlString() string {
	return hardcodedPVLString
}
<|endoftext|>


<commit_before>package ddevapp

import (
	"fmt"
	"io/ioutil"

	"github.com/drud/ddev/pkg/fileutil"
	"github.com/drud/ddev/pkg/output"
	"github.com/drud/ddev/pkg/util"
	"os"
	"path/filepath"
)

const typo3AdditionalConfigTemplate = `<?php
/** ` + DdevFileSignature + `: Automatically generated TYPO3 AdditionalConfiguration.php file.
 ddev manages this file and may delete or overwrite the file unless this comment is removed.
 */

$GLOBALS['TYPO3_CONF_VARS']['DB']['Connections']['Default'] = array_merge($GLOBALS['TYPO3_CONF_VARS']['DB']['Connections']['Default'], [
                    'dbname' => 'db',
                    'host' => 'db',
                    'password' => 'db',
                    'port' => '3306',
                    'user' => 'db',
]);`

// createTypo3SettingsFile creates the app's LocalConfiguration.php and
// AdditionalConfiguration.php, adding things like database host, name, and
// password. Returns the fullpath to settings file and error
func createTypo3SettingsFile(app *DdevApp) (string, error) {

	if !fileutil.FileExists(app.SiteSettingsPath) {
		util.Warning("TYPO3 does not seem to have been set up yet, missing LocalConfiguration.php (%s)", app.SiteLocalSettingsPath)
	}

	settingsFilePath, err := app.DetermineSettingsPathLocation()
	if err != nil {
		return "", fmt.Errorf("Failed to get TYPO3 AdditionalConfiguration.php file path: %v", err.Error())
	}
	output.UserOut.Printf("Generating %s file for database connection.", filepath.Base(settingsFilePath))

	err = writeTypo3SettingsFile(app)
	if err != nil {
		return settingsFilePath, fmt.Errorf("Failed to write TYPO3 AdditionalConfiguration.php file: %v", err.Error())
	}

	return settingsFilePath, nil
}

// writeTypo3SettingsFile produces AdditionalConfiguration.php file
// It's assumed that the LocalConfiguration.php must already exist, and we're
// overriding the db config values in it.
func writeTypo3SettingsFile(app *DdevApp) error {

	filePath := app.SiteLocalSettingsPath

	// Ensure target directory is writable.
	dir := filepath.Dir(filePath)
	err := os.Chmod(dir, 0755)
	if err != nil {
		return err
	}

	file, err := os.Create(filePath)
	if err != nil {
		return err
	}
	contents := []byte(typo3AdditionalConfigTemplate)
	err = ioutil.WriteFile(filePath, contents, 0644)
	if err != nil {
		return err
	}
	util.CheckClose(file)
	return nil
}

// getTypo3UploadDir just returns a static upload files (public files) dir.
// This can be made more sophisticated in the future, for example by adding
// the directory to the ddev config.yaml.
func getTypo3UploadDir(app *DdevApp) string {
	// @todo: Check to see if this gets overridden in LocalConfiguration.php
	return "uploads"
}

// Typo3Hooks adds a TYPO3-specific hooks example for post-import-db
const Typo3Hooks = `
#  post-start:
#    - exec: "composer install -d /var/www/html"`

// getTypo3Hooks for appending as byte array
func getTypo3Hooks() []byte {
	// We don't have anything new to add yet.
	return []byte(Typo3Hooks)
}

// setTypo3SiteSettingsPaths sets the paths to settings.php/settings.local.php
// for templating.
func setTypo3SiteSettingsPaths(app *DdevApp) {
	settingsFileBasePath := filepath.Join(app.AppRoot, app.Docroot)
	var settingsFilePath, localSettingsFilePath string
	settingsFilePath = filepath.Join(settingsFileBasePath, "typo3conf", "LocalConfiguration.php")
	localSettingsFilePath = filepath.Join(settingsFileBasePath, "typo3conf", "AdditionalConfiguration.php")
	app.SiteSettingsPath = settingsFilePath
	app.SiteLocalSettingsPath = localSettingsFilePath
}

// isTypoApp returns true if the app is of type typo3
func isTypo3App(app *DdevApp) bool {
	if _, err := os.Stat(filepath.Join(app.AppRoot, app.Docroot, "typo3")); err == nil {
		return true
	}
	return false
}
<commit_msg>[BUGFIX] Trusted hosts pattern (#1) (#808)<commit_after>package ddevapp

import (
	"fmt"
	"io/ioutil"

	"github.com/drud/ddev/pkg/fileutil"
	"github.com/drud/ddev/pkg/output"
	"github.com/drud/ddev/pkg/util"
	"os"
	"path/filepath"
)

const typo3AdditionalConfigTemplate = `<?php
/** ` + DdevFileSignature + `: Automatically generated TYPO3 AdditionalConfiguration.php file.
 ddev manages this file and may delete or overwrite the file unless this comment is removed.
 */

$GLOBALS['TYPO3_CONF_VARS']['SYS']['trustedHostsPattern'] = '.*';

$GLOBALS['TYPO3_CONF_VARS']['DB']['Connections']['Default'] = array_merge($GLOBALS['TYPO3_CONF_VARS']['DB']['Connections']['Default'], [
                    'dbname' => 'db',
                    'host' => 'db',
                    'password' => 'db',
                    'port' => '3306',
                    'user' => 'db',
]);`

// createTypo3SettingsFile creates the app's LocalConfiguration.php and
// AdditionalConfiguration.php, adding things like database host, name, and
// password. Returns the fullpath to settings file and error
func createTypo3SettingsFile(app *DdevApp) (string, error) {

	if !fileutil.FileExists(app.SiteSettingsPath) {
		util.Warning("TYPO3 does not seem to have been set up yet, missing LocalConfiguration.php (%s)", app.SiteLocalSettingsPath)
	}

	settingsFilePath, err := app.DetermineSettingsPathLocation()
	if err != nil {
		return "", fmt.Errorf("Failed to get TYPO3 AdditionalConfiguration.php file path: %v", err.Error())
	}
	output.UserOut.Printf("Generating %s file for database connection.", filepath.Base(settingsFilePath))

	err = writeTypo3SettingsFile(app)
	if err != nil {
		return settingsFilePath, fmt.Errorf("Failed to write TYPO3 AdditionalConfiguration.php file: %v", err.Error())
	}

	return settingsFilePath, nil
}

// writeTypo3SettingsFile produces AdditionalConfiguration.php file
// It's assumed that the LocalConfiguration.php must already exist, and we're
// overriding the db config values in it.
func writeTypo3SettingsFile(app *DdevApp) error {

	filePath := app.SiteLocalSettingsPath

	// Ensure target directory is writable.
	dir := filepath.Dir(filePath)
	err := os.Chmod(dir, 0755)
	if err != nil {
		return err
	}

	file, err := os.Create(filePath)
	if err != nil {
		return err
	}
	contents := []byte(typo3AdditionalConfigTemplate)
	err = ioutil.WriteFile(filePath, contents, 0644)
	if err != nil {
		return err
	}
	util.CheckClose(file)
	return nil
}

// getTypo3UploadDir just returns a static upload files (public files) dir.
// This can be made more sophisticated in the future, for example by adding
// the directory to the ddev config.yaml.
func getTypo3UploadDir(app *DdevApp) string {
	// @todo: Check to see if this gets overridden in LocalConfiguration.php
	return "uploads"
}

// Typo3Hooks adds a TYPO3-specific hooks example for post-import-db
const Typo3Hooks = `
#  post-start:
#    - exec: "composer install -d /var/www/html"`

// getTypo3Hooks for appending as byte array
func getTypo3Hooks() []byte {
	// We don't have anything new to add yet.
	return []byte(Typo3Hooks)
}

// setTypo3SiteSettingsPaths sets the paths to settings.php/settings.local.php
// for templating.
func setTypo3SiteSettingsPaths(app *DdevApp) {
	settingsFileBasePath := filepath.Join(app.AppRoot, app.Docroot)
	var settingsFilePath, localSettingsFilePath string
	settingsFilePath = filepath.Join(settingsFileBasePath, "typo3conf", "LocalConfiguration.php")
	localSettingsFilePath = filepath.Join(settingsFileBasePath, "typo3conf", "AdditionalConfiguration.php")
	app.SiteSettingsPath = settingsFilePath
	app.SiteLocalSettingsPath = localSettingsFilePath
}

// isTypoApp returns true if the app is of type typo3
func isTypo3App(app *DdevApp) bool {
	if _, err := os.Stat(filepath.Join(app.AppRoot, app.Docroot, "typo3")); err == nil {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package engine

import (
	"bytes"
	"fmt"
	"path"
	"strings"
	"text/template"

	"github.com/Masterminds/sprig"

	"k8s.io/helm/pkg/chartutil"
	"k8s.io/helm/pkg/proto/hapi/chart"
)

// Engine is an implementation of 'cmd/tiller/environment'.Engine that uses Go templates.
type Engine struct {
	// FuncMap contains the template functions that will be passed to each
	// render call. This may only be modified before the first call to Render.
	FuncMap template.FuncMap
	// If strict is enabled, template rendering will fail if a template references
	// a value that was not passed in.
	Strict bool
}

// New creates a new Go template Engine instance.
//
// The FuncMap is initialized here. You may modify the FuncMap _prior to_ the
// first invocation of Render.
//
// The FuncMap sets all of the Sprig functions except for those that provide
// access to the underlying OS (env, expandenv).
func New() *Engine {
	f := FuncMap()
	return &Engine{
		FuncMap: f,
	}
}

// FuncMap returns a mapping of all of the functions that Engine has.
//
// Because some functions are late-bound (e.g. contain context-sensitive
// data), the functions may not all perform identically outside of an
// Engine as they will inside of an Engine.
//
// Known late-bound functions:
//
//	- "include": This is late-bound in Engine.Render(). The version
//	   included in the FuncMap is a placeholder.
//      - "required": This is late-bound in Engine.Render(). The version
//	   included in the FuncMap is a placeholder.
//      - "tpl": This is late-bound in Engine.Render(). The version
//	   included in the FuncMap is a placeholder.
func FuncMap() template.FuncMap {
	f := sprig.TxtFuncMap()
	delete(f, "env")
	delete(f, "expandenv")

	// Add some extra functionality
	extra := template.FuncMap{
		"toToml":   chartutil.ToToml,
		"toYaml":   chartutil.ToYaml,
		"fromYaml": chartutil.FromYaml,
		"toJson":   chartutil.ToJson,
		"fromJson": chartutil.FromJson,

		// This is a placeholder for the "include" function, which is
		// late-bound to a template. By declaring it here, we preserve the
		// integrity of the linter.
		"include":  func(string, interface{}) string { return "not implemented" },
		"required": func(string, interface{}) interface{} { return "not implemented" },
		"tpl":      func(string, interface{}) interface{} { return "not implemented" },
	}

	for k, v := range extra {
		f[k] = v
	}

	return f
}

// Render takes a chart, optional values, and value overrides, and attempts to render the Go templates.
//
// Render can be called repeatedly on the same engine.
//
// This will look in the chart's 'templates' data (e.g. the 'templates/' directory)
// and attempt to render the templates there using the values passed in.
//
// Values are scoped to their templates. A dependency template will not have
// access to the values set for its parent. If chart "foo" includes chart "bar",
// "bar" will not have access to the values for "foo".
//
// Values should be prepared with something like `chartutils.ReadValues`.
//
// Values are passed through the templates according to scope. If the top layer
// chart includes the chart foo, which includes the chart bar, the values map
// will be examined for a table called "foo". If "foo" is found in vals,
// that section of the values will be passed into the "foo" chart. And if that
// section contains a value named "bar", that value will be passed on to the
// bar chart during render time.
func (e *Engine) Render(chrt *chart.Chart, values chartutil.Values) (map[string]string, error) {
	// Render the charts
	tmap := allTemplates(chrt, values)
	return e.render(tmap)
}

// renderable is an object that can be rendered.
type renderable struct {
	// tpl is the current template.
	tpl string
	// vals are the values to be supplied to the template.
	vals chartutil.Values
	// namespace prefix to the templates of the current chart
	basePath string
}

// alterFuncMap takes the Engine's FuncMap and adds context-specific functions.
//
// The resulting FuncMap is only valid for the passed-in template.
func (e *Engine) alterFuncMap(t *template.Template) template.FuncMap {
	// Clone the func map because we are adding context-specific functions.
	var funcMap template.FuncMap = map[string]interface{}{}
	for k, v := range e.FuncMap {
		funcMap[k] = v
	}

	// Add the 'include' function here so we can close over t.
	funcMap["include"] = func(name string, data interface{}) string {
		buf := bytes.NewBuffer(nil)
		if err := t.ExecuteTemplate(buf, name, data); err != nil {
			buf.WriteString(err.Error())
		}
		return buf.String()
	}

	// Add the 'required' function here
	funcMap["required"] = func(warn string, val interface{}) (interface{}, error) {
		if val == nil {
			return val, fmt.Errorf(warn)
		} else if _, ok := val.(string); ok {
			if val == "" {
				return val, fmt.Errorf(warn)
			}
		}
		return val, nil
	}

	// Add the 'tpl' function here
	funcMap["tpl"] = func(tpl string, vals chartutil.Values) (string, error) {
		r := renderable{
			tpl:  tpl,
			vals: vals,
		}

		templates := map[string]renderable{}
		templates["template"] = r

		result, err := e.render(templates)
		if err != nil {
			return "", fmt.Errorf("Error during tpl function execution for %q: %s", tpl, err.Error())
		}
		return result["template"], nil
	}

	return funcMap
}

// render takes a map of templates/values and renders them.
func (e *Engine) render(tpls map[string]renderable) (map[string]string, error) {
	// Basically, what we do here is start with an empty parent template and then
	// build up a list of templates -- one for each file. Once all of the templates
	// have been parsed, we loop through again and execute every template.
	//
	// The idea with this process is to make it possible for more complex templates
	// to share common blocks, but to make the entire thing feel like a file-based
	// template engine.
	t := template.New("gotpl")
	if e.Strict {
		t.Option("missingkey=error")
	} else {
		// Not that zero will attempt to add default values for types it knows,
		// but will still emit <no value> for others. We mitigate that later.
		t.Option("missingkey=zero")
	}

	funcMap := e.alterFuncMap(t)

	files := []string{}
	for fname, r := range tpls {
		t = t.New(fname).Funcs(funcMap)
		if _, err := t.Parse(r.tpl); err != nil {
			return map[string]string{}, fmt.Errorf("parse error in %q: %s", fname, err)
		}
		files = append(files, fname)
	}

	rendered := make(map[string]string, len(files))
	var buf bytes.Buffer
	for _, file := range files {
		// At render time, add information about the template that is being rendered.
		vals := tpls[file].vals
		vals["Template"] = map[string]interface{}{"Name": file, "BasePath": tpls[file].basePath}
		if err := t.ExecuteTemplate(&buf, file, vals); err != nil {
			return map[string]string{}, fmt.Errorf("render error in %q: %s", file, err)
		}

		// Work around the issue where Go will emit "<no value>" even if Options(missing=zero)
		// is set. Since missing=error will never get here, we do not need to handle
		// the Strict case.
		rendered[file] = strings.Replace(buf.String(), "<no value>", "", -1)
		buf.Reset()
	}

	return rendered, nil
}

// allTemplates returns all templates for a chart and its dependencies.
//
// As it goes, it also prepares the values in a scope-sensitive manner.
func allTemplates(c *chart.Chart, vals chartutil.Values) map[string]renderable {
	templates := map[string]renderable{}
	recAllTpls(c, templates, vals, true, "")
	return templates
}

// recAllTpls recurses through the templates in a chart.
//
// As it recurses, it also sets the values to be appropriate for the template
// scope.
func recAllTpls(c *chart.Chart, templates map[string]renderable, parentVals chartutil.Values, top bool, parentID string) {
	// This should never evaluate to a nil map. That will cause problems when
	// values are appended later.
	cvals := chartutil.Values{}
	if top {
		// If this is the top of the rendering tree, assume that parentVals
		// is already resolved to the authoritative values.
		cvals = parentVals
	} else if c.Metadata != nil && c.Metadata.Name != "" {
		// If there is a {{.Values.ThisChart}} in the parent metadata,
		// copy that into the {{.Values}} for this template.
		newVals := chartutil.Values{}
		if vs, err := parentVals.Table("Values"); err == nil {
			if tmp, err := vs.Table(c.Metadata.Name); err == nil {
				newVals = tmp
			}
		}

		cvals = map[string]interface{}{
			"Values":       newVals,
			"Release":      parentVals["Release"],
			"Chart":        c.Metadata,
			"Files":        chartutil.NewFiles(c.Files),
			"Capabilities": parentVals["Capabilities"],
		}
	}

	newParentID := c.Metadata.Name
	if parentID != "" {
		// We artificially reconstruct the chart path to child templates. This
		// creates a namespaced filename that can be used to track down the source
		// of a particular template declaration.
		newParentID = path.Join(parentID, "charts", newParentID)
	}

	for _, child := range c.Dependencies {
		recAllTpls(child, templates, cvals, false, newParentID)
	}
	for _, t := range c.Templates {
		templates[path.Join(newParentID, t.Name)] = renderable{
			tpl:      string(t.Data),
			vals:     cvals,
			basePath: path.Join(newParentID, "templates"),
		}
	}
}
<commit_msg>Added storage of parsed templates in engine<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package engine

import (
	"bytes"
	"fmt"
	"path"
	"strings"
	"text/template"

	"github.com/Masterminds/sprig"

	"k8s.io/helm/pkg/chartutil"
	"k8s.io/helm/pkg/proto/hapi/chart"
)

// Engine is an implementation of 'cmd/tiller/environment'.Engine that uses Go templates.
type Engine struct {
	// FuncMap contains the template functions that will be passed to each
	// render call. This may only be modified before the first call to Render.
	FuncMap template.FuncMap
	// If strict is enabled, template rendering will fail if a template references
	// a value that was not passed in.
	Strict           bool
	CurrentTemplates map[string]renderable
}

// New creates a new Go template Engine instance.
//
// The FuncMap is initialized here. You may modify the FuncMap _prior to_ the
// first invocation of Render.
//
// The FuncMap sets all of the Sprig functions except for those that provide
// access to the underlying OS (env, expandenv).
func New() *Engine {
	f := FuncMap()
	return &Engine{
		FuncMap: f,
	}
}

// FuncMap returns a mapping of all of the functions that Engine has.
//
// Because some functions are late-bound (e.g. contain context-sensitive
// data), the functions may not all perform identically outside of an
// Engine as they will inside of an Engine.
//
// Known late-bound functions:
//
//	- "include": This is late-bound in Engine.Render(). The version
//	   included in the FuncMap is a placeholder.
//      - "required": This is late-bound in Engine.Render(). The version
//	   included in the FuncMap is a placeholder.
//      - "tpl": This is late-bound in Engine.Render(). The version
//	   included in the FuncMap is a placeholder.
func FuncMap() template.FuncMap {
	f := sprig.TxtFuncMap()
	delete(f, "env")
	delete(f, "expandenv")

	// Add some extra functionality
	extra := template.FuncMap{
		"toToml":   chartutil.ToToml,
		"toYaml":   chartutil.ToYaml,
		"fromYaml": chartutil.FromYaml,
		"toJson":   chartutil.ToJson,
		"fromJson": chartutil.FromJson,

		// This is a placeholder for the "include" function, which is
		// late-bound to a template. By declaring it here, we preserve the
		// integrity of the linter.
		"include":  func(string, interface{}) string { return "not implemented" },
		"required": func(string, interface{}) interface{} { return "not implemented" },
		"tpl":      func(string, interface{}) interface{} { return "not implemented" },
	}

	for k, v := range extra {
		f[k] = v
	}

	return f
}

// Render takes a chart, optional values, and value overrides, and attempts to render the Go templates.
//
// Render can be called repeatedly on the same engine.
//
// This will look in the chart's 'templates' data (e.g. the 'templates/' directory)
// and attempt to render the templates there using the values passed in.
//
// Values are scoped to their templates. A dependency template will not have
// access to the values set for its parent. If chart "foo" includes chart "bar",
// "bar" will not have access to the values for "foo".
//
// Values should be prepared with something like `chartutils.ReadValues`.
//
// Values are passed through the templates according to scope. If the top layer
// chart includes the chart foo, which includes the chart bar, the values map
// will be examined for a table called "foo". If "foo" is found in vals,
// that section of the values will be passed into the "foo" chart. And if that
// section contains a value named "bar", that value will be passed on to the
// bar chart during render time.
func (e *Engine) Render(chrt *chart.Chart, values chartutil.Values) (map[string]string, error) {
	// Render the charts
	tmap := allTemplates(chrt, values)
	e.CurrentTemplates = tmap
	return e.render(tmap)
}

// renderable is an object that can be rendered.
type renderable struct {
	// tpl is the current template.
	tpl string
	// vals are the values to be supplied to the template.
	vals chartutil.Values
	// namespace prefix to the templates of the current chart
	basePath string
}

// alterFuncMap takes the Engine's FuncMap and adds context-specific functions.
//
// The resulting FuncMap is only valid for the passed-in template.
func (e *Engine) alterFuncMap(t *template.Template) template.FuncMap {
	// Clone the func map because we are adding context-specific functions.
	var funcMap template.FuncMap = map[string]interface{}{}
	for k, v := range e.FuncMap {
		funcMap[k] = v
	}

	// Add the 'include' function here so we can close over t.
	funcMap["include"] = func(name string, data interface{}) string {
		buf := bytes.NewBuffer(nil)
		if err := t.ExecuteTemplate(buf, name, data); err != nil {
			buf.WriteString(err.Error())
		}
		return buf.String()
	}

	// Add the 'required' function here
	funcMap["required"] = func(warn string, val interface{}) (interface{}, error) {
		if val == nil {
			return val, fmt.Errorf(warn)
		} else if _, ok := val.(string); ok {
			if val == "" {
				return val, fmt.Errorf(warn)
			}
		}
		return val, nil
	}

	// Add the 'tpl' function here
	funcMap["tpl"] = func(tpl string, vals chartutil.Values) (string, error) {
		r := renderable{
			tpl:  tpl,
			vals: vals,
		}

		templates := map[string]renderable{}
		templates["template"] = r

		result, err := e.render(templates)
		if err != nil {
			return "", fmt.Errorf("Error during tpl function execution for %q: %s", tpl, err.Error())
		}
		return result["template"], nil
	}

	return funcMap
}

// render takes a map of templates/values and renders them.
func (e *Engine) render(tpls map[string]renderable) (map[string]string, error) {
	// Basically, what we do here is start with an empty parent template and then
	// build up a list of templates -- one for each file. Once all of the templates
	// have been parsed, we loop through again and execute every template.
	//
	// The idea with this process is to make it possible for more complex templates
	// to share common blocks, but to make the entire thing feel like a file-based
	// template engine.
	t := template.New("gotpl")
	if e.Strict {
		t.Option("missingkey=error")
	} else {
		// Not that zero will attempt to add default values for types it knows,
		// but will still emit <no value> for others. We mitigate that later.
		t.Option("missingkey=zero")
	}

	// Adding templates from the engine context but do not overwrite
	for k, v := range e.CurrentTemplates {
		if _, exists := tpls[k]; exists {
			tpls[k] = v
		}
	}
	funcMap := e.alterFuncMap(t)

	files := []string{}
	for fname, r := range tpls {
		t = t.New(fname).Funcs(funcMap)
		if _, err := t.Parse(r.tpl); err != nil {
			return map[string]string{}, fmt.Errorf("parse error in %q: %s", fname, err)
		}
		files = append(files, fname)
	}

	rendered := make(map[string]string, len(files))
	var buf bytes.Buffer
	for _, file := range files {
		// At render time, add information about the template that is being rendered.
		vals := tpls[file].vals
		vals["Template"] = map[string]interface{}{"Name": file, "BasePath": tpls[file].basePath}
		if err := t.ExecuteTemplate(&buf, file, vals); err != nil {
			return map[string]string{}, fmt.Errorf("render error in %q: %s", file, err)
		}

		// Work around the issue where Go will emit "<no value>" even if Options(missing=zero)
		// is set. Since missing=error will never get here, we do not need to handle
		// the Strict case.
		rendered[file] = strings.Replace(buf.String(), "<no value>", "", -1)
		buf.Reset()
	}

	return rendered, nil
}

// allTemplates returns all templates for a chart and its dependencies.
//
// As it goes, it also prepares the values in a scope-sensitive manner.
func allTemplates(c *chart.Chart, vals chartutil.Values) map[string]renderable {
	templates := map[string]renderable{}
	recAllTpls(c, templates, vals, true, "")
	return templates
}

// recAllTpls recurses through the templates in a chart.
//
// As it recurses, it also sets the values to be appropriate for the template
// scope.
func recAllTpls(c *chart.Chart, templates map[string]renderable, parentVals chartutil.Values, top bool, parentID string) {
	// This should never evaluate to a nil map. That will cause problems when
	// values are appended later.
	cvals := chartutil.Values{}
	if top {
		// If this is the top of the rendering tree, assume that parentVals
		// is already resolved to the authoritative values.
		cvals = parentVals
	} else if c.Metadata != nil && c.Metadata.Name != "" {
		// If there is a {{.Values.ThisChart}} in the parent metadata,
		// copy that into the {{.Values}} for this template.
		newVals := chartutil.Values{}
		if vs, err := parentVals.Table("Values"); err == nil {
			if tmp, err := vs.Table(c.Metadata.Name); err == nil {
				newVals = tmp
			}
		}

		cvals = map[string]interface{}{
			"Values":       newVals,
			"Release":      parentVals["Release"],
			"Chart":        c.Metadata,
			"Files":        chartutil.NewFiles(c.Files),
			"Capabilities": parentVals["Capabilities"],
		}
	}

	newParentID := c.Metadata.Name
	if parentID != "" {
		// We artificially reconstruct the chart path to child templates. This
		// creates a namespaced filename that can be used to track down the source
		// of a particular template declaration.
		newParentID = path.Join(parentID, "charts", newParentID)
	}

	for _, child := range c.Dependencies {
		recAllTpls(child, templates, cvals, false, newParentID)
	}
	for _, t := range c.Templates {
		templates[path.Join(newParentID, t.Name)] = renderable{
			tpl:      string(t.Data),
			vals:     cvals,
			basePath: path.Join(newParentID, "templates"),
		}
	}
}
<|endoftext|>


<commit_before>package net_test

import (
	"net"
	"testing"

	v2net "github.com/v2ray/v2ray-core/common/net"
	v2netassert "github.com/v2ray/v2ray-core/common/net/testing/assert"
	v2testing "github.com/v2ray/v2ray-core/testing"
	"github.com/v2ray/v2ray-core/testing/assert"
)

func TestIPv4Address(t *testing.T) {
	v2testing.Current(t)

	ip := []byte{byte(1), byte(2), byte(3), byte(4)}
	addr := v2net.IPAddress(ip)

	v2netassert.Address(addr).IsIPv4()
	v2netassert.Address(addr).IsNotIPv6()
	v2netassert.Address(addr).IsNotDomain()
	assert.Bytes(addr.IP()).Equals(ip)
	assert.String(addr).Equals("1.2.3.4")
}

func TestIPv6Address(t *testing.T) {
	v2testing.Current(t)

	ip := []byte{
		byte(1), byte(2), byte(3), byte(4),
		byte(1), byte(2), byte(3), byte(4),
		byte(1), byte(2), byte(3), byte(4),
		byte(1), byte(2), byte(3), byte(4),
	}
	addr := v2net.IPAddress(ip)

	v2netassert.Address(addr).IsIPv6()
	v2netassert.Address(addr).IsNotIPv4()
	v2netassert.Address(addr).IsNotDomain()
	assert.Bytes(addr.IP()).Equals(ip)
	assert.String(addr).Equals("[102:304:102:304:102:304:102:304]")
}

func TestDomainAddress(t *testing.T) {
	v2testing.Current(t)

	domain := "v2ray.com"
	addr := v2net.DomainAddress(domain)

	v2netassert.Address(addr).IsDomain()
	v2netassert.Address(addr).IsNotIPv6()
	v2netassert.Address(addr).IsNotIPv4()
	assert.StringLiteral(addr.Domain()).Equals(domain)
	assert.String(addr).Equals("v2ray.com")
}

func TestNetIPv4Address(t *testing.T) {
	v2testing.Current(t)

	ip := net.IPv4(1, 2, 3, 4)
	addr := v2net.IPAddress(ip)
	v2netassert.Address(addr).IsIPv4()
	assert.String(addr).Equals("1.2.3.4")
}
<commit_msg>test case for ipv4 address in ipv6 format<commit_after>package net_test

import (
	"net"
	"testing"

	v2net "github.com/v2ray/v2ray-core/common/net"
	v2netassert "github.com/v2ray/v2ray-core/common/net/testing/assert"
	v2testing "github.com/v2ray/v2ray-core/testing"
	"github.com/v2ray/v2ray-core/testing/assert"
)

func TestIPv4Address(t *testing.T) {
	v2testing.Current(t)

	ip := []byte{byte(1), byte(2), byte(3), byte(4)}
	addr := v2net.IPAddress(ip)

	v2netassert.Address(addr).IsIPv4()
	v2netassert.Address(addr).IsNotIPv6()
	v2netassert.Address(addr).IsNotDomain()
	assert.Bytes(addr.IP()).Equals(ip)
	assert.String(addr).Equals("1.2.3.4")
}

func TestIPv6Address(t *testing.T) {
	v2testing.Current(t)

	ip := []byte{
		byte(1), byte(2), byte(3), byte(4),
		byte(1), byte(2), byte(3), byte(4),
		byte(1), byte(2), byte(3), byte(4),
		byte(1), byte(2), byte(3), byte(4),
	}
	addr := v2net.IPAddress(ip)

	v2netassert.Address(addr).IsIPv6()
	v2netassert.Address(addr).IsNotIPv4()
	v2netassert.Address(addr).IsNotDomain()
	assert.Bytes(addr.IP()).Equals(ip)
	assert.String(addr).Equals("[102:304:102:304:102:304:102:304]")
}

func TestIPv4Asv6(t *testing.T) {
  v2testing.Current(t)
  ip := []byte{
		byte(0), byte(0), byte(0), byte(0),
		byte(0), byte(0), byte(0), byte(0),
		byte(0), byte(0), byte(255), byte(255),
		byte(1), byte(2), byte(3), byte(4),
	}
	addr := v2net.IPAddress(ip)
	assert.String(addr).Equals("1.2.3.4")
}

func TestDomainAddress(t *testing.T) {
	v2testing.Current(t)

	domain := "v2ray.com"
	addr := v2net.DomainAddress(domain)

	v2netassert.Address(addr).IsDomain()
	v2netassert.Address(addr).IsNotIPv6()
	v2netassert.Address(addr).IsNotIPv4()
	assert.StringLiteral(addr.Domain()).Equals(domain)
	assert.String(addr).Equals("v2ray.com")
}

func TestNetIPv4Address(t *testing.T) {
	v2testing.Current(t)

	ip := net.IPv4(1, 2, 3, 4)
	addr := v2net.IPAddress(ip)
	v2netassert.Address(addr).IsIPv4()
	assert.String(addr).Equals("1.2.3.4")
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/api"
	"github.com/Aptomi/aptomi/pkg/api/middleware"
	"github.com/Aptomi/aptomi/pkg/config"
	"github.com/Aptomi/aptomi/pkg/external"
	"github.com/Aptomi/aptomi/pkg/external/secrets"
	"github.com/Aptomi/aptomi/pkg/external/users"
	"github.com/Aptomi/aptomi/pkg/runtime"
	"github.com/Aptomi/aptomi/pkg/runtime/store"
	"github.com/Aptomi/aptomi/pkg/runtime/store/core"
	"github.com/Aptomi/aptomi/pkg/runtime/store/generic/bolt"
	"github.com/Aptomi/aptomi/pkg/server/ui"
	log "github.com/Sirupsen/logrus"
	"github.com/gorilla/handlers"
	"github.com/julienschmidt/httprouter"
	"github.com/rs/cors"
	"net/http"
	"os"
	"time"
)

// Server is Aptomi server. It serves UI front-end, API calls, as well as does policy resolution & continuous state enforcement
type Server struct {
	cfg              *config.Server
	backgroundErrors chan string

	externalData *external.Data
	store        store.Core

	httpServer *http.Server

	enforcementIdx uint
}

// NewServer creates a new Aptomi Server
func NewServer(cfg *config.Server) *Server {
	s := &Server{
		cfg:              cfg,
		backgroundErrors: make(chan string),
	}

	return s
}

// Start initializes Aptomi server, starts API & UI processing, and as well as runs the required background jobs for
// continuous policy resolution and state enforcement
func (server *Server) Start() {
	server.initStore()
	server.initExternalData()

	// See if policy initialization needs to happen on the first run
	server.initPolicyOnFirstRun()

	// Start API, UI and Enforcer
	server.startHTTPServer()
	server.startEnforcer()

	// Wait for jobs to complete (it essentially hangs forever)
	server.wait()
}

func (server *Server) initPolicyOnFirstRun() {
	policy, _, err := server.store.GetPolicy(runtime.LastGen)
	if err != nil {
		panic(fmt.Sprintf("error while getting latest policy: %s", err))
	}

	// if policy does not exist, let's create the first version (it should be created here, before we start the server)
	if policy == nil {
		log.Infof("Policy not found in the store (likely, it's a first run of Aptomi server). Creating empty policy")
		err := server.store.InitPolicy()
		if err != nil {
			panic(fmt.Sprintf("error while creating empty policy: %s", err))
		}
	}
}

func (server *Server) initExternalData() {
	userLoaders := []users.UserLoader{}
	for _, ldap := range server.cfg.Users.LDAP {
		userLoaders = append(userLoaders, users.NewUserLoaderFromLDAP(ldap, server.cfg.DomainAdminOverrides))
	}
	for _, file := range server.cfg.Users.File {
		userLoaders = append(userLoaders, users.NewUserLoaderFromFile(file, server.cfg.DomainAdminOverrides))
	}
	server.externalData = external.NewData(
		users.NewUserLoaderMultipleSources(userLoaders),
		secrets.NewSecretLoaderFromDir(server.cfg.SecretsDir),
	)
}

func (server *Server) initStore() {
	registry := runtime.NewRegistry().Append(store.Objects...)
	b := bolt.NewGenericStore(registry)
	err := b.Open(server.cfg.DB)
	if err != nil {
		panic(fmt.Sprintf("Can't open object store: %s", err))
	}
	server.store = core.NewStore(b)
}

func (server *Server) startHTTPServer() {
	router := httprouter.New()

	api.Serve(router, server.store, server.externalData)
	server.serveUI(router)

	var handler http.Handler = router

	// todo write to logrus
	handler = handlers.CombinedLoggingHandler(os.Stdout, handler) // todo(slukjanov): make it at least somehow configurable - for example, select file to write to with rotation
	handler = cors.Default().Handler(handler)
	handler = middleware.NewPanicHandler(handler)
	// todo(slukjanov): add configurable handlers.ProxyHeaders to f behind the nginx or any other proxy
	// todo(slukjanov): add compression handler and compress by default in client

	server.httpServer = &http.Server{
		Handler:      handler,
		Addr:         server.cfg.API.ListenAddr(),
		WriteTimeout: 5 * time.Second,
		ReadTimeout:  30 * time.Second,
	}

	// Start HTTP server
	server.runInBackground("HTTP Server / API", true, func() {
		panic(server.httpServer.ListenAndServe())
	})
}

func (server *Server) serveUI(router *httprouter.Router) {
	if !server.cfg.UI.Enable {
		log.Infof("UI isn't enabled. UI will not be served")
		return
	}

	fileServer := http.FileServer(ui.HTTP)

	router.GET("/", func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		req.URL.Path = "/"
		fileServer.ServeHTTP(w, req)
	})

	router.GET("/index.html", func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		req.URL.Path = "/"
		fileServer.ServeHTTP(w, req)
	})

	router.GET("/static/*filepath", func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		req.URL.Path = "/static/" + ps.ByName("filepath")
		fileServer.ServeHTTP(w, req)
	})
}

func (server *Server) startEnforcer() {
	// Start policy enforcement job
	if !server.cfg.Enforcer.Disabled {
		server.runInBackground("Policy Enforcer", true, func() {
			panic(server.enforceLoop())
		})
	}
}
<commit_msg>Use nil slice<commit_after>package server

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/api"
	"github.com/Aptomi/aptomi/pkg/api/middleware"
	"github.com/Aptomi/aptomi/pkg/config"
	"github.com/Aptomi/aptomi/pkg/external"
	"github.com/Aptomi/aptomi/pkg/external/secrets"
	"github.com/Aptomi/aptomi/pkg/external/users"
	"github.com/Aptomi/aptomi/pkg/runtime"
	"github.com/Aptomi/aptomi/pkg/runtime/store"
	"github.com/Aptomi/aptomi/pkg/runtime/store/core"
	"github.com/Aptomi/aptomi/pkg/runtime/store/generic/bolt"
	"github.com/Aptomi/aptomi/pkg/server/ui"
	log "github.com/Sirupsen/logrus"
	"github.com/gorilla/handlers"
	"github.com/julienschmidt/httprouter"
	"github.com/rs/cors"
	"net/http"
	"os"
	"time"
)

// Server is Aptomi server. It serves UI front-end, API calls, as well as does policy resolution & continuous state enforcement
type Server struct {
	cfg              *config.Server
	backgroundErrors chan string

	externalData *external.Data
	store        store.Core

	httpServer *http.Server

	enforcementIdx uint
}

// NewServer creates a new Aptomi Server
func NewServer(cfg *config.Server) *Server {
	s := &Server{
		cfg:              cfg,
		backgroundErrors: make(chan string),
	}

	return s
}

// Start initializes Aptomi server, starts API & UI processing, and as well as runs the required background jobs for
// continuous policy resolution and state enforcement
func (server *Server) Start() {
	server.initStore()
	server.initExternalData()

	// See if policy initialization needs to happen on the first run
	server.initPolicyOnFirstRun()

	// Start API, UI and Enforcer
	server.startHTTPServer()
	server.startEnforcer()

	// Wait for jobs to complete (it essentially hangs forever)
	server.wait()
}

func (server *Server) initPolicyOnFirstRun() {
	policy, _, err := server.store.GetPolicy(runtime.LastGen)
	if err != nil {
		panic(fmt.Sprintf("error while getting latest policy: %s", err))
	}

	// if policy does not exist, let's create the first version (it should be created here, before we start the server)
	if policy == nil {
		log.Infof("Policy not found in the store (likely, it's a first run of Aptomi server). Creating empty policy")
		err := server.store.InitPolicy()
		if err != nil {
			panic(fmt.Sprintf("error while creating empty policy: %s", err))
		}
	}
}

func (server *Server) initExternalData() {
	var userLoaders []users.UserLoader
	for _, ldap := range server.cfg.Users.LDAP {
		userLoaders = append(userLoaders, users.NewUserLoaderFromLDAP(ldap, server.cfg.DomainAdminOverrides))
	}
	for _, file := range server.cfg.Users.File {
		userLoaders = append(userLoaders, users.NewUserLoaderFromFile(file, server.cfg.DomainAdminOverrides))
	}
	server.externalData = external.NewData(
		users.NewUserLoaderMultipleSources(userLoaders),
		secrets.NewSecretLoaderFromDir(server.cfg.SecretsDir),
	)
}

func (server *Server) initStore() {
	registry := runtime.NewRegistry().Append(store.Objects...)
	b := bolt.NewGenericStore(registry)
	err := b.Open(server.cfg.DB)
	if err != nil {
		panic(fmt.Sprintf("Can't open object store: %s", err))
	}
	server.store = core.NewStore(b)
}

func (server *Server) startHTTPServer() {
	router := httprouter.New()

	api.Serve(router, server.store, server.externalData)
	server.serveUI(router)

	var handler http.Handler = router

	// todo write to logrus
	handler = handlers.CombinedLoggingHandler(os.Stdout, handler) // todo(slukjanov): make it at least somehow configurable - for example, select file to write to with rotation
	handler = cors.Default().Handler(handler)
	handler = middleware.NewPanicHandler(handler)
	// todo(slukjanov): add configurable handlers.ProxyHeaders to f behind the nginx or any other proxy
	// todo(slukjanov): add compression handler and compress by default in client

	server.httpServer = &http.Server{
		Handler:      handler,
		Addr:         server.cfg.API.ListenAddr(),
		WriteTimeout: 5 * time.Second,
		ReadTimeout:  30 * time.Second,
	}

	// Start HTTP server
	server.runInBackground("HTTP Server / API", true, func() {
		panic(server.httpServer.ListenAndServe())
	})
}

func (server *Server) serveUI(router *httprouter.Router) {
	if !server.cfg.UI.Enable {
		log.Infof("UI isn't enabled. UI will not be served")
		return
	}

	fileServer := http.FileServer(ui.HTTP)

	router.GET("/", func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		req.URL.Path = "/"
		fileServer.ServeHTTP(w, req)
	})

	router.GET("/index.html", func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		req.URL.Path = "/"
		fileServer.ServeHTTP(w, req)
	})

	router.GET("/static/*filepath", func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		req.URL.Path = "/static/" + ps.ByName("filepath")
		fileServer.ServeHTTP(w, req)
	})
}

func (server *Server) startEnforcer() {
	// Start policy enforcement job
	if !server.cfg.Enforcer.Disabled {
		server.runInBackground("Policy Enforcer", true, func() {
			panic(server.enforceLoop())
		})
	}
}
<|endoftext|>


<commit_before>package server

import (
	"context"
	"fmt"
	"net"
	"net/http"
	"reflect"
	"time"

	"github.com/hellofresh/janus/pkg/api"
	"github.com/hellofresh/janus/pkg/config"
	"github.com/hellofresh/janus/pkg/errors"
	httpErrors "github.com/hellofresh/janus/pkg/errors"
	"github.com/hellofresh/janus/pkg/loader"
	"github.com/hellofresh/janus/pkg/middleware"
	"github.com/hellofresh/janus/pkg/plugin"
	"github.com/hellofresh/janus/pkg/proxy"
	"github.com/hellofresh/janus/pkg/router"
	"github.com/hellofresh/janus/pkg/web"
	"github.com/hellofresh/stats-go/client"
	log "github.com/sirupsen/logrus"
)

// Server is the Janus server
type Server struct {
	server                *http.Server
	provider              api.Repository
	register              *proxy.Register
	defLoader             *loader.APILoader
	currentConfigurations *api.Configuration
	configurationChan     chan api.ConfigurationChanged
	stopChan              chan struct{}
	globalConfig          *config.Specification
	statsClient           client.Client
	webServer             *web.Server
}

// New creates a new instance of Server
func New(opts ...Option) *Server {
	s := Server{
		configurationChan: make(chan api.ConfigurationChanged, 100),
		stopChan:          make(chan struct{}, 1),
	}

	for _, opt := range opts {
		opt(&s)
	}

	return &s
}

// Start starts the server
func (s *Server) Start() error {
	return s.StartWithContext(context.Background())
}

// StartWithContext starts the server and Stop/Close it when context is Done
func (s *Server) StartWithContext(ctx context.Context) error {
	go func() {
		defer s.Close()
		<-ctx.Done()
		log.Info("I have to go...")
		reqAcceptGraceTimeOut := time.Duration(s.globalConfig.GraceTimeOut)
		if reqAcceptGraceTimeOut > 0 {
			log.Infof("Waiting %s for incoming requests to cease", reqAcceptGraceTimeOut)
			time.Sleep(reqAcceptGraceTimeOut)
		}
		log.Info("Stopping server gracefully")
	}()
	go func() {
		if err := s.startHTTPServers(); err != nil {
			log.WithError(err).Fatal("Could not start http servers")
		}
	}()

	go s.listenProviders(s.stopChan)

	defs, err := s.provider.FindAll()
	if err != nil {
		return errors.Wrap(err, "could not find all configurations from the provider")
	}

	s.currentConfigurations = &api.Configuration{Definitions: defs}
	if err := s.startProvider(ctx); err != nil {
		log.WithError(err).Fatal("Could not start providers")
	}

	event := plugin.OnStartup{
		StatsClient:   s.statsClient,
		Register:      s.register,
		Config:        s.globalConfig,
		Configuration: defs,
	}

	if mgoRepo, ok := s.provider.(*api.MongoRepository); ok {
		event.MongoSession = mgoRepo.Session
	}

	plugin.EmitEvent(plugin.StartupEvent, event)
	s.defLoader.RegisterAPIs(defs)

	log.Info("Janus started")

	return nil
}

// Wait blocks until server is shutted down.
func (s *Server) Wait() {
	<-s.stopChan
}

// Stop stops the server
func (s *Server) Stop() {
	defer log.Info("Server stopped")

	graceTimeOut := time.Duration(s.globalConfig.GraceTimeOut)
	ctx, cancel := context.WithTimeout(context.Background(), graceTimeOut)
	defer cancel()
	log.Debugf("Waiting %s seconds before killing connections...", graceTimeOut)
	if err := s.server.Shutdown(ctx); err != nil {
		log.WithError(err).Debug("Wait is over due to error")
		s.server.Close()
	}
	log.Debug("Server closed")

	s.stopChan <- struct{}{}
}

// Close closes the server
func (s *Server) Close() error {
	defer close(s.stopChan)
	defer close(s.configurationChan)
	defer s.webServer.Stop()

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	go func(ctx context.Context) {
		<-ctx.Done()
		if ctx.Err() == context.Canceled {
			return
		} else if ctx.Err() == context.DeadlineExceeded {
			panic("Timeout while stopping janus, killing instance ✝")
		}
	}(ctx)

	return s.server.Close()
}

func (s *Server) startHTTPServers() error {
	r := s.createRouter()
	s.register = proxy.NewRegister(r, proxy.Params{
		StatsClient:            s.statsClient,
		FlushInterval:          s.globalConfig.BackendFlushInterval,
		IdleConnectionsPerHost: s.globalConfig.MaxIdleConnsPerHost,
		CloseIdleConnsPeriod:   s.globalConfig.CloseIdleConnsPeriod,
	})
	s.defLoader = loader.NewAPILoader(s.register)

	return s.listenAndServe(r)
}

func (s *Server) startProvider(ctx context.Context) error {
	s.webServer = web.New(
		web.WithConfigurations(s.currentConfigurations),
		web.WithPort(s.globalConfig.Web.Port),
		web.WithTLS(s.globalConfig.Web.TLS),
		web.WithCredentials(s.globalConfig.Web.Credentials),
		web.ReadOnly(s.globalConfig.Web.ReadOnly),
	)

	if err := s.webServer.Start(); err != nil {
		return errors.Wrap(err, "could not start Janus web API")
	}

	if listener, ok := s.provider.(api.Listener); ok {
		go func() {
			ch := make(chan api.ConfigurationMessage)
			listener.Listen(ctx, ch)
			for {
				select {
				case c, more := <-s.webServer.ConfigurationChan:
					if !more {
						return
					}

					s.updateConfigurations(c)
					s.handleEvent(s.currentConfigurations)
					ch <- c
				case <-ctx.Done():
					close(ch)
					return
				}
			}
		}()
	}

	if watcher, ok := s.provider.(api.Watcher); ok {
		watcher.Watch(ctx, s.configurationChan)
	}

	return nil
}

func (s *Server) listenProviders(stop chan struct{}) {
	for {
		select {
		case <-stop:
			return
		case configMsg, ok := <-s.configurationChan:
			if !ok {
				return
			}

			if reflect.DeepEqual(s.currentConfigurations, configMsg.Configurations) {
				log.Debug("Skipping same configuration")
				continue
			}

			s.currentConfigurations.Definitions = configMsg.Configurations.Definitions
			s.handleEvent(configMsg.Configurations)
		}
	}
}

func (s *Server) listenAndServe(handler http.Handler) error {
	address := fmt.Sprintf(":%v", s.globalConfig.Port)
	logger := log.WithField("address", address)
	s.server = &http.Server{Addr: address, Handler: handler}
	listener, err := net.Listen("tcp", address)
	if err != nil {
		return errors.Wrap(err, "error opening listener")
	}

	if s.globalConfig.TLS.IsHTTPS() {
		s.server.Addr = fmt.Sprintf(":%v", s.globalConfig.TLS.Port)

		if s.globalConfig.TLS.Redirect {
			go func() {
				logger.Info("Listening HTTP redirects to HTTPS")
				log.Fatal(http.Serve(listener, web.RedirectHTTPS(s.globalConfig.TLS.Port)))
			}()
		}

		logger.Info("Listening HTTPS")
		return s.server.ServeTLS(listener, s.globalConfig.TLS.CertFile, s.globalConfig.TLS.KeyFile)
	}

	logger.Info("Certificate and certificate key were not found, defaulting to HTTP")
	return s.server.Serve(listener)
}

func (s *Server) createRouter() router.Router {
	// create router with a custom not found handler
	router.DefaultOptions.NotFoundHandler = errors.NotFound
	r := router.NewChiRouterWithOptions(router.DefaultOptions)
	r.Use(
		middleware.NewStats(s.statsClient).Handler,
		middleware.NewLogger().Handler,
		middleware.NewRecovery(errors.RecoveryHandler),
		middleware.NewOpenTracing(s.globalConfig.TLS.IsHTTPS()).Handler,
	)

	if s.globalConfig.RequestID {
		r.Use(middleware.RequestID)
	}

	// some routers may panic when have empty routes list, so add one dummy 404 route to avoid this
	if r.RoutesCount() < 1 {
		r.Any("/", httpErrors.NotFound)
	}

	return r
}

func (s *Server) updateConfigurations(cfg api.ConfigurationMessage) {
	currentDefinitions := s.currentConfigurations.Definitions

	switch cfg.Operation {
	case api.AddedOperation:
		currentDefinitions = append(currentDefinitions, cfg.Configuration)
	case api.UpdatedOperation:
		for i, d := range currentDefinitions {
			if d.Name == cfg.Configuration.Name {
				currentDefinitions[i] = cfg.Configuration
			}
		}
	case api.RemovedOperation:
		for i, d := range currentDefinitions {
			if d.Name == cfg.Configuration.Name {
				copy(currentDefinitions[i:], currentDefinitions[i+1:])
				// currentDefinitions[len(currentDefinitions)-1] = nil // or the zero value of T
				currentDefinitions = currentDefinitions[:len(currentDefinitions)-1]
			}
		}
	}

	s.currentConfigurations.Definitions = currentDefinitions
}

func (s *Server) handleEvent(cfg *api.Configuration) {
	log.Debug("Refreshing configuration")
	newRouter := s.createRouter()

	s.register.UpdateRouter(newRouter)
	s.defLoader.RegisterAPIs(cfg.Definitions)

	plugin.EmitEvent(plugin.ReloadEvent, plugin.OnReload{Configurations: cfg.Definitions})

	s.server.Handler = newRouter
	log.Debug("Configuration refresh done")
}
<commit_msg>Passing the base context to all requests<commit_after>package server

import (
	"context"
	"fmt"
	"net"
	"net/http"
	"reflect"
	"time"

	"github.com/go-chi/chi"
	"github.com/hellofresh/janus/pkg/api"
	"github.com/hellofresh/janus/pkg/config"
	"github.com/hellofresh/janus/pkg/errors"
	httpErrors "github.com/hellofresh/janus/pkg/errors"
	"github.com/hellofresh/janus/pkg/loader"
	"github.com/hellofresh/janus/pkg/middleware"
	"github.com/hellofresh/janus/pkg/plugin"
	"github.com/hellofresh/janus/pkg/proxy"
	"github.com/hellofresh/janus/pkg/router"
	"github.com/hellofresh/janus/pkg/web"
	"github.com/hellofresh/stats-go/client"
	log "github.com/sirupsen/logrus"
)

// Server is the Janus server
type Server struct {
	server                *http.Server
	provider              api.Repository
	register              *proxy.Register
	defLoader             *loader.APILoader
	currentConfigurations *api.Configuration
	configurationChan     chan api.ConfigurationChanged
	stopChan              chan struct{}
	globalConfig          *config.Specification
	statsClient           client.Client
	webServer             *web.Server
}

// New creates a new instance of Server
func New(opts ...Option) *Server {
	s := Server{
		configurationChan: make(chan api.ConfigurationChanged, 100),
		stopChan:          make(chan struct{}, 1),
	}

	for _, opt := range opts {
		opt(&s)
	}

	return &s
}

// Start starts the server
func (s *Server) Start() error {
	return s.StartWithContext(context.Background())
}

// StartWithContext starts the server and Stop/Close it when context is Done
func (s *Server) StartWithContext(ctx context.Context) error {
	go func() {
		defer s.Close()
		<-ctx.Done()
		log.Info("I have to go...")
		reqAcceptGraceTimeOut := time.Duration(s.globalConfig.GraceTimeOut)
		if reqAcceptGraceTimeOut > 0 {
			log.Infof("Waiting %s for incoming requests to cease", reqAcceptGraceTimeOut)
			time.Sleep(reqAcceptGraceTimeOut)
		}
		log.Info("Stopping server gracefully")
	}()
	go func() {
		if err := s.startHTTPServers(ctx); err != nil {
			log.WithError(err).Fatal("Could not start http servers")
		}
	}()

	go s.listenProviders(s.stopChan)

	defs, err := s.provider.FindAll()
	if err != nil {
		return errors.Wrap(err, "could not find all configurations from the provider")
	}

	s.currentConfigurations = &api.Configuration{Definitions: defs}
	if err := s.startProvider(ctx); err != nil {
		log.WithError(err).Fatal("Could not start providers")
	}

	event := plugin.OnStartup{
		StatsClient:   s.statsClient,
		Register:      s.register,
		Config:        s.globalConfig,
		Configuration: defs,
	}

	if mgoRepo, ok := s.provider.(*api.MongoRepository); ok {
		event.MongoSession = mgoRepo.Session
	}

	plugin.EmitEvent(plugin.StartupEvent, event)
	s.defLoader.RegisterAPIs(defs)

	log.Info("Janus started")

	return nil
}

// Wait blocks until server is shutted down.
func (s *Server) Wait() {
	<-s.stopChan
}

// Stop stops the server
func (s *Server) Stop() {
	defer log.Info("Server stopped")

	graceTimeOut := time.Duration(s.globalConfig.GraceTimeOut)
	ctx, cancel := context.WithTimeout(context.Background(), graceTimeOut)
	defer cancel()
	log.Debugf("Waiting %s seconds before killing connections...", graceTimeOut)
	if err := s.server.Shutdown(ctx); err != nil {
		log.WithError(err).Debug("Wait is over due to error")
		s.server.Close()
	}
	log.Debug("Server closed")

	s.stopChan <- struct{}{}
}

// Close closes the server
func (s *Server) Close() error {
	defer close(s.stopChan)
	defer close(s.configurationChan)
	defer s.webServer.Stop()

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	go func(ctx context.Context) {
		<-ctx.Done()
		if ctx.Err() == context.Canceled {
			return
		} else if ctx.Err() == context.DeadlineExceeded {
			panic("Timeout while stopping janus, killing instance ✝")
		}
	}(ctx)

	return s.server.Close()
}

func (s *Server) startHTTPServers(ctx context.Context) error {
	r := s.createRouter()
	s.register = proxy.NewRegister(r, proxy.Params{
		StatsClient:            s.statsClient,
		FlushInterval:          s.globalConfig.BackendFlushInterval,
		IdleConnectionsPerHost: s.globalConfig.MaxIdleConnsPerHost,
		CloseIdleConnsPeriod:   s.globalConfig.CloseIdleConnsPeriod,
	})
	s.defLoader = loader.NewAPILoader(s.register)

	return s.listenAndServe(chi.ServerBaseContext(ctx, r))
}

func (s *Server) startProvider(ctx context.Context) error {
	s.webServer = web.New(
		web.WithConfigurations(s.currentConfigurations),
		web.WithPort(s.globalConfig.Web.Port),
		web.WithTLS(s.globalConfig.Web.TLS),
		web.WithCredentials(s.globalConfig.Web.Credentials),
		web.ReadOnly(s.globalConfig.Web.ReadOnly),
	)

	if err := s.webServer.Start(); err != nil {
		return errors.Wrap(err, "could not start Janus web API")
	}

	if listener, ok := s.provider.(api.Listener); ok {
		go func() {
			ch := make(chan api.ConfigurationMessage)
			listener.Listen(ctx, ch)
			for {
				select {
				case c, more := <-s.webServer.ConfigurationChan:
					if !more {
						return
					}

					s.updateConfigurations(c)
					s.handleEvent(s.currentConfigurations)
					ch <- c
				case <-ctx.Done():
					close(ch)
					return
				}
			}
		}()
	}

	if watcher, ok := s.provider.(api.Watcher); ok {
		watcher.Watch(ctx, s.configurationChan)
	}

	return nil
}

func (s *Server) listenProviders(stop chan struct{}) {
	for {
		select {
		case <-stop:
			return
		case configMsg, ok := <-s.configurationChan:
			if !ok {
				return
			}

			if reflect.DeepEqual(s.currentConfigurations, configMsg.Configurations) {
				log.Debug("Skipping same configuration")
				continue
			}

			s.currentConfigurations.Definitions = configMsg.Configurations.Definitions
			s.handleEvent(configMsg.Configurations)
		}
	}
}

func (s *Server) listenAndServe(handler http.Handler) error {
	address := fmt.Sprintf(":%v", s.globalConfig.Port)
	logger := log.WithField("address", address)
	s.server = &http.Server{Addr: address, Handler: handler}
	listener, err := net.Listen("tcp", address)
	if err != nil {
		return errors.Wrap(err, "error opening listener")
	}

	if s.globalConfig.TLS.IsHTTPS() {
		s.server.Addr = fmt.Sprintf(":%v", s.globalConfig.TLS.Port)

		if s.globalConfig.TLS.Redirect {
			go func() {
				logger.Info("Listening HTTP redirects to HTTPS")
				log.Fatal(http.Serve(listener, web.RedirectHTTPS(s.globalConfig.TLS.Port)))
			}()
		}

		logger.Info("Listening HTTPS")
		return s.server.ServeTLS(listener, s.globalConfig.TLS.CertFile, s.globalConfig.TLS.KeyFile)
	}

	logger.Info("Certificate and certificate key were not found, defaulting to HTTP")
	return s.server.Serve(listener)
}

func (s *Server) createRouter() router.Router {
	// create router with a custom not found handler
	router.DefaultOptions.NotFoundHandler = errors.NotFound
	r := router.NewChiRouterWithOptions(router.DefaultOptions)
	r.Use(
		middleware.NewStats(s.statsClient).Handler,
		middleware.NewLogger().Handler,
		middleware.NewRecovery(errors.RecoveryHandler),
		middleware.NewOpenTracing(s.globalConfig.TLS.IsHTTPS()).Handler,
	)

	if s.globalConfig.RequestID {
		r.Use(middleware.RequestID)
	}

	// some routers may panic when have empty routes list, so add one dummy 404 route to avoid this
	if r.RoutesCount() < 1 {
		r.Any("/", httpErrors.NotFound)
	}

	return r
}

func (s *Server) updateConfigurations(cfg api.ConfigurationMessage) {
	currentDefinitions := s.currentConfigurations.Definitions

	switch cfg.Operation {
	case api.AddedOperation:
		currentDefinitions = append(currentDefinitions, cfg.Configuration)
	case api.UpdatedOperation:
		for i, d := range currentDefinitions {
			if d.Name == cfg.Configuration.Name {
				currentDefinitions[i] = cfg.Configuration
			}
		}
	case api.RemovedOperation:
		for i, d := range currentDefinitions {
			if d.Name == cfg.Configuration.Name {
				copy(currentDefinitions[i:], currentDefinitions[i+1:])
				// currentDefinitions[len(currentDefinitions)-1] = nil // or the zero value of T
				currentDefinitions = currentDefinitions[:len(currentDefinitions)-1]
			}
		}
	}

	s.currentConfigurations.Definitions = currentDefinitions
}

func (s *Server) handleEvent(cfg *api.Configuration) {
	log.Debug("Refreshing configuration")
	newRouter := s.createRouter()

	s.register.UpdateRouter(newRouter)
	s.defLoader.RegisterAPIs(cfg.Definitions)

	plugin.EmitEvent(plugin.ReloadEvent, plugin.OnReload{Configurations: cfg.Definitions})

	s.server.Handler = newRouter
	log.Debug("Configuration refresh done")
}
<|endoftext|>