<commit_before>// Copyright 2016 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hugolib

import (
	"fmt"
	"path/filepath"
	"testing"

	"github.com/gohugoio/hugo/deps"
	"github.com/gohugoio/hugo/hugofs"

	"github.com/spf13/viper"
)

func TestTemplateLookupOrder(t *testing.T) {
	t.Parallel()
	var (
		fs  *hugofs.Fs
		cfg *viper.Viper
		th  testHelper
	)

	// Variants base templates:
	//   1. <current-path>/<template-name>-baseof.<suffix>, e.g. list-baseof.<suffix>.
	//   2. <current-path>/baseof.<suffix>
	//   3. _default/<template-name>-baseof.<suffix>, e.g. list-baseof.<suffix>.
	//   4. _default/baseof.<suffix>
	for i, this := range []struct {
		setup  func(t *testing.T)
		assert func(t *testing.T)
	}{
		{
			// Variant 1
			func(t *testing.T) {
				writeSource(t, fs, filepath.Join("layouts", "section", "sect1-baseof.html"), `Base: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("layouts", "section", "sect1.html"), `{{define "main"}}sect{{ end }}`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "sect1", "index.html"), "Base: sect")
			},
		},
		{
			// Variant 2
			func(t *testing.T) {
				writeSource(t, fs, filepath.Join("layouts", "baseof.html"), `Base: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("layouts", "index.html"), `{{define "main"}}index{{ end }}`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "index.html"), "Base: index")
			},
		},
		{
			// Variant 3
			func(t *testing.T) {
				writeSource(t, fs, filepath.Join("layouts", "_default", "list-baseof.html"), `Base: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("layouts", "_default", "list.html"), `{{define "main"}}list{{ end }}`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "sect1", "index.html"), "Base: list")
			},
		},
		{
			// Variant 4
			func(t *testing.T) {
				writeSource(t, fs, filepath.Join("layouts", "_default", "baseof.html"), `Base: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("layouts", "_default", "list.html"), `{{define "main"}}list{{ end }}`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "sect1", "index.html"), "Base: list")
			},
		},
		{
			// Variant 1, theme,  use project's base
			func(t *testing.T) {
				cfg.Set("theme", "mytheme")
				writeSource(t, fs, filepath.Join("layouts", "section", "sect1-baseof.html"), `Base: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "section", "sect-baseof.html"), `Base Theme: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("layouts", "section", "sect1.html"), `{{define "main"}}sect{{ end }}`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "sect1", "index.html"), "Base: sect")
			},
		},
		{
			// Variant 1, theme,  use theme's base
			func(t *testing.T) {
				cfg.Set("theme", "mytheme")
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "section", "sect1-baseof.html"), `Base Theme: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("layouts", "section", "sect1.html"), `{{define "main"}}sect{{ end }}`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "sect1", "index.html"), "Base Theme: sect")
			},
		},
		{
			// Variant 4, theme, use project's base
			func(t *testing.T) {
				cfg.Set("theme", "mytheme")
				writeSource(t, fs, filepath.Join("layouts", "_default", "baseof.html"), `Base: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "_default", "baseof.html"), `Base Theme: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "_default", "list.html"), `{{define "main"}}list{{ end }}`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "sect1", "index.html"), "Base: list")
			},
		},
		{
			// Variant 4, theme, use themes's base
			func(t *testing.T) {
				cfg.Set("theme", "mytheme")
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "_default", "baseof.html"), `Base Theme: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "_default", "list.html"), `{{define "main"}}list{{ end }}`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "sect1", "index.html"), "Base Theme: list")
			},
		},
		{
			// Test section list and single template selection.
			// Issue #3116
			func(t *testing.T) {
				cfg.Set("theme", "mytheme")

				writeSource(t, fs, filepath.Join("layouts", "_default", "baseof.html"), `Base: {{block "main" .}}block{{end}}`)

				// Both single and list template in /SECTION/
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "sect1", "list.html"), `sect list`)
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "_default", "list.html"), `default list`)
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "sect1", "single.html"), `sect single`)
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "_default", "single.html"), `default single`)

				// sect2 with list template in /section
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "section", "sect2.html"), `sect2 list`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "sect1", "index.html"), "sect list")
				th.assertFileContent(filepath.Join("public", "sect1", "page1", "index.html"), "sect single")
				th.assertFileContent(filepath.Join("public", "sect2", "index.html"), "sect2 list")
			},
		},
		{
			// Test section list and single template selection with base template.
			// Issue #2995
			func(t *testing.T) {

				writeSource(t, fs, filepath.Join("layouts", "_default", "baseof.html"), `Base Default: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("layouts", "sect1", "baseof.html"), `Base Sect1: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("layouts", "section", "sect2-baseof.html"), `Base Sect2: {{block "main" .}}block{{end}}`)

				// Both single and list + base template in /SECTION/
				writeSource(t, fs, filepath.Join("layouts", "sect1", "list.html"), `{{define "main"}}sect1 list{{ end }}`)
				writeSource(t, fs, filepath.Join("layouts", "_default", "list.html"), `{{define "main"}}default list{{ end }}`)
				writeSource(t, fs, filepath.Join("layouts", "sect1", "single.html"), `{{define "main"}}sect single{{ end }}`)
				writeSource(t, fs, filepath.Join("layouts", "_default", "single.html"), `{{define "main"}}default single{{ end }}`)

				// sect2 with list template in /section
				writeSource(t, fs, filepath.Join("layouts", "section", "sect2.html"), `{{define "main"}}sect2 list{{ end }}`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "sect1", "index.html"), "Base Sect1", "sect1 list")
				th.assertFileContent(filepath.Join("public", "sect1", "page1", "index.html"), "Base Sect1", "sect single")
				th.assertFileContent(filepath.Join("public", "sect2", "index.html"), "Base Sect2", "sect2 list")

				// Note that this will get the default base template and not the one in /sect2 -- because there are no
				// single template defined in /sect2.
				th.assertFileContent(filepath.Join("public", "sect2", "page2", "index.html"), "Base Default", "default single")
			},
		},
	} {

		cfg, fs = newTestCfg()
		th = testHelper{cfg, fs, t}

		for i := 1; i <= 3; i++ {
			writeSource(t, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", i)), `---
title: Template test
---
Some content
`)
		}

		this.setup(t)

		buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{})
		t.Log("Template Lookup test", i)
		this.assert(t)

	}
}
<commit_msg>hugolib: Add additional test to TestTemplateLookupOrder<commit_after>// Copyright 2016 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hugolib

import (
	"fmt"
	"path/filepath"
	"testing"

	"github.com/gohugoio/hugo/deps"
	"github.com/gohugoio/hugo/hugofs"

	"github.com/spf13/viper"
)

func TestTemplateLookupOrder(t *testing.T) {
	t.Parallel()
	var (
		fs  *hugofs.Fs
		cfg *viper.Viper
		th  testHelper
	)

	// Variants base templates:
	//   1. <current-path>/<template-name>-baseof.<suffix>, e.g. list-baseof.<suffix>.
	//   2. <current-path>/baseof.<suffix>
	//   3. _default/<template-name>-baseof.<suffix>, e.g. list-baseof.<suffix>.
	//   4. _default/baseof.<suffix>
	for _, this := range []struct {
		name   string
		setup  func(t *testing.T)
		assert func(t *testing.T)
	}{
		{
			"Variant 1",
			func(t *testing.T) {
				writeSource(t, fs, filepath.Join("layouts", "section", "sect1-baseof.html"), `Base: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("layouts", "section", "sect1.html"), `{{define "main"}}sect{{ end }}`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "sect1", "index.html"), "Base: sect")
			},
		},
		{
			"Variant 2",
			func(t *testing.T) {
				writeSource(t, fs, filepath.Join("layouts", "baseof.html"), `Base: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("layouts", "index.html"), `{{define "main"}}index{{ end }}`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "index.html"), "Base: index")
			},
		},
		{
			"Variant 3",
			func(t *testing.T) {
				writeSource(t, fs, filepath.Join("layouts", "_default", "list-baseof.html"), `Base: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("layouts", "_default", "list.html"), `{{define "main"}}list{{ end }}`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "sect1", "index.html"), "Base: list")
			},
		},
		{
			"Variant 4",
			func(t *testing.T) {
				writeSource(t, fs, filepath.Join("layouts", "_default", "baseof.html"), `Base: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("layouts", "_default", "list.html"), `{{define "main"}}list{{ end }}`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "sect1", "index.html"), "Base: list")
			},
		},
		{
			"Variant 1, theme, use site base",
			func(t *testing.T) {
				cfg.Set("theme", "mytheme")
				writeSource(t, fs, filepath.Join("layouts", "section", "sect1-baseof.html"), `Base: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "section", "sect-baseof.html"), `Base Theme: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("layouts", "section", "sect1.html"), `{{define "main"}}sect{{ end }}`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "sect1", "index.html"), "Base: sect")
			},
		},
		{
			"Variant 1, theme, use theme base",
			func(t *testing.T) {
				cfg.Set("theme", "mytheme")
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "section", "sect1-baseof.html"), `Base Theme: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("layouts", "section", "sect1.html"), `{{define "main"}}sect{{ end }}`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "sect1", "index.html"), "Base Theme: sect")
			},
		},
		{
			"Variant 4, theme, use site base",
			func(t *testing.T) {
				cfg.Set("theme", "mytheme")
				writeSource(t, fs, filepath.Join("layouts", "_default", "baseof.html"), `Base: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "_default", "baseof.html"), `Base Theme: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "_default", "list.html"), `{{define "main"}}list{{ end }}`)
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "index.html"), `{{define "main"}}index{{ end }}`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "sect1", "index.html"), "Base: list")
				th.assertFileContent(filepath.Join("public", "index.html"), "Base: index") // Issue #3505
			},
		},
		{
			"Variant 4, theme, use themes base",
			func(t *testing.T) {
				cfg.Set("theme", "mytheme")
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "_default", "baseof.html"), `Base Theme: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "_default", "list.html"), `{{define "main"}}list{{ end }}`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "sect1", "index.html"), "Base Theme: list")
			},
		},
		{
			// Issue #3116
			"Test section list and single template selection",
			func(t *testing.T) {
				cfg.Set("theme", "mytheme")

				writeSource(t, fs, filepath.Join("layouts", "_default", "baseof.html"), `Base: {{block "main" .}}block{{end}}`)

				// Both single and list template in /SECTION/
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "sect1", "list.html"), `sect list`)
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "_default", "list.html"), `default list`)
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "sect1", "single.html"), `sect single`)
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "_default", "single.html"), `default single`)

				// sect2 with list template in /section
				writeSource(t, fs, filepath.Join("themes", "mytheme", "layouts", "section", "sect2.html"), `sect2 list`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "sect1", "index.html"), "sect list")
				th.assertFileContent(filepath.Join("public", "sect1", "page1", "index.html"), "sect single")
				th.assertFileContent(filepath.Join("public", "sect2", "index.html"), "sect2 list")
			},
		},
		{
			// Issue #2995
			"Test section list and single template selection with base template",
			func(t *testing.T) {

				writeSource(t, fs, filepath.Join("layouts", "_default", "baseof.html"), `Base Default: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("layouts", "sect1", "baseof.html"), `Base Sect1: {{block "main" .}}block{{end}}`)
				writeSource(t, fs, filepath.Join("layouts", "section", "sect2-baseof.html"), `Base Sect2: {{block "main" .}}block{{end}}`)

				// Both single and list + base template in /SECTION/
				writeSource(t, fs, filepath.Join("layouts", "sect1", "list.html"), `{{define "main"}}sect1 list{{ end }}`)
				writeSource(t, fs, filepath.Join("layouts", "_default", "list.html"), `{{define "main"}}default list{{ end }}`)
				writeSource(t, fs, filepath.Join("layouts", "sect1", "single.html"), `{{define "main"}}sect single{{ end }}`)
				writeSource(t, fs, filepath.Join("layouts", "_default", "single.html"), `{{define "main"}}default single{{ end }}`)

				// sect2 with list template in /section
				writeSource(t, fs, filepath.Join("layouts", "section", "sect2.html"), `{{define "main"}}sect2 list{{ end }}`)

			},
			func(t *testing.T) {
				th.assertFileContent(filepath.Join("public", "sect1", "index.html"), "Base Sect1", "sect1 list")
				th.assertFileContent(filepath.Join("public", "sect1", "page1", "index.html"), "Base Sect1", "sect single")
				th.assertFileContent(filepath.Join("public", "sect2", "index.html"), "Base Sect2", "sect2 list")

				// Note that this will get the default base template and not the one in /sect2 -- because there are no
				// single template defined in /sect2.
				th.assertFileContent(filepath.Join("public", "sect2", "page2", "index.html"), "Base Default", "default single")
			},
		},
	} {

		cfg, fs = newTestCfg()
		th = testHelper{cfg, fs, t}

		for i := 1; i <= 3; i++ {
			writeSource(t, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", i)), `---
title: Template test
---
Some content
`)
		}

		this.setup(t)

		buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{})
		t.Log(this.name)
		this.assert(t)

	}
}
<|endoftext|>


<commit_before>package gonx

import (
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/suite"
	"strings"
	"testing"
)

type ParserTestSuite struct {
	suite.Suite
	format string
	parser *Parser
}

func (suite *ParserTestSuite) SetupTest() {
	suite.format = "$remote_addr [$time_local] \"$request\" $status"
	suite.parser = NewParser(suite.format)
}

func TestParserTestSuite(t *testing.T) {
	suite.Run(t, new(ParserTestSuite))
}

func (suite *ParserTestSuite) TestFormatSaved() {
	assert.Equal(suite.T(), suite.parser.format, suite.format)
}

func (suite *ParserTestSuite) TestRegexp() {
	assert.Equal(suite.T(),
		suite.parser.regexp.String(),
		`^(?P<remote_addr>[^ ]*) \[(?P<time_local>[^]]*)\] "(?P<request>[^"]*)" (?P<status>[^ ]*)$`)
}

func (suite *ParserTestSuite) TestParseString() {
	line := `89.234.89.123 [08/Nov/2013:13:39:18 +0000] "GET /api/foo/bar HTTP/1.1" 200`
	expected := NewEntry(Fields{
		"remote_addr": "89.234.89.123",
		"time_local":  "08/Nov/2013:13:39:18 +0000",
		"request":     "GET /api/foo/bar HTTP/1.1",
		"status":      "200",
	})
	entry, err := suite.parser.ParseString(line)
	assert.NoError(suite.T(), err)
	assert.Equal(suite.T(), entry, expected)
}

func (suite *ParserTestSuite) TestParseInvalidString() {
	line := `GET /api/foo/bar HTTP/1.1`
	_, err := suite.parser.ParseString(line)
	assert.Error(suite.T(), err)
}

func (suite *ParserTestSuite) TestEmptyValue() {
	line := `89.234.89.123 [08/Nov/2013:13:39:18 +0000] "" 200`
	expected := NewEntry(Fields{
		"remote_addr": "89.234.89.123",
		"time_local":  "08/Nov/2013:13:39:18 +0000",
		"request":     "",
		"status":      "200",
	})
	entry, err := suite.parser.ParseString(line)
	assert.NoError(suite.T(), err)
	assert.Equal(suite.T(), entry, expected)
}

func TestNginxParser(t *testing.T) {
	expected := "$remote_addr - $remote_user [$time_local] \"$request\" $status \"$http_referer\" \"$http_user_agent\""
	conf := strings.NewReader(`
        http {
            include      conf/mime.types;
            log_format   minimal  '$remote_addr [$time_local] "$request"';
            log_format   main     '$remote_addr - $remote_user [$time_local] '
                                  '"$request" $status '
                                  '"$http_referer" "$http_user_agent"';
            log_format   download '$remote_addr - $remote_user [$time_local] '
                                  '"$request" $status $bytes_sent '
                                  '"$http_referer" "$http_user_agent" '
                                  '"$http_range" "$sent_http_content_range"';
        }
    `)
	parser, err := NewNginxParser(conf, "main")
	assert.NoError(t, err)
	assert.Equal(t, parser.format, expected)
}
<commit_msg>goconvey for parser tests<commit_after>package gonx

import (
	. "github.com/smartystreets/goconvey/convey"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/suite"
	"strings"
	"testing"
)

func TestParser(t *testing.T) {
	Convey("Test Parser", t, func() {
		format := "$remote_addr [$time_local] \"$request\" $status"
		parser := NewParser(format)

		Convey("Enfure parser format is ok", func() {
			So(parser.format, ShouldEqual, format)
		})
	})
}

type ParserTestSuite struct {
	suite.Suite
	format string
	parser *Parser
}

func (suite *ParserTestSuite) SetupTest() {
	suite.format = "$remote_addr [$time_local] \"$request\" $status"
	suite.parser = NewParser(suite.format)
}

func TestParserTestSuite(t *testing.T) {
	suite.Run(t, new(ParserTestSuite))
}

func (suite *ParserTestSuite) TestRegexp() {
	assert.Equal(suite.T(),
		suite.parser.regexp.String(),
		`^(?P<remote_addr>[^ ]*) \[(?P<time_local>[^]]*)\] "(?P<request>[^"]*)" (?P<status>[^ ]*)$`)
}

func (suite *ParserTestSuite) TestParseString() {
	line := `89.234.89.123 [08/Nov/2013:13:39:18 +0000] "GET /api/foo/bar HTTP/1.1" 200`
	expected := NewEntry(Fields{
		"remote_addr": "89.234.89.123",
		"time_local":  "08/Nov/2013:13:39:18 +0000",
		"request":     "GET /api/foo/bar HTTP/1.1",
		"status":      "200",
	})
	entry, err := suite.parser.ParseString(line)
	assert.NoError(suite.T(), err)
	assert.Equal(suite.T(), entry, expected)
}

func (suite *ParserTestSuite) TestParseInvalidString() {
	line := `GET /api/foo/bar HTTP/1.1`
	_, err := suite.parser.ParseString(line)
	assert.Error(suite.T(), err)
}

func (suite *ParserTestSuite) TestEmptyValue() {
	line := `89.234.89.123 [08/Nov/2013:13:39:18 +0000] "" 200`
	expected := NewEntry(Fields{
		"remote_addr": "89.234.89.123",
		"time_local":  "08/Nov/2013:13:39:18 +0000",
		"request":     "",
		"status":      "200",
	})
	entry, err := suite.parser.ParseString(line)
	assert.NoError(suite.T(), err)
	assert.Equal(suite.T(), entry, expected)
}

func TestNginxParser(t *testing.T) {
	expected := "$remote_addr - $remote_user [$time_local] \"$request\" $status \"$http_referer\" \"$http_user_agent\""
	conf := strings.NewReader(`
        http {
            include      conf/mime.types;
            log_format   minimal  '$remote_addr [$time_local] "$request"';
            log_format   main     '$remote_addr - $remote_user [$time_local] '
                                  '"$request" $status '
                                  '"$http_referer" "$http_user_agent"';
            log_format   download '$remote_addr - $remote_user [$time_local] '
                                  '"$request" $status $bytes_sent '
                                  '"$http_referer" "$http_user_agent" '
                                  '"$http_range" "$sent_http_content_range"';
        }
    `)
	parser, err := NewNginxParser(conf, "main")
	assert.NoError(t, err)
	assert.Equal(t, parser.format, expected)
}
<|endoftext|>


<commit_before>package hook

import (
	"path/filepath"
	"runtime"
	"strconv"
	"strings"

	"github.com/Sirupsen/logrus"
)

// CallerHook 需要的钩子
type CallerHook struct {
}

// Fire 实现
func (hook *CallerHook) Fire(entry *logrus.Entry) error {
	var delta int
	if _, ok := entry.Data["caller"]; ok {
		delta = -1
	}
	if len(entry.Data)+delta == 0 { // if don't use withfields
		entry.Data["caller"] = hook.caller(7)
	} else { // if use withfields
		entry.Data["caller"] = hook.caller(6)
	}
	return nil
}

// Levels 实现的级别
func (hook *CallerHook) Levels() []logrus.Level {
	return []logrus.Level{
		logrus.PanicLevel,
		logrus.FatalLevel,
		logrus.ErrorLevel,
		logrus.WarnLevel,
		logrus.InfoLevel,
		logrus.DebugLevel,
	}
}

// caller 带withfields的函数调用
func (hook *CallerHook) caller(skip int) string {
	if _, file, line, ok := runtime.Caller(skip); ok {
		return strings.Join([]string{filepath.Base(file), strconv.Itoa(line)}, ":")
	}
	// not sure what the convention should be here
	return "???"
}
<commit_msg>change caller depth<commit_after>package hook

import (
	"path/filepath"
	"runtime"
	"strconv"
	"strings"

	"github.com/sirupsen/logrus"
)

// CallerHook 需要的钩子
type CallerHook struct {
}

// Fire 实现
func (hook *CallerHook) Fire(entry *logrus.Entry) error {
	var delta int
	if _, ok := entry.Data["caller"]; ok {
		delta = -1
	}
	if len(entry.Data)+delta == 0 { // if don't use withfields
		entry.Data["caller"] = hook.caller(9)
	} else { // if use withfields
		entry.Data["caller"] = hook.caller(7)
	}
	return nil
}

// Levels 实现的级别
func (hook *CallerHook) Levels() []logrus.Level {
	return []logrus.Level{
		logrus.PanicLevel,
		logrus.FatalLevel,
		logrus.ErrorLevel,
		logrus.WarnLevel,
		logrus.InfoLevel,
		logrus.DebugLevel,
	}
}

// caller 带withfields的函数调用
func (hook *CallerHook) caller(skip int) string {
	if _, file, line, ok := runtime.Caller(skip); ok {
		return strings.Join([]string{filepath.Base(file), strconv.Itoa(line)}, ":")
	}
	// not sure what the convention should be here
	return "???"
}
<|endoftext|>


<commit_before>package matching

import (
	"github.com/SpectoLabs/hoverfly/core/models"
)

func FieldMatcher(field *models.RequestFieldMatchers, toMatch string) bool {
	matches := []bool{}
	if field == nil {
		return true
	}

	if field.ExactMatch != nil {
		matches = append(matches, ExactMatch(*field.ExactMatch, toMatch))
	}

	if field.XpathMatch != nil {
		matches = append(matches, XpathMatch(*field.XpathMatch, toMatch))
	}

	if field.JsonPathMatch != nil {
		matches = append(matches, JsonPathMatch(*field.JsonPathMatch, toMatch))
	}

	if field.RegexMatch != nil {
		matches = append(matches, RegexMatch(*field.RegexMatch, toMatch))
	}

	if field.GlobMatch != nil {
		matches = append(matches, GlobMatch(*field.GlobMatch, toMatch))
	}

	for _, match := range matches {
		if !match {
			return false
		}

	}

	return true
}
<commit_msg>Removed boolean array and loop from FieldMatcher<commit_after>package matching

import (
	"github.com/SpectoLabs/hoverfly/core/models"
)

func FieldMatcher(field *models.RequestFieldMatchers, toMatch string) bool {
	if field == nil {
		return true
	}

	if field.ExactMatch != nil && !ExactMatch(*field.ExactMatch, toMatch) {
		return false
	}

	if field.XpathMatch != nil && !XpathMatch(*field.XpathMatch, toMatch) {
		return false
	}

	if field.JsonPathMatch != nil && !JsonPathMatch(*field.JsonPathMatch, toMatch) {
		return false
	}

	if field.RegexMatch != nil && !RegexMatch(*field.RegexMatch, toMatch) {
		return false
	}

	if field.GlobMatch != nil && !GlobMatch(*field.GlobMatch, toMatch) {
		return false
	}

	return true
}
<|endoftext|>


<commit_before>package chartjs

import (
	"encoding/json"
	"html/template"
	"io"
)

// this file implements some syntactic sugar for creating charts

// JQuery holds the path to hosted JQuery
var JQuery = "https://code.jquery.com/jquery-2.2.4.min.js"

// ChartJS holds the path to hosted ChartJS
var ChartJS = "https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.5.0/Chart.bundle.js"

const tmpl = `<!DOCTYPE html>

	{{ $height := index . "height" }}
	{{ $width := index . "width" }}
	{{ range $i, $json := index . "charts" }}
	<canvas id="canvas{{ $i }}" style="height:{{ $height }}px;width:{{ $width }}px"></canvas>

	{{ end }}
	{{ index . "customHTML" }}

`

// SaveCharts writes the charts and the required HTML to an io.Writer
func SaveCharts(w io.Writer, tmap map[string]interface{}, charts ...Chart) error {
	if tmap == nil {
		tmap = make(map[string]interface{})
	}

	if _, ok := tmap["height"]; !ok {
		tmap["height"] = 400
	}
	if _, ok := tmap["width"]; !ok {
		tmap["width"] = 400
	}
	jscharts := make([]template.JS, 0, len(charts))
	for _, c := range charts {
		cjson, err := json.Marshal(c)
		if err != nil {
			return err
		}
		jscharts = append(jscharts, template.JS(cjson))
	}
	for k, v := range tmap {
		if chart, ok := v.(Chart); ok {
			cjson, err := json.Marshal(chart)
			if err != nil {
				return err
			}
			tmap[k] = template.JS(cjson)
		}
	}

	tmap["charts"] = jscharts
	if _, ok := tmap["JQuery"]; !ok {
		tmap["JQuery"] = JQuery
	}
	if _, ok := tmap["ChartJS"]; !ok {
		tmap["ChartJS"] = ChartJS
	}
	if _, ok := tmap["custom"]; !ok {
		tmap["custom"] = ""
	}
	if _, ok := tmap["customHTML"]; !ok {
		tmap["customHTML"] = ""
	}
	if _, ok := tmap["template"]; !ok {
		tmap["template"] = tmpl
	}
	t, err := template.New("chartjs").Parse(tmap["template"].(string))
	if err != nil {
		return err
	}
	return t.Execute(w, tmap)
}

// SaveHTML writes the chart and minimal HTML to an io.Writer.
func (c Chart) SaveHTML(w io.Writer, tmap map[string]interface{}) error {
	return SaveCharts(w, tmap, c)
}
<commit_msg>bump to latest Chart.js bundle<commit_after>package chartjs

import (
	"encoding/json"
	"html/template"
	"io"
)

// this file implements some syntactic sugar for creating charts

// JQuery holds the path to hosted JQuery
var JQuery = "https://code.jquery.com/jquery-2.2.4.min.js"

// ChartJS holds the path to hosted ChartJS
var ChartJS = "https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.6.0/Chart.bundle.js"

const tmpl = `<!DOCTYPE html>

	{{ $height := index . "height" }}
	{{ $width := index . "width" }}
	{{ range $i, $json := index . "charts" }}
	<canvas id="canvas{{ $i }}" style="height:{{ $height }}px;width:{{ $width }}px"></canvas>

	{{ end }}
	{{ index . "customHTML" }}

`

// SaveCharts writes the charts and the required HTML to an io.Writer
func SaveCharts(w io.Writer, tmap map[string]interface{}, charts ...Chart) error {
	if tmap == nil {
		tmap = make(map[string]interface{})
	}

	if _, ok := tmap["height"]; !ok {
		tmap["height"] = 400
	}
	if _, ok := tmap["width"]; !ok {
		tmap["width"] = 400
	}
	jscharts := make([]template.JS, 0, len(charts))
	for _, c := range charts {
		cjson, err := json.Marshal(c)
		if err != nil {
			return err
		}
		jscharts = append(jscharts, template.JS(cjson))
	}
	for k, v := range tmap {
		if chart, ok := v.(Chart); ok {
			cjson, err := json.Marshal(chart)
			if err != nil {
				return err
			}
			tmap[k] = template.JS(cjson)
		}
	}

	tmap["charts"] = jscharts
	if _, ok := tmap["JQuery"]; !ok {
		tmap["JQuery"] = JQuery
	}
	if _, ok := tmap["ChartJS"]; !ok {
		tmap["ChartJS"] = ChartJS
	}
	if _, ok := tmap["custom"]; !ok {
		tmap["custom"] = ""
	}
	if _, ok := tmap["customHTML"]; !ok {
		tmap["customHTML"] = ""
	}
	if _, ok := tmap["template"]; !ok {
		tmap["template"] = tmpl
	}
	t, err := template.New("chartjs").Parse(tmap["template"].(string))
	if err != nil {
		return err
	}
	return t.Execute(w, tmap)
}

// SaveHTML writes the chart and minimal HTML to an io.Writer.
func (c Chart) SaveHTML(w io.Writer, tmap map[string]interface{}) error {
	return SaveCharts(w, tmap, c)
}
<|endoftext|>


<commit_before>package goutils

import (
	"math"
	"runtime"
)

type concurrencyLimit struct {
	lock chan (bool)
}

func NewConcurrencyLimit(limit int) concurrencyLimit {
	ob := concurrencyLimit{}
	ob.lock = make(chan bool, limit)
	return ob
}

func NewConcurrencyLimitCPU() concurrencyLimit {
	return NewConcurrencyLimit(runtime.NumCPU())
}

func NewConcurrencyLimitCPUFactore(f float64) concurrencyLimit {
	t := float64(runtime.NumCPU()) * f
	t2 := math.Ceil(t)
	return NewConcurrencyLimit(int(t2))
}

func NewConcurrencyLimitOne() concurrencyLimit {
	return NewConcurrencyLimit(1)
}

func (l *concurrencyLimit) Use() {
	l.lock <- true
}

func (l *concurrencyLimit) Free() {
	<-l.lock
}

// usage: defer x.Limit()()
//                     ^ ^ double ()
func (l *concurrencyLimit) Limit() func() {
	l.Use()
	return func() {
		l.Free()
	}
}
<commit_msg>concurrencyLimit added Len() and Size()<commit_after>package goutils

import (
	"math"
	"runtime"
)

type concurrencyLimit struct {
	lock chan (bool)
	size int
}

func NewConcurrencyLimit(limit int) concurrencyLimit {
	ob := concurrencyLimit{}
	ob.lock = make(chan bool, limit)
	ob.size = limit
	return ob
}

func NewConcurrencyLimitCPU() concurrencyLimit {
	return NewConcurrencyLimit(runtime.NumCPU())
}

func NewConcurrencyLimitCPUFactore(f float64) concurrencyLimit {
	t := float64(runtime.NumCPU()) * f
	t2 := math.Ceil(t)
	return NewConcurrencyLimit(int(t2))
}

func NewConcurrencyLimitOne() concurrencyLimit {
	return NewConcurrencyLimit(1)
}

func (l *concurrencyLimit) Use() {
	l.lock <- true
}

func (l *concurrencyLimit) Free() {
	<-l.lock
}

func (l *concurrencyLimit) Len() int {
	return len(l.lock)
}

func (l *concurrencyLimit) Size() int {
	return l.size
}

// usage: defer x.Limit()()
//                     ^ ^ double ()
func (l *concurrencyLimit) Limit() func() {
	l.Use()
	return func() {
		l.Free()
	}
}
<|endoftext|>


<commit_before>package network

import (
	"github.com/Nekitosss/authorization/network/controller/structures"
	"encoding/json"
	"io"
	"net/http"
	"time"

	"github.com/gorilla/mux"
	"github.com/satori/go.uuid"
)

type serverResponse map[string]interface{}

func AuthVerifiedUser(writer http.ResponseWriter, request *http.Request) {

	var params structures.AuthInfo

	err := json.NewDecoder(request.Body).Decode(&params)

	if err != nil && err != io.EOF {
		setErrorResult(writer, err)
		return
	}

	session, err := executor.AuthUser(params)

	if err != nil {
		setErrorResult(writer, err)

	} else {
		expiration := time.Now().AddDate(10, 0, 0)
		cookie := http.Cookie{Name: "session", Value: session.SessionID.String(), Expires: expiration}
		http.SetCookie(writer, &cookie)

		var result = serverResponse{"userID" : session.UserModelID.String()}
		json.NewEncoder(writer).Encode(result)
	}
}

func VerifyRegistration(w http.ResponseWriter, r *http.Request) {

	var registrationIDString = mux.Vars(r)["id"]

	var registrationID, err = uuid.FromString(registrationIDString)

	if err != nil {
		setErrorResult(w, err)
		return
	}

	err = executor.VerifyRegister(registrationID)

	if err != nil {
		setErrorResult(w, err)
		return
	}

	http.Redirect(w, r, "fooddly://authorization", 301)
}

func Login(writer http.ResponseWriter, request *http.Request) {

	var params structures.LoginInfo

	err := json.NewDecoder(request.Body).Decode(&params)

	if err != nil && err != io.EOF {
		setErrorResult(writer, err)
		return
	}

	registrationID, err := executor.Login(params)

	if err != nil {
		setErrorResult(writer, err)

	} else {
		result := serverResponse{"registrationID": registrationID.String()}
		json.NewEncoder(writer).Encode(result)
	}
}

func ValidateSession(writer http.ResponseWriter, request *http.Request) {

	var info structures.ValidateSessionInfo
	err := json.NewDecoder(request.Body).Decode(&info)

	if err != nil {
		setErrorResult(writer, err)
		return
	}

	userID, err := executor.ValidateSession(info)

	if err != nil {
		setErrorResult(writer, err)
		return
	}

	var result = serverResponse{"userID": userID.String()}
	json.NewEncoder(writer).Encode(&result)
}

func sendSimpleSuccess(w http.ResponseWriter) {
	result := serverResponse{"success": true}

	json.NewEncoder(w).Encode(result)
}
<commit_msg>send html response<commit_after>package network

import (
	"github.com/Nekitosss/authorization/network/controller/structures"
	"encoding/json"
	"io"
	"net/http"
	"time"

	"github.com/gorilla/mux"
	"github.com/satori/go.uuid"
)

type serverResponse map[string]interface{}

func AuthVerifiedUser(writer http.ResponseWriter, request *http.Request) {

	var params structures.AuthInfo

	err := json.NewDecoder(request.Body).Decode(&params)

	if err != nil && err != io.EOF {
		setErrorResult(writer, err)
		return
	}

	session, err := executor.AuthUser(params)

	if err != nil {
		setErrorResult(writer, err)

	} else {
		expiration := time.Now().AddDate(10, 0, 0)
		cookie := http.Cookie{Name: "session", Value: session.SessionID.String(), Expires: expiration}
		http.SetCookie(writer, &cookie)

		var result = serverResponse{"userID" : session.UserModelID.String()}
		json.NewEncoder(writer).Encode(result)
	}
}

func VerifyRegistration(w http.ResponseWriter, r *http.Request) {

	var registrationIDString = mux.Vars(r)["id"]

	var registrationID, err = uuid.FromString(registrationIDString)

	if err != nil {
		setErrorResult(w, err)
		return
	}

	err = executor.VerifyRegister(registrationID)

	if err != nil {
		setErrorResult(w, err)
		return
	}

	var html = `
	<!DOCTYPE html>

	<title>Success confirmation</title>
	<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />

Ok result, you should go back to application

	`

	r.Write(html)
}

func Login(writer http.ResponseWriter, request *http.Request) {

	var params structures.LoginInfo

	err := json.NewDecoder(request.Body).Decode(&params)

	if err != nil && err != io.EOF {
		setErrorResult(writer, err)
		return
	}

	registrationID, err := executor.Login(params)

	if err != nil {
		setErrorResult(writer, err)

	} else {
		result := serverResponse{"registrationID": registrationID.String()}
		json.NewEncoder(writer).Encode(result)
	}
}

func ValidateSession(writer http.ResponseWriter, request *http.Request) {

	var info structures.ValidateSessionInfo
	err := json.NewDecoder(request.Body).Decode(&info)

	if err != nil {
		setErrorResult(writer, err)
		return
	}

	userID, err := executor.ValidateSession(info)

	if err != nil {
		setErrorResult(writer, err)
		return
	}

	var result = serverResponse{"userID": userID.String()}
	json.NewEncoder(writer).Encode(&result)
}

func sendSimpleSuccess(w http.ResponseWriter) {
	result := serverResponse{"success": true}

	json.NewEncoder(w).Encode(result)
}
<|endoftext|>


<commit_before>package gateway

import (
	"io"
	"sync"

	"google.golang.org/grpc/metadata"

	context "golang.org/x/net/context"

	pb "github.com/TheThingsNetwork/ttn/api/gateway"
	pb_noc "github.com/TheThingsNetwork/ttn/api/monitor"
	pb_router "github.com/TheThingsNetwork/ttn/api/router"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/apex/log"
)

func (g *Gateway) monitorContext() (ctx context.Context) {
	return metadata.NewContext(context.Background(), metadata.Pairs(
		"id", g.ID,
		"token", g.Token,
	))
}

type monitorConn struct {
	clients map[string]pb_noc.MonitorClient

	status struct {
		streams map[string]pb_noc.Monitor_GatewayStatusClient
		sync.RWMutex
	}

	uplink struct {
		streams map[string]pb_noc.Monitor_GatewayUplinkClient
		sync.RWMutex
	}

	downlink struct {
		streams map[string]pb_noc.Monitor_GatewayDownlinkClient
		sync.RWMutex
	}
}

func (g *Gateway) SetMonitors(clients map[string]pb_noc.MonitorClient) {
	g.monitor = NewMonitorConn(clients)
}

func NewMonitorConn(clients map[string]pb_noc.MonitorClient) (conn *monitorConn) {
	conn = &monitorConn{clients: clients}
	conn.uplink.streams = map[string]pb_noc.Monitor_GatewayUplinkClient{}
	conn.downlink.streams = map[string]pb_noc.Monitor_GatewayDownlinkClient{}
	conn.status.streams = map[string]pb_noc.Monitor_GatewayStatusClient{}
	return conn
}

func (g *Gateway) pushStatusToMonitor(ctx log.Interface, name string, status *pb.Status) (err error) {
	defer func() {
		switch err {
		case nil:
			ctx.Info("Pushed status to monitor")

		case io.EOF:
			ctx.Info("Stream closed, retrying")
			g.monitor.status.Lock()
			delete(g.monitor.status.streams, name)
			g.monitor.status.Unlock()
			err = g.pushStatusToMonitor(ctx, name, status)

		default:
			ctx.WithError(errors.FromGRPCError(err)).Warn("Monitor status push failed")
		}
	}()

	g.monitor.status.RLock()
	stream, ok := g.monitor.status.streams[name]
	g.monitor.status.RUnlock()

	if !ok {
		g.monitor.status.Lock()
		if _, ok := g.monitor.status.streams[name]; !ok {
			cl, ok := g.monitor.clients[name]
			if !ok {
				// Should not happen
				return errors.New("Monitor not found")
			}

			if stream, err = cl.GatewayStatus(g.monitorContext()); err != nil {
				// TODO check if err returned is GRPC error
				err = errors.FromGRPCError(err)
				ctx.WithError(err).Warn("Failed to open new status stream")
				return err
			}
			ctx.Info("Opened new status stream")
			g.monitor.status.streams[name] = stream
		}
		g.monitor.status.Unlock()
	}

	return stream.Send(status)
}

func (g *Gateway) pushUplinkToMonitor(ctx log.Interface, name string, uplink *pb_router.UplinkMessage) (err error) {
	defer func() {
		switch err {
		case nil:
			ctx.Info("Pushed uplink to monitor")

		case io.EOF:
			ctx.Info("Stream closed, retrying")
			g.monitor.uplink.Lock()
			delete(g.monitor.uplink.streams, name)
			g.monitor.uplink.Unlock()
			err = g.pushUplinkToMonitor(ctx, name, uplink)

		default:
			ctx.WithError(errors.FromGRPCError(err)).Warn("Monitor uplink push failed")
		}
	}()

	g.monitor.uplink.RLock()
	stream, ok := g.monitor.uplink.streams[name]
	g.monitor.uplink.RUnlock()

	if !ok {
		g.monitor.uplink.Lock()
		if _, ok := g.monitor.uplink.streams[name]; !ok {
			cl, ok := g.monitor.clients[name]
			if !ok {
				// Should not happen
				return errors.New("Monitor not found")
			}

			if stream, err = cl.GatewayUplink(g.monitorContext()); err != nil {
				// TODO check if err returned is GRPC error
				err = errors.FromGRPCError(err)
				ctx.WithError(err).Warn("Failed to open new uplink stream")
				return err
			}
			g.monitor.uplink.streams[name] = stream
		}
		g.monitor.uplink.Unlock()
	}

	return stream.Send(uplink)
}

func (g *Gateway) pushDownlinkToMonitor(ctx log.Interface, name string, downlink *pb_router.DownlinkMessage) (err error) {
	defer func() {
		switch err {
		case nil:
			ctx.Info("Pushed downlink to monitor")

		case io.EOF:
			ctx.Info("Stream closed, retrying")
			g.monitor.downlink.Lock()
			delete(g.monitor.downlink.streams, name)
			g.monitor.downlink.Unlock()
			err = g.pushDownlinkToMonitor(ctx, name, downlink)

		default:
			ctx.WithError(errors.FromGRPCError(err)).Warn("Monitor downlink push failed")
		}
	}()

	g.monitor.downlink.RLock()
	stream, ok := g.monitor.downlink.streams[name]
	g.monitor.downlink.RUnlock()

	if !ok {
		g.monitor.downlink.Lock()
		if _, ok := g.monitor.downlink.streams[name]; !ok {
			cl, ok := g.monitor.clients[name]
			if !ok {
				// Should not happen
				return errors.New("Monitor not found")
			}

			if stream, err = cl.GatewayDownlink(g.monitorContext()); err != nil {
				// TODO check if err returned is GRPC error
				err = errors.FromGRPCError(err)
				ctx.WithError(err).Warn("Failed to open new downlink stream")
				return err
			}
			ctx.Info("Opened new downlink stream")
			g.monitor.downlink.streams[name] = stream
		}
		g.monitor.downlink.Unlock()
	}

	return stream.Send(downlink)
}
<commit_msg>core/router/gateway/monitor.go: import fix<commit_after>package gateway

import (
	"io"
	"sync"

	pb "github.com/TheThingsNetwork/ttn/api/gateway"
	pb_noc "github.com/TheThingsNetwork/ttn/api/monitor"
	pb_router "github.com/TheThingsNetwork/ttn/api/router"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/apex/log"
	context "golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
)

func (g *Gateway) monitorContext() (ctx context.Context) {
	return metadata.NewContext(context.Background(), metadata.Pairs(
		"id", g.ID,
		"token", g.Token,
	))
}

type monitorConn struct {
	clients map[string]pb_noc.MonitorClient

	status struct {
		streams map[string]pb_noc.Monitor_GatewayStatusClient
		sync.RWMutex
	}

	uplink struct {
		streams map[string]pb_noc.Monitor_GatewayUplinkClient
		sync.RWMutex
	}

	downlink struct {
		streams map[string]pb_noc.Monitor_GatewayDownlinkClient
		sync.RWMutex
	}
}

func (g *Gateway) SetMonitors(clients map[string]pb_noc.MonitorClient) {
	g.monitor = NewMonitorConn(clients)
}

func NewMonitorConn(clients map[string]pb_noc.MonitorClient) (conn *monitorConn) {
	conn = &monitorConn{clients: clients}
	conn.uplink.streams = map[string]pb_noc.Monitor_GatewayUplinkClient{}
	conn.downlink.streams = map[string]pb_noc.Monitor_GatewayDownlinkClient{}
	conn.status.streams = map[string]pb_noc.Monitor_GatewayStatusClient{}
	return conn
}

func (g *Gateway) pushStatusToMonitor(ctx log.Interface, name string, status *pb.Status) (err error) {
	defer func() {
		switch err {
		case nil:
			ctx.Info("Pushed status to monitor")

		case io.EOF:
			ctx.Info("Stream closed, retrying")
			g.monitor.status.Lock()
			delete(g.monitor.status.streams, name)
			g.monitor.status.Unlock()
			err = g.pushStatusToMonitor(ctx, name, status)

		default:
			ctx.WithError(errors.FromGRPCError(err)).Warn("Monitor status push failed")
		}
	}()

	g.monitor.status.RLock()
	stream, ok := g.monitor.status.streams[name]
	g.monitor.status.RUnlock()

	if !ok {
		g.monitor.status.Lock()
		if _, ok := g.monitor.status.streams[name]; !ok {
			cl, ok := g.monitor.clients[name]
			if !ok {
				// Should not happen
				return errors.New("Monitor not found")
			}

			if stream, err = cl.GatewayStatus(g.monitorContext()); err != nil {
				// TODO check if err returned is GRPC error
				err = errors.FromGRPCError(err)
				ctx.WithError(err).Warn("Failed to open new status stream")
				return err
			}
			ctx.Info("Opened new status stream")
			g.monitor.status.streams[name] = stream
		}
		g.monitor.status.Unlock()
	}

	return stream.Send(status)
}

func (g *Gateway) pushUplinkToMonitor(ctx log.Interface, name string, uplink *pb_router.UplinkMessage) (err error) {
	defer func() {
		switch err {
		case nil:
			ctx.Info("Pushed uplink to monitor")

		case io.EOF:
			ctx.Info("Stream closed, retrying")
			g.monitor.uplink.Lock()
			delete(g.monitor.uplink.streams, name)
			g.monitor.uplink.Unlock()
			err = g.pushUplinkToMonitor(ctx, name, uplink)

		default:
			ctx.WithError(errors.FromGRPCError(err)).Warn("Monitor uplink push failed")
		}
	}()

	g.monitor.uplink.RLock()
	stream, ok := g.monitor.uplink.streams[name]
	g.monitor.uplink.RUnlock()

	if !ok {
		g.monitor.uplink.Lock()
		if _, ok := g.monitor.uplink.streams[name]; !ok {
			cl, ok := g.monitor.clients[name]
			if !ok {
				// Should not happen
				return errors.New("Monitor not found")
			}

			if stream, err = cl.GatewayUplink(g.monitorContext()); err != nil {
				// TODO check if err returned is GRPC error
				err = errors.FromGRPCError(err)
				ctx.WithError(err).Warn("Failed to open new uplink stream")
				return err
			}
			g.monitor.uplink.streams[name] = stream
		}
		g.monitor.uplink.Unlock()
	}

	return stream.Send(uplink)
}

func (g *Gateway) pushDownlinkToMonitor(ctx log.Interface, name string, downlink *pb_router.DownlinkMessage) (err error) {
	defer func() {
		switch err {
		case nil:
			ctx.Info("Pushed downlink to monitor")

		case io.EOF:
			ctx.Info("Stream closed, retrying")
			g.monitor.downlink.Lock()
			delete(g.monitor.downlink.streams, name)
			g.monitor.downlink.Unlock()
			err = g.pushDownlinkToMonitor(ctx, name, downlink)

		default:
			ctx.WithError(errors.FromGRPCError(err)).Warn("Monitor downlink push failed")
		}
	}()

	g.monitor.downlink.RLock()
	stream, ok := g.monitor.downlink.streams[name]
	g.monitor.downlink.RUnlock()

	if !ok {
		g.monitor.downlink.Lock()
		if _, ok := g.monitor.downlink.streams[name]; !ok {
			cl, ok := g.monitor.clients[name]
			if !ok {
				// Should not happen
				return errors.New("Monitor not found")
			}

			if stream, err = cl.GatewayDownlink(g.monitorContext()); err != nil {
				// TODO check if err returned is GRPC error
				err = errors.FromGRPCError(err)
				ctx.WithError(err).Warn("Failed to open new downlink stream")
				return err
			}
			ctx.Info("Opened new downlink stream")
			g.monitor.downlink.streams[name] = stream
		}
		g.monitor.downlink.Unlock()
	}

	return stream.Send(downlink)
}
<|endoftext|>


<commit_before>package main

import (
	"./update/types"
	"bytes"
	"crypto/sha1"
	"crypto/sha256"
	"encoding/base64"
	"encoding/xml"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
)

var cmdNewVersion = &Command{
	UsageLine: "new-version -k [key] -a [app-id] -v [version] -t [track] -p [url path] [filename]",
	Short:     "update the version database for a given file",
	Long: `
Takes a file path and some meta data and update the information used in the datastore.
	`,
}

var versionD = cmdNewVersion.Flag.Bool("d", false, "dry run, print out the xml payload")
var versionK = cmdNewVersion.Flag.String("k", "", "api key for the admin user")

var versionA = cmdNewVersion.Flag.String("a", "", "application id")
var versionV = cmdNewVersion.Flag.String("v", "", "version ")
var versionT = cmdNewVersion.Flag.String("t", "", "track")
var versionP = cmdNewVersion.Flag.String("p", "", "url path")

func init() {
	cmdNewVersion.Run = runNewVersion
}

func calculateHashes(filename string, pkg *types.Package) {
	var (
		writers []io.Writer
		hashes  []hash.Hash
	)

	push := func(h hash.Hash) {
		writers = append(writers, h)
		hashes = append(hashes, h)
	}

	push(sha256.New())
	push(sha1.New())

	in, err := os.Open(filename)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	io.Copy(io.MultiWriter(writers...), in)

	formatHash := func(hash hash.Hash) string {
		return base64.StdEncoding.EncodeToString(hash.Sum(nil))
	}

	pkg.Sha256Sum = formatHash(hashes[0])
	pkg.Sha1Sum = formatHash(hashes[1])
}

func runNewVersion(cmd *Command, args []string) {
	dryRun := *versionD
	key := *versionK
	appId := *versionA
	version := *versionV
	track := *versionT
	path := *versionP

	if dryRun == false && key == "" {
		fmt.Printf("key or dry-run required")
		os.Exit(-1)
	}

	if appId == "" || version == "" || track == "" || path == "" {
		fmt.Printf("one of the required fields was not present\n")
		os.Exit(-1)
	}

	if len(args) != 1 {
		fmt.Printf("update file name not provided\n")
		os.Exit(-1)
	}

	file := args[0]
	fileBase := filepath.Base(file)
	fi, err := os.Stat(file)
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		os.Exit(-1)
	}

	fileSize := strconv.FormatInt(fi.Size(), 10)

	app := types.App{Id: appId, Version: version, Track: track}
	pkg := types.Package{Name: fileBase, Size: fileSize, Path: path}
	ver := types.Version{App: &app, Package: &pkg}
	calculateHashes(file, ver.Package)

	raw, err := xml.MarshalIndent(ver, "", " ")
	if err != nil {
		fmt.Printf(err.Error())
		os.Exit(-1)
	}

	body := []byte(xml.Header)
	body = append(body, raw...)

	adminURL, _ := url.Parse(updateURL.String())
	adminURL.Path = "/admin/version"

	req, _ := http.NewRequest("POST", adminURL.String(), bytes.NewBuffer(body))
	req.Header.Set("Content-Type", "text/xml")
	req.SetBasicAuth("admin", key)

	if dryRun || *debug {
		req.Write(os.Stdout)
	}

	if dryRun {
		return
	}

	client := http.Client{}
	resp, err := client.Do(req)

	if err != nil {
		fmt.Printf(err.Error())
		os.Exit(-1)
	}

	body, _ = ioutil.ReadAll(resp.Body)
	os.Stdout.Write(body)
	fmt.Printf("\n")

	if resp.StatusCode != 200 {
		fmt.Printf("Error: bad return code %s\n", resp.Status)
		os.Exit(-1)
	}

	return
}
<commit_msg>fix(new-version): use the full package path<commit_after>package main

import (
	"github.com/coreos/core-admin/update/types"
	"bytes"
	"crypto/sha1"
	"crypto/sha256"
	"encoding/base64"
	"encoding/xml"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
)

var cmdNewVersion = &Command{
	UsageLine: "new-version -k [key] -a [app-id] -v [version] -t [track] -p [url path] [filename]",
	Short:     "update the version database for a given file",
	Long: `
Takes a file path and some meta data and update the information used in the datastore.
	`,
}

var versionD = cmdNewVersion.Flag.Bool("d", false, "dry run, print out the xml payload")
var versionK = cmdNewVersion.Flag.String("k", "", "api key for the admin user")

var versionA = cmdNewVersion.Flag.String("a", "", "application id")
var versionV = cmdNewVersion.Flag.String("v", "", "version ")
var versionT = cmdNewVersion.Flag.String("t", "", "track")
var versionP = cmdNewVersion.Flag.String("p", "", "url path")

func init() {
	cmdNewVersion.Run = runNewVersion
}

func calculateHashes(filename string, pkg *types.Package) {
	var (
		writers []io.Writer
		hashes  []hash.Hash
	)

	push := func(h hash.Hash) {
		writers = append(writers, h)
		hashes = append(hashes, h)
	}

	push(sha256.New())
	push(sha1.New())

	in, err := os.Open(filename)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	io.Copy(io.MultiWriter(writers...), in)

	formatHash := func(hash hash.Hash) string {
		return base64.StdEncoding.EncodeToString(hash.Sum(nil))
	}

	pkg.Sha256Sum = formatHash(hashes[0])
	pkg.Sha1Sum = formatHash(hashes[1])
}

func runNewVersion(cmd *Command, args []string) {
	dryRun := *versionD
	key := *versionK
	appId := *versionA
	version := *versionV
	track := *versionT
	path := *versionP

	if dryRun == false && key == "" {
		fmt.Printf("key or dry-run required")
		os.Exit(-1)
	}

	if appId == "" || version == "" || track == "" || path == "" {
		fmt.Printf("one of the required fields was not present\n")
		os.Exit(-1)
	}

	if len(args) != 1 {
		fmt.Printf("update file name not provided\n")
		os.Exit(-1)
	}

	file := args[0]
	fileBase := filepath.Base(file)
	fi, err := os.Stat(file)
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		os.Exit(-1)
	}

	fileSize := strconv.FormatInt(fi.Size(), 10)

	app := types.App{Id: appId, Version: version, Track: track}
	pkg := types.Package{Name: fileBase, Size: fileSize, Path: path}
	ver := types.Version{App: &app, Package: &pkg}
	calculateHashes(file, ver.Package)

	raw, err := xml.MarshalIndent(ver, "", " ")
	if err != nil {
		fmt.Printf(err.Error())
		os.Exit(-1)
	}

	body := []byte(xml.Header)
	body = append(body, raw...)

	adminURL, _ := url.Parse(updateURL.String())
	adminURL.Path = "/admin/version"

	req, _ := http.NewRequest("POST", adminURL.String(), bytes.NewBuffer(body))
	req.Header.Set("Content-Type", "text/xml")
	req.SetBasicAuth("admin", key)

	if dryRun || *debug {
		req.Write(os.Stdout)
	}

	if dryRun {
		return
	}

	client := http.Client{}
	resp, err := client.Do(req)

	if err != nil {
		fmt.Printf(err.Error())
		os.Exit(-1)
	}

	body, _ = ioutil.ReadAll(resp.Body)
	os.Stdout.Write(body)
	fmt.Printf("\n")

	if resp.StatusCode != 200 {
		fmt.Printf("Error: bad return code %s\n", resp.Status)
		os.Exit(-1)
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2017 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"context"
	"encoding/binary"
	"fmt"
	"math"
	"os"
	"time"

	v3 "go.etcd.io/etcd/clientv3"
	"go.etcd.io/etcd/pkg/report"

	"github.com/spf13/cobra"
	"golang.org/x/time/rate"
	"gopkg.in/cheggaaa/pb.v1"
)

// txnPutCmd represents the txnPut command
var txnPutCmd = &cobra.Command{
	Use:   "txn-put",
	Short: "Benchmark txn-put",

	Run: txnPutFunc,
}

var (
	txnPutTotal     int
	txnPutRate      int
	txnPutOpsPerTxn int
)

func init() {
	RootCmd.AddCommand(txnPutCmd)
	txnPutCmd.Flags().IntVar(&keySize, "key-size", 8, "Key size of txn put")
	txnPutCmd.Flags().IntVar(&valSize, "val-size", 8, "Value size of txn put")
	txnPutCmd.Flags().IntVar(&txnPutOpsPerTxn, "txn-ops", 1, "Number of puts per txn")
	txnPutCmd.Flags().IntVar(&txnPutRate, "rate", 0, "Maximum txns per second (0 is no limit)")

	txnPutCmd.Flags().IntVar(&txnPutTotal, "total", 10000, "Total number of txn requests")
	txnPutCmd.Flags().IntVar(&keySpaceSize, "key-space-size", 1, "Maximum possible keys")
}

func txnPutFunc(cmd *cobra.Command, args []string) {
	if keySpaceSize <= 0 {
		fmt.Fprintf(os.Stderr, "expected positive --key-space-size, got (%v)", keySpaceSize)
		os.Exit(1)
	}

	requests := make(chan []v3.Op, totalClients)
	if txnPutRate == 0 {
		txnPutRate = math.MaxInt32
	}
	limit := rate.NewLimiter(rate.Limit(txnPutRate), 1)
	clients := mustCreateClients(totalClients, totalConns)
	k, v := make([]byte, keySize), string(mustRandBytes(valSize))

	bar = pb.New(txnPutTotal)
	bar.Format("Bom !")
	bar.Start()

	r := newReport()
	for i := range clients {
		wg.Add(1)
		go func(c *v3.Client) {
			defer wg.Done()
			for ops := range requests {
				limit.Wait(context.Background())
				st := time.Now()
				_, err := c.Txn(context.TODO()).Then(ops...).Commit()
				r.Results() <- report.Result{Err: err, Start: st, End: time.Now()}
				bar.Increment()
			}
		}(clients[i])
	}

	go func() {
		for i := 0; i < txnPutTotal; i++ {
			ops := make([]v3.Op, txnPutOpsPerTxn)
			for j := 0; j < txnPutOpsPerTxn; j++ {
				binary.PutVarint(k, int64(((i*txnPutOpsPerTxn)+j)%keySpaceSize))
				ops[j] = v3.OpPut(string(k), v)
			}
			requests <- ops
		}
		close(requests)
	}()

	rc := r.Run()
	wg.Wait()
	close(r.Results())
	bar.Finish()
	fmt.Println(<-rc)
}
<commit_msg>tools/benchmark: exit when txn-ops is larger than key-space-size to avoid key duplicated error. (#11499)<commit_after>// Copyright 2017 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"context"
	"encoding/binary"
	"fmt"
	"math"
	"os"
	"time"

	v3 "go.etcd.io/etcd/clientv3"
	"go.etcd.io/etcd/pkg/report"

	"github.com/spf13/cobra"
	"golang.org/x/time/rate"
	"gopkg.in/cheggaaa/pb.v1"
)

// txnPutCmd represents the txnPut command
var txnPutCmd = &cobra.Command{
	Use:   "txn-put",
	Short: "Benchmark txn-put",

	Run: txnPutFunc,
}

var (
	txnPutTotal     int
	txnPutRate      int
	txnPutOpsPerTxn int
)

func init() {
	RootCmd.AddCommand(txnPutCmd)
	txnPutCmd.Flags().IntVar(&keySize, "key-size", 8, "Key size of txn put")
	txnPutCmd.Flags().IntVar(&valSize, "val-size", 8, "Value size of txn put")
	txnPutCmd.Flags().IntVar(&txnPutOpsPerTxn, "txn-ops", 1, "Number of puts per txn")
	txnPutCmd.Flags().IntVar(&txnPutRate, "rate", 0, "Maximum txns per second (0 is no limit)")

	txnPutCmd.Flags().IntVar(&txnPutTotal, "total", 10000, "Total number of txn requests")
	txnPutCmd.Flags().IntVar(&keySpaceSize, "key-space-size", 1, "Maximum possible keys")
}

func txnPutFunc(cmd *cobra.Command, args []string) {
	if keySpaceSize <= 0 {
		fmt.Fprintf(os.Stderr, "expected positive --key-space-size, got (%v)", keySpaceSize)
		os.Exit(1)
	}

	if txnPutOpsPerTxn > keySpaceSize {
		fmt.Fprintf(os.Stderr, "expected --txn-ops no larger than --key-space-size, "+
			"got txn-ops(%v) key-space-size(%v)\n", txnPutOpsPerTxn, keySpaceSize)
		os.Exit(1)
	}

	requests := make(chan []v3.Op, totalClients)
	if txnPutRate == 0 {
		txnPutRate = math.MaxInt32
	}
	limit := rate.NewLimiter(rate.Limit(txnPutRate), 1)
	clients := mustCreateClients(totalClients, totalConns)
	k, v := make([]byte, keySize), string(mustRandBytes(valSize))

	bar = pb.New(txnPutTotal)
	bar.Format("Bom !")
	bar.Start()

	r := newReport()
	for i := range clients {
		wg.Add(1)
		go func(c *v3.Client) {
			defer wg.Done()
			for ops := range requests {
				limit.Wait(context.Background())
				st := time.Now()
				_, err := c.Txn(context.TODO()).Then(ops...).Commit()
				r.Results() <- report.Result{Err: err, Start: st, End: time.Now()}
				bar.Increment()
			}
		}(clients[i])
	}

	go func() {
		for i := 0; i < txnPutTotal; i++ {
			ops := make([]v3.Op, txnPutOpsPerTxn)
			for j := 0; j < txnPutOpsPerTxn; j++ {
				binary.PutVarint(k, int64(((i*txnPutOpsPerTxn)+j)%keySpaceSize))
				ops[j] = v3.OpPut(string(k), v)
			}
			requests <- ops
		}
		close(requests)
	}()

	rc := r.Run()
	wg.Wait()
	close(r.Results())
	bar.Finish()
	fmt.Println(<-rc)
}
<|endoftext|>


<commit_before>package paths

import (
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path"
)

// GetPaths gets config file and data directory paths, creating directories and
// the config file if needed.
func GetPaths() (configFile string, dataDir string) {
	usr, err := user.Current()
	if err != nil {
		log.Fatalf("Unable to get the user directory: %v", err)
	}

	xdgConfig := os.Getenv("XDG_CONFIG_HOME")
	xdgData := os.Getenv("XDG_DATA_HOME")
	if xdgData == "" {
		dataDir = path.Join(usr.HomeDir, ".local/share/", "irchuu/")
	} else {
		dataDir = path.Join(xdgData, "irchuu/")
	}
	if xdgConfig == "" {
		configFile = path.Join(usr.HomeDir, ".config/", "irchuu.conf")
	} else {
		configFile = path.Join(xdgConfig, "irchuu.conf")
	}

	CreateDirOrExit(dataDir, os.FileMode(0700))
	CreateDirOrExit(path.Dir(configFile), os.FileMode(0700))
	_, err = os.Stat(configFile)
	if err != nil {
		err = PopulateConfig(configFile)
	}
	if err != nil {
		log.Fatalf("Failed to populate config: %v\n", err)
	}
	return
}

// CreateDirOrExit creates the directory if it doesn't exist, exits if failed.
func CreateDirOrExit(dir string, mode os.FileMode) {
	_, err := os.Stat(dir)
	if err != nil {
		err = os.MkdirAll(dir, mode)
		if err != nil {
			log.Fatalf("Failed to create directory: %v\n", dir)
		} else {
			log.Printf("Created directory: %v\n", dir)
		}
	}
}

// PopulateConfig copies the sample config to <path>.
func PopulateConfig(file string) error {
	config := `# IRChuu configuration file. See https://github.com/26000/irchuu for help.
[telegram]
token = myToken
group = 7654321

[irc]
server = irc.rizon.net
port = 6777
channel = #irchuu
password =
`
	return ioutil.WriteFile(file, []byte(config), os.FileMode(0600))
}
<commit_msg>Restructure paths<commit_after>package paths

import (
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path"
)

// GetPaths gets config file and data directory paths.
func GetPaths() (configFile string, dataDir string) {
	usr, err := user.Current()
	if err != nil {
		log.Fatalf("Unable to get the user directory: %v", err)
	}

	xdgConfig := os.Getenv("XDG_CONFIG_HOME")
	xdgData := os.Getenv("XDG_DATA_HOME")
	if xdgData == "" {
		dataDir = path.Join(usr.HomeDir, ".local/share/", "irchuu/")
	} else {
		dataDir = path.Join(xdgData, "irchuu/")
	}
	if xdgConfig == "" {
		configFile = path.Join(usr.HomeDir, ".config/", "irchuu.conf")
	} else {
		configFile = path.Join(xdgConfig, "irchuu.conf")
	}
	return
}

// MakePaths creates the configuration file and directories if needed.
func MakePaths(configFile string, dataDir string) error {
	err := CreateDir(dataDir, os.FileMode(0700))
	if err != nil {
		log.Printf("Failed to create directory: %v\n", dataDir)
		return err
	}
	err = CreateDir(path.Dir(configFile), os.FileMode(0700))
	if err != nil {
		log.Printf("Failed to create directory: %v\n", path.Dir(configFile))
		return err
	}

	_, err = os.Stat(configFile)
	if err != nil {
		err = PopulateConfig(configFile)
		if err == nil {
			log.Printf("New configuration file was populated. Edit %v and run `irchuu` again!\n", configFile)
			defer os.Exit(0)
		} else {
			log.Fatalf("Failed to populate config: %v\n", err)
		}
	}
	return nil
}

// CreateDir creates the directory if it doesn't exist.
func CreateDir(dir string, mode os.FileMode) error {
	_, err := os.Stat(dir)
	if err != nil {
		err = os.MkdirAll(dir, mode)
		if err != nil {
			return err
		} else {
			log.Printf("Created directory: %v\n", dir)
		}
	}
	return nil
}

// PopulateConfig copies the sample config to <path>.
func PopulateConfig(file string) error {
	config := `# IRChuu configuration file. See https://github.com/26000/irchuu for help.
[telegram]
token = myToken
group = 7654321

[irc]
server = irc.rizon.net
port = 6777
channel = irchuu # without '#'!
password = # leave blank if not set
`
	return ioutil.WriteFile(file, []byte(config), os.FileMode(0600))
}
<|endoftext|>


<commit_before><commit_msg>manager: Use the correct user's auth token for SystemA calls<commit_after><|endoftext|>


<commit_before><commit_msg>libdokan: Fix after rebase<commit_after><|endoftext|>


<commit_before>package contract

type Monitor struct {
	scanner  Scanner
	watcher  Watcher
	executor Executor
	server   Server
	sleep    func()
	// ScanForever() // infinite for loop, calls Scan() between time.Sleep() (when no tests were run)
}

func (self *Monitor) Scan() {
	root := self.watcher.Root()

	if self.scanner.Scan(root) {
		watched := self.watcher.WatchedFolders()
		output := self.executor.ExecuteTests(watched)
		self.server.ReceiveUpdate(output)
	} else {
		self.sleep()
	}
}

func NewMonitor(scanner Scanner, watcher Watcher, executor Executor, server Server, sleep func()) *Monitor {
	self := &Monitor{}
	self.scanner = scanner
	self.watcher = watcher
	self.executor = executor
	self.server = server
	self.sleep = sleep
	return self
}
<commit_msg>Created ScanForever method on Monitor.<commit_after>package contract

type Monitor struct {
	scanner  Scanner
	watcher  Watcher
	executor Executor
	server   Server
	sleep    func()
}

func (self *Monitor) ScanForever() {
	for {
		self.Scan()
	}
}

func (self *Monitor) Scan() {
	root := self.watcher.Root()

	if self.scanner.Scan(root) {
		self.executeTests()
	} else {
		self.sleep()
	}
}

func (self *Monitor) executeTests() {
	watched := self.watcher.WatchedFolders()
	output := self.executor.ExecuteTests(watched)
	self.server.ReceiveUpdate(output)
}

func NewMonitor(scanner Scanner, watcher Watcher, executor Executor, server Server, sleep func()) *Monitor {
	self := &Monitor{}
	self.scanner = scanner
	self.watcher = watcher
	self.executor = executor
	self.server = server
	self.sleep = sleep
	return self
}
<|endoftext|>


<commit_before>package s3

import (
	"io"
	"strings"
	"time"

	"go.uber.org/zap"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/cloudfoundry-incubator/bits-service"
	"github.com/cloudfoundry-incubator/bits-service/blobstores/s3/signer"
	"github.com/cloudfoundry-incubator/bits-service/blobstores/validate"
	"github.com/cloudfoundry-incubator/bits-service/config"
	"github.com/cloudfoundry-incubator/bits-service/logger"
	log "github.com/cloudfoundry-incubator/bits-service/logger"
	"github.com/pkg/errors"
)

type Blobstore struct {
	s3Client *s3.S3
	bucket   string
	signer   S3Signer
}

type S3Signer interface {
	Sign(req *request.Request, bucket, path string, expires time.Time) (string, error)
}

func NewBlobstore(config config.S3BlobstoreConfig) *Blobstore {
	return NewBlobstoreWithLogger(config, log.Log)
}

func NewBlobstoreWithLogger(config config.S3BlobstoreConfig, logger *zap.SugaredLogger) *Blobstore {
	validate.NotEmpty(config.AccessKeyID)
	validate.NotEmpty(config.Bucket)
	validate.NotEmpty(config.SecretAccessKey)

	var s3Signer S3Signer = &signer.Default{}
	if config.Host == "storage.googleapis.com" {
		s3Signer = &signer.Google{
			AccessID:        config.AccessKeyID,
			SecretAccessKey: config.SecretAccessKey,
		}
	}

	// Make SDK happy. The AWS Client requires a region, although it never uses that region,
	// because S3 is region independent. (It's using that region when used with other AWS services.)
	if config.Region == "" {
		config.Region = "us-east-1"
		log.Log.Infow("No AWS region specified for blobstore. Using a default value.", "bucket", config.Bucket, "default-region", config.Region)
	}

	return &Blobstore{
		s3Client: newS3Client(config.Region, config.AccessKeyID, config.SecretAccessKey, config.Host, logger, config.S3DebugLogLevel),
		bucket:   config.Bucket,
		signer:   s3Signer,
	}
}

func (blobstore *Blobstore) Exists(path string) (bool, error) {
	_, e := blobstore.s3Client.HeadObject(&s3.HeadObjectInput{
		Bucket: &blobstore.bucket,
		Key:    &path,
	})
	if e != nil {
		if isS3NotFoundError(e) {
			return false, nil
		}
		return false, errors.Wrapf(e, "Failed to check for %v/%v", blobstore.bucket, path)
	}
	return true, nil
}

func (blobstore *Blobstore) HeadOrRedirectAsGet(path string) (redirectLocation string, err error) {
	request, _ := blobstore.s3Client.GetObjectRequest(&s3.GetObjectInput{
		Bucket: &blobstore.bucket,
		Key:    &path,
	})
	return blobstore.signer.Sign(request, blobstore.bucket, path, time.Now().Add(time.Hour))
}

func (blobstore *Blobstore) Get(path string) (body io.ReadCloser, err error) {
	logger.Log.Debugw("Get from S3", "bucket", blobstore.bucket, "path", path)
	output, e := blobstore.s3Client.GetObject(&s3.GetObjectInput{
		Bucket: &blobstore.bucket,
		Key:    &path,
	})
	if e != nil {
		if isS3NotFoundError(e) {
			return nil, bitsgo.NewNotFoundError()
		}
		return nil, errors.Wrapf(e, "Path %v", path)
	}
	return output.Body, nil
}

func (blobstore *Blobstore) GetOrRedirect(path string) (body io.ReadCloser, redirectLocation string, err error) {
	request, _ := blobstore.s3Client.GetObjectRequest(&s3.GetObjectInput{
		Bucket: &blobstore.bucket,
		Key:    &path,
	})
	signedUrl, e := blobstore.signer.Sign(request, blobstore.bucket, path, time.Now().Add(time.Hour))
	return nil, signedUrl, e
}

func (blobstore *Blobstore) Put(path string, src io.ReadSeeker) error {
	logger.Log.Debugw("Put to S3", "bucket", blobstore.bucket, "path", path)
	_, e := blobstore.s3Client.PutObject(&s3.PutObjectInput{
		Bucket: &blobstore.bucket,
		Key:    &path,
		Body:   src,
	})
	if e != nil {
		return errors.Wrapf(e, "Path %v", path)
	}
	return nil
}

func (blobstore *Blobstore) Copy(src, dest string) error {
	// see https://forums.aws.amazon.com/thread.jspa?threadID=55746:
	src = strings.Replace(src, "+", "%2B", -1)

	logger.Log.Debugw("Copy in S3", "bucket", blobstore.bucket, "src", src, "dest", dest)
	_, e := blobstore.s3Client.CopyObject(&s3.CopyObjectInput{
		Key:        &dest,
		CopySource: aws.String(blobstore.bucket + "/" + src),
		Bucket:     &blobstore.bucket,
	})
	if e != nil {
		if isS3NotFoundError(e) {
			return bitsgo.NewNotFoundError()
		}
		return errors.Wrapf(e, "Error while trying to copy src %v to dest %v in bucket %v", src, dest, blobstore.bucket)
	}
	return nil
}

func (blobstore *Blobstore) Delete(path string) error {
	_, e := blobstore.s3Client.DeleteObject(&s3.DeleteObjectInput{
		Bucket: &blobstore.bucket,
		Key:    &path,
	})
	if e != nil {
		if isS3NotFoundError(e) {
			return bitsgo.NewNotFoundError()
		}
		return errors.Wrapf(e, "Path %v", path)
	}
	return nil
}

func (blobstore *Blobstore) DeleteDir(prefix string) error {
	deletionErrs := []error{}
	e := blobstore.s3Client.ListObjectsPages(
		&s3.ListObjectsInput{
			Bucket: &blobstore.bucket,
			Prefix: &prefix,
		},
		func(p *s3.ListObjectsOutput, lastPage bool) (shouldContinue bool) {
			for _, object := range p.Contents {
				e := blobstore.Delete(*object.Key)
				if e != nil {
					if _, isNotFoundError := e.(*bitsgo.NotFoundError); !isNotFoundError {
						deletionErrs = append(deletionErrs, e)
					}
				}
			}
			return true
		})
	if e != nil {
		return errors.Wrapf(e, "Prefix %v, errors from deleting: %v", prefix, deletionErrs)
	}
	if len(deletionErrs) != 0 {
		return errors.Errorf("Prefix %v, errors from deleting: %v", prefix, deletionErrs)
	}
	return nil
}

func (signer *Blobstore) Sign(resource string, method string, expirationTime time.Time) (signedURL string) {
	var request *request.Request
	switch strings.ToLower(method) {
	case "put":
		request, _ = signer.s3Client.PutObjectRequest(&s3.PutObjectInput{
			Bucket: aws.String(signer.bucket),
			Key:    aws.String(resource),
		})
	case "get":
		request, _ = signer.s3Client.GetObjectRequest(&s3.GetObjectInput{
			Bucket: aws.String(signer.bucket),
			Key:    aws.String(resource),
		})
	default:
		panic("The only supported methods are 'put' and 'get'. But got '" + method + "'")
	}
	// TODO use clock
	signedURL, e := signer.signer.Sign(request, signer.bucket, resource, expirationTime)
	if e != nil {
		panic(e)
	}
	logger.Log.Debugw("Signed URL", "verb", method, "signed-url", signedURL)
	return
}
<commit_msg>Improve error messages for S3 Blobstore NotFound<commit_after>package s3

import (
	"io"
	"strings"
	"time"

	"go.uber.org/zap"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/cloudfoundry-incubator/bits-service"
	"github.com/cloudfoundry-incubator/bits-service/blobstores/s3/signer"
	"github.com/cloudfoundry-incubator/bits-service/blobstores/validate"
	"github.com/cloudfoundry-incubator/bits-service/config"
	"github.com/cloudfoundry-incubator/bits-service/logger"
	log "github.com/cloudfoundry-incubator/bits-service/logger"
	"github.com/pkg/errors"
)

type Blobstore struct {
	s3Client *s3.S3
	bucket   string
	signer   S3Signer
}

type S3Signer interface {
	Sign(req *request.Request, bucket, path string, expires time.Time) (string, error)
}

func NewBlobstore(config config.S3BlobstoreConfig) *Blobstore {
	return NewBlobstoreWithLogger(config, log.Log)
}

func NewBlobstoreWithLogger(config config.S3BlobstoreConfig, logger *zap.SugaredLogger) *Blobstore {
	validate.NotEmpty(config.AccessKeyID)
	validate.NotEmpty(config.Bucket)
	validate.NotEmpty(config.SecretAccessKey)

	var s3Signer S3Signer = &signer.Default{}
	if config.Host == "storage.googleapis.com" {
		s3Signer = &signer.Google{
			AccessID:        config.AccessKeyID,
			SecretAccessKey: config.SecretAccessKey,
		}
	}

	// Make SDK happy. The AWS Client requires a region, although it never uses that region,
	// because S3 is region independent. (It's using that region when used with other AWS services.)
	if config.Region == "" {
		config.Region = "us-east-1"
		log.Log.Infow("No AWS region specified for blobstore. Using a default value.", "bucket", config.Bucket, "default-region", config.Region)
	}

	return &Blobstore{
		s3Client: newS3Client(config.Region, config.AccessKeyID, config.SecretAccessKey, config.Host, logger, config.S3DebugLogLevel),
		bucket:   config.Bucket,
		signer:   s3Signer,
	}
}

func (blobstore *Blobstore) Exists(path string) (bool, error) {
	_, e := blobstore.s3Client.HeadObject(&s3.HeadObjectInput{
		Bucket: &blobstore.bucket,
		Key:    &path,
	})
	if e != nil {
		if isS3NotFoundError(e) {
			return false, nil
		}
		return false, errors.Wrapf(e, "Failed to check for %v/%v", blobstore.bucket, path)
	}
	return true, nil
}

func (blobstore *Blobstore) HeadOrRedirectAsGet(path string) (redirectLocation string, err error) {
	request, _ := blobstore.s3Client.GetObjectRequest(&s3.GetObjectInput{
		Bucket: &blobstore.bucket,
		Key:    &path,
	})
	return blobstore.signer.Sign(request, blobstore.bucket, path, time.Now().Add(time.Hour))
}

func (blobstore *Blobstore) Get(path string) (body io.ReadCloser, err error) {
	logger.Log.Debugw("Get from S3", "bucket", blobstore.bucket, "path", path)
	output, e := blobstore.s3Client.GetObject(&s3.GetObjectInput{
		Bucket: &blobstore.bucket,
		Key:    &path,
	})
	if e != nil {
		if isS3NotFoundError(e) {
			return nil, bitsgo.NewNotFoundErrorWithKey(path)
		}
		return nil, errors.Wrapf(e, "Path %v", path)
	}
	return output.Body, nil
}

func (blobstore *Blobstore) GetOrRedirect(path string) (body io.ReadCloser, redirectLocation string, err error) {
	request, _ := blobstore.s3Client.GetObjectRequest(&s3.GetObjectInput{
		Bucket: &blobstore.bucket,
		Key:    &path,
	})
	signedUrl, e := blobstore.signer.Sign(request, blobstore.bucket, path, time.Now().Add(time.Hour))
	return nil, signedUrl, e
}

func (blobstore *Blobstore) Put(path string, src io.ReadSeeker) error {
	logger.Log.Debugw("Put to S3", "bucket", blobstore.bucket, "path", path)
	_, e := blobstore.s3Client.PutObject(&s3.PutObjectInput{
		Bucket: &blobstore.bucket,
		Key:    &path,
		Body:   src,
	})
	if e != nil {
		return errors.Wrapf(e, "Path %v", path)
	}
	return nil
}

func (blobstore *Blobstore) Copy(src, dest string) error {
	// see https://forums.aws.amazon.com/thread.jspa?threadID=55746:
	src = strings.Replace(src, "+", "%2B", -1)

	logger.Log.Debugw("Copy in S3", "bucket", blobstore.bucket, "src", src, "dest", dest)
	_, e := blobstore.s3Client.CopyObject(&s3.CopyObjectInput{
		Key:        &dest,
		CopySource: aws.String(blobstore.bucket + "/" + src),
		Bucket:     &blobstore.bucket,
	})
	if e != nil {
		if isS3NotFoundError(e) {
			return bitsgo.NewNotFoundErrorWithKey(src)
		}
		return errors.Wrapf(e, "Error while trying to copy src %v to dest %v in bucket %v", src, dest, blobstore.bucket)
	}
	return nil
}

func (blobstore *Blobstore) Delete(path string) error {
	_, e := blobstore.s3Client.DeleteObject(&s3.DeleteObjectInput{
		Bucket: &blobstore.bucket,
		Key:    &path,
	})
	if e != nil {
		if isS3NotFoundError(e) {
			return bitsgo.NewNotFoundErrorWithKey(path)
		}
		return errors.Wrapf(e, "Path %v", path)
	}
	return nil
}

func (blobstore *Blobstore) DeleteDir(prefix string) error {
	deletionErrs := []error{}
	e := blobstore.s3Client.ListObjectsPages(
		&s3.ListObjectsInput{
			Bucket: &blobstore.bucket,
			Prefix: &prefix,
		},
		func(p *s3.ListObjectsOutput, lastPage bool) (shouldContinue bool) {
			for _, object := range p.Contents {
				e := blobstore.Delete(*object.Key)
				if e != nil {
					if _, isNotFoundError := e.(*bitsgo.NotFoundError); !isNotFoundError {
						deletionErrs = append(deletionErrs, e)
					}
				}
			}
			return true
		})
	if e != nil {
		return errors.Wrapf(e, "Prefix %v, errors from deleting: %v", prefix, deletionErrs)
	}
	if len(deletionErrs) != 0 {
		return errors.Errorf("Prefix %v, errors from deleting: %v", prefix, deletionErrs)
	}
	return nil
}

func (signer *Blobstore) Sign(resource string, method string, expirationTime time.Time) (signedURL string) {
	var request *request.Request
	switch strings.ToLower(method) {
	case "put":
		request, _ = signer.s3Client.PutObjectRequest(&s3.PutObjectInput{
			Bucket: aws.String(signer.bucket),
			Key:    aws.String(resource),
		})
	case "get":
		request, _ = signer.s3Client.GetObjectRequest(&s3.GetObjectInput{
			Bucket: aws.String(signer.bucket),
			Key:    aws.String(resource),
		})
	default:
		panic("The only supported methods are 'put' and 'get'. But got '" + method + "'")
	}
	// TODO use clock
	signedURL, e := signer.signer.Sign(request, signer.bucket, resource, expirationTime)
	if e != nil {
		panic(e)
	}
	logger.Log.Debugw("Signed URL", "verb", method, "signed-url", signedURL)
	return
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package parse builds parse trees for configurations as defined by conf.
// Clients should use that package to construct configurations rather than this
// one, which provides shared internal data structures not intended for general
// use.
package parse

import (
	"fmt"
	"runtime"
	"strconv"
	"strings"
)

// Tree is the representation of a single parsed configuration.
type Tree struct {
	Name string    // name of the template represented by the tree.
	Root *ListNode // top-level root of the tree.
	text string    // text parsed to create the template (or its parent)
	// Parsing only; cleared after parse.
	lex       *lexer
	token     [1]item // one-token lookahead for parser.
	peekCount int
}

// Parse returns a Tree, created by parsing the configuration described in the
// argument string. If an error is encountered, parsing stops and an empty Tree
// is returned with the error.
func Parse(name, text string) (t *Tree, err error) {
	t = New(name)
	t.text = text
	err = t.Parse(text)
	return
}

// next returns the next token.
func (t *Tree) next() item {
	if t.peekCount > 0 {
		t.peekCount--
	} else {
		t.token[0] = t.lex.nextItem()
	}
	return t.token[t.peekCount]
}

// backup backs the input stream up one token.
func (t *Tree) backup() {
	t.peekCount++
}

// peek returns but does not consume the next token.
func (t *Tree) peek() item {
	if t.peekCount > 0 {
		return t.token[t.peekCount-1]
	}
	t.peekCount = 1
	t.token[0] = t.lex.nextItem()
	return t.token[0]
}

// Parsing.

// New allocates a new parse tree with the given name.
func New(name string) *Tree {
	return &Tree{
		Name: name,
	}
}

// ErrorContext returns a textual representation of the location of the node in the input text.
func (t *Tree) ErrorContext(n Node) (location, context string) {
	pos := int(n.Position())
	text := t.text[:pos]
	byteNum := strings.LastIndex(text, "\n")
	if byteNum == -1 {
		byteNum = pos // On first line.
	} else {
		byteNum++ // After the newline.
		byteNum = pos - byteNum
	}
	lineNum := 1 + strings.Count(text, "\n")
	context = n.String()
	if len(context) > 20 {
		context = fmt.Sprintf("%.20s...", context)
	}
	return fmt.Sprintf("%s:%d:%d", t.Name, lineNum, byteNum), context
}

// errorf formats the error and terminates processing.
func (t *Tree) errorf(format string, args ...interface{}) {
	t.Root = nil
	format = fmt.Sprintf("parse: %s:%d: %s", t.Name, t.lex.lineNumber(), format)
	panic(fmt.Errorf(format, args...))
}

// error terminates processing.
func (t *Tree) error(err error) {
	t.errorf("%s", err)
}

// expect consumes the next token and guarantees it has the required type.
func (t *Tree) expect(expected itemType, context string) item {
	token := t.next()
	if token.typ != expected {
		t.unexpected(token, context)
	}
	return token
}

// expectOneOf consumes the next token and guarantees it has one of the required types.
func (t *Tree) expectOneOf(expected1, expected2 itemType, context string) item {
	token := t.next()
	if token.typ != expected1 && token.typ != expected2 {
		t.unexpected(token, context)
	}
	return token
}

// unexpected complains about the token and terminates processing.
func (t *Tree) unexpected(token item, context string) {
	t.errorf("unexpected %s in %s", token, context)
}

// recover is the handler that turns panics into returns from the top level of Parse.
func (t *Tree) recover(errp *error) {
	e := recover()
	if e != nil {
		if _, ok := e.(runtime.Error); ok {
			panic(e)
		}
		if t != nil {
			t.stopParse()
		}
		*errp = e.(error)
	}
	return
}

// startParse initializes the parser, using the lexer.
func (t *Tree) startParse(lex *lexer) {
	t.Root = nil
	t.lex = lex
}

// stopParse terminates parsing.
func (t *Tree) stopParse() {
	t.lex = nil
}

// Parse parses the template definition string to construct a representation of
// the template for execution. If either action delimiter string is empty, the
// default ("{{" or "}}") is used. Embedded template definitions are added to
// the treeSet map.
func (t *Tree) Parse(text string) (err error) {
	defer t.recover(&err)
	t.startParse(lex(t.Name, text))
	t.text = text
	t.parse()
	t.stopParse()
	return nil
}

// parse is the top-level parser for a conf.
// It runs to EOF.
func (t *Tree) parse() {
	t.Root = newList(t.peek().pos)
	var n Node
	for {
		switch token := t.peek(); token.typ {
		case itemIdentifier:
			n = t.parsePair()
		case itemLeftDelim:
			n = t.parseSection()
		case itemEOF:
			return
		default:
			t.unexpected(token, "input")
		}
		t.Root.append(n)
	}
}

func (t *Tree) parsePair() *PairNode {
	const context = "key=value declaration"
	token := t.expect(itemIdentifier, context)
	p := newPair(token.pos)
	p.Key = newString(token.pos, token.val, token.val)
	t.expect(itemEqual, context)
	token = t.expectOneOf(itemString, itemRawString, context)
	switch token.typ {
	case itemString:
		p.Val = newString(token.pos, token.val, token.val)
	case itemRawString:
		s, err := strconv.Unquote(token.val)
		if err != nil {
			t.error(err)
		}
		p.Val = newString(token.pos, token.val, s)
	default:
		t.unexpected(token, context)
	}
	return p
}

func (t *Tree) parseSection() *SectionNode {
	const context = "section declaration"
	s := newSection(t.expect(itemLeftDelim, context).pos)
	token := t.expect(itemIdentifier, context)
	s.Name = newString(token.pos, token.val, token.val)
	t.expect(itemRightDelim, context)
	for {
		switch token := t.peek(); token.typ {
		case itemIdentifier:
			s.append(t.parsePair())
		case itemLeftDelim, itemEOF:
			return s
		default:
			t.unexpected(token, context)
		}
	}
}
<commit_msg>Trim node newlines on error<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package parse builds parse trees for configurations as defined by conf.
// Clients should use that package to construct configurations rather than this
// one, which provides shared internal data structures not intended for general
// use.
package parse

import (
	"fmt"
	"runtime"
	"strconv"
	"strings"
)

// Tree is the representation of a single parsed configuration.
type Tree struct {
	Name string    // name of the template represented by the tree.
	Root *ListNode // top-level root of the tree.
	text string    // text parsed to create the template (or its parent)
	// Parsing only; cleared after parse.
	lex       *lexer
	token     [1]item // one-token lookahead for parser.
	peekCount int
}

// Parse returns a Tree, created by parsing the configuration described in the
// argument string. If an error is encountered, parsing stops and an empty Tree
// is returned with the error.
func Parse(name, text string) (t *Tree, err error) {
	t = New(name)
	t.text = text
	err = t.Parse(text)
	return
}

// next returns the next token.
func (t *Tree) next() item {
	if t.peekCount > 0 {
		t.peekCount--
	} else {
		t.token[0] = t.lex.nextItem()
	}
	return t.token[t.peekCount]
}

// backup backs the input stream up one token.
func (t *Tree) backup() {
	t.peekCount++
}

// peek returns but does not consume the next token.
func (t *Tree) peek() item {
	if t.peekCount > 0 {
		return t.token[t.peekCount-1]
	}
	t.peekCount = 1
	t.token[0] = t.lex.nextItem()
	return t.token[0]
}

// Parsing.

// New allocates a new parse tree with the given name.
func New(name string) *Tree {
	return &Tree{
		Name: name,
	}
}

// ErrorContext returns a textual representation of the location of the node in the input text.
func (t *Tree) ErrorContext(n Node) (location, context string) {
	pos := int(n.Position())
	text := t.text[:pos]
	byteNum := strings.LastIndex(text, "\n")
	if byteNum == -1 {
		byteNum = pos // On first line.
	} else {
		byteNum++ // After the newline.
		byteNum = pos - byteNum
	}
	lineNum := 1 + strings.Count(text, "\n")
	context = n.String()
	if len(context) > 20 {
		context = fmt.Sprintf("%.20s...", context)
	}
	context = strings.TrimSpace(context)
	return fmt.Sprintf("%s:%d:%d", t.Name, lineNum, byteNum), context
}

// errorf formats the error and terminates processing.
func (t *Tree) errorf(format string, args ...interface{}) {
	t.Root = nil
	format = fmt.Sprintf("parse: %s:%d: %s", t.Name, t.lex.lineNumber(), format)
	panic(fmt.Errorf(format, args...))
}

// error terminates processing.
func (t *Tree) error(err error) {
	t.errorf("%s", err)
}

// expect consumes the next token and guarantees it has the required type.
func (t *Tree) expect(expected itemType, context string) item {
	token := t.next()
	if token.typ != expected {
		t.unexpected(token, context)
	}
	return token
}

// expectOneOf consumes the next token and guarantees it has one of the required types.
func (t *Tree) expectOneOf(expected1, expected2 itemType, context string) item {
	token := t.next()
	if token.typ != expected1 && token.typ != expected2 {
		t.unexpected(token, context)
	}
	return token
}

// unexpected complains about the token and terminates processing.
func (t *Tree) unexpected(token item, context string) {
	t.errorf("unexpected %s in %s", token, context)
}

// recover is the handler that turns panics into returns from the top level of Parse.
func (t *Tree) recover(errp *error) {
	e := recover()
	if e != nil {
		if _, ok := e.(runtime.Error); ok {
			panic(e)
		}
		if t != nil {
			t.stopParse()
		}
		*errp = e.(error)
	}
	return
}

// startParse initializes the parser, using the lexer.
func (t *Tree) startParse(lex *lexer) {
	t.Root = nil
	t.lex = lex
}

// stopParse terminates parsing.
func (t *Tree) stopParse() {
	t.lex = nil
}

// Parse parses the template definition string to construct a representation of
// the template for execution. If either action delimiter string is empty, the
// default ("{{" or "}}") is used. Embedded template definitions are added to
// the treeSet map.
func (t *Tree) Parse(text string) (err error) {
	defer t.recover(&err)
	t.startParse(lex(t.Name, text))
	t.text = text
	t.parse()
	t.stopParse()
	return nil
}

// parse is the top-level parser for a conf.
// It runs to EOF.
func (t *Tree) parse() {
	t.Root = newList(t.peek().pos)
	var n Node
	for {
		switch token := t.peek(); token.typ {
		case itemIdentifier:
			n = t.parsePair()
		case itemLeftDelim:
			n = t.parseSection()
		case itemEOF:
			return
		default:
			t.unexpected(token, "input")
		}
		t.Root.append(n)
	}
}

func (t *Tree) parsePair() *PairNode {
	const context = "key=value declaration"
	token := t.expect(itemIdentifier, context)
	p := newPair(token.pos)
	p.Key = newString(token.pos, token.val, token.val)
	t.expect(itemEqual, context)
	token = t.expectOneOf(itemString, itemRawString, context)
	switch token.typ {
	case itemString:
		p.Val = newString(token.pos, token.val, token.val)
	case itemRawString:
		s, err := strconv.Unquote(token.val)
		if err != nil {
			t.error(err)
		}
		p.Val = newString(token.pos, token.val, s)
	default:
		t.unexpected(token, context)
	}
	return p
}

func (t *Tree) parseSection() *SectionNode {
	const context = "section declaration"
	s := newSection(t.expect(itemLeftDelim, context).pos)
	token := t.expect(itemIdentifier, context)
	s.Name = newString(token.pos, token.val, token.val)
	t.expect(itemRightDelim, context)
	for {
		switch token := t.peek(); token.typ {
		case itemIdentifier:
			s.append(t.parsePair())
		case itemLeftDelim, itemEOF:
			return s
		default:
			t.unexpected(token, context)
		}
	}
}
<|endoftext|>


<commit_before>package models

type FieldTyp string

const (
	PRE_UPPER             FieldTyp = "pre_upper"
	PRE_LOWER             FieldTyp = "pre_lower"
	PRE_NUMBER            FieldTyp = "pre_number"
	PRE_UNDERLINE         FieldTyp = "pre_underline"
	PRE_MIDDLELINE        FieldTyp = "pre_middleline"
	PRE_DOUBLE_UNDERLINE  FieldTyp = "pre_double_underline"
	PRE_DOUBLE_MIDDLELINE FieldTyp = "pre_double_middleline"
	PRE_SYMBOL            FieldTyp = "pre_symbol"
	PRE_CHINESE           FieldTyp = "pre_chinese"
)

var fieldList = map[string]map[FieldTyp]string{}

func Register(typ FieldTyp, field, val string) {
	fields, ok := fieldList[field]
	if !ok {
		fields = make(map[FieldTyp]string)
		fieldList[field] = fields
	}

	fields[typ] = val
}

func GetFieldValue(typ FieldTyp, field string) (val string) {
	if fields, ok := fieldList[field]; ok {
		val = fields[typ]
	}
	return
}
<commit_msg>fill field type<commit_after>package models

type FieldType string

const (
	DEFAULT                  FieldType = "default"
	UPPER                    FieldType = "upper"
	LOWER                    FieldType = "lower"
	NUMBER                   FieldType = "number"
	UNDERLINE                FieldType = "underline"
	MIDDLELINE               FieldType = "middleline"
	SYMBOL                   FieldType = "symbol"
	CHINESE                  FieldType = "chinese"
	PRE_UPPER                FieldType = "pre_upper"
	PRE_LOWER                FieldType = "pre_lower"
	PRE_NUMBER               FieldType = "pre_number"
	PRE_UNDERLINE            FieldType = "pre_underline"
	PRE_MIDDLELINE           FieldType = "pre_middleline"
	PRE_DOUBLE_UNDERLINE     FieldType = "pre_double_underline"
	PRE_DOUBLE_MIDDLELINE    FieldType = "pre_double_middleline"
	PRE_YMBOL                FieldType = "inside_symbol"
	PRE_CHINESE              FieldType = "inside_chinese"
	INSIDE_UPPER             FieldType = "inside_upper"
	INSIDE_LOWER             FieldType = "inside_lower"
	INSIDE_NUMBER            FieldType = "inside_number"
	INSIDE_UNDERLINE         FieldType = "inside_underline"
	INSIDE_MIDDLELINE        FieldType = "inside_middleline"
	INSIDE_DOUBLE_UNDERLINE  FieldType = "inside_double_underline"
	INSIDE_DOUBLE_MIDDLELINE FieldType = "inside_double_middleline"
	INSIDE_SYMBOL            FieldType = "inside_symbol"
	INSIDE_CHINESE           FieldType = "inside_chinese"
	SUF_UPPER                FieldType = "suf_upper"
	SUF_LOWER                FieldType = "suf_lower"
	SUF_NUMBER               FieldType = "suf_number"
	SUF_UNDERLINE            FieldType = "suf_underline"
	SUF_MIDDLELINE           FieldType = "suf_middleline"
	SUF_DOUBLE_UNDERLINE     FieldType = "suf_double_underline"
	SUF_DOUBLE_MIDDLELINE    FieldType = "suf_double_middleline"
	SUF_SYMBOL               FieldType = "suf_symbol"
	SUF_CHINESE              FieldType = "suf_chinese"
)

var fieldList = map[string]map[FieldType]string{}

func Register(typ FieldType, field, val string) {
	fields, ok := fieldList[field]
	if !ok {
		fields = make(map[FieldType]string)
		fieldList[field] = fields
	}

	fields[typ] = val
}

func GetFieldValue(typ FieldType, field string) (val string) {
	if fields, ok := fieldList[field]; ok {
		val = fields[typ]
	}
	return
}
<|endoftext|>


<commit_before>package http

import (
	"net/http"

	// TODO: use platform version of the code
	"github.com/influxdata/chronograf"
	"github.com/influxdata/chronograf/dist"
)

const (
	// Dir is prefix of the assets in the bindata
	Dir = "../chronograf/ui/build"
	// Default is the default item to load if 404
	Default = "../chronograf/ui/build/index.html"
	// DebugDir is the prefix of the assets in development mode
	DebugDir = "chronograf/ui/build"
	// DebugDefault is the default item to load if 404
	DebugDefault = "chronograf/ui/build/index.html"
	// DefaultContentType is the content-type to return for the Default file
	DefaultContentType = "text/html; charset=utf-8"
)

// AssetHandler is an http handler for serving chronograf assets.
type AssetHandler struct {
	Develop bool
}

// NewAssetHandler is the constructor an asset handler.
func NewAssetHandler() *AssetHandler {
	return &AssetHandler{
		Develop: true,
	}
}

// ServeHTTP implements the http handler interface for serving assets.
func (h *AssetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var assets chronograf.Assets
	if h.Develop {
		assets = &dist.DebugAssets{
			Dir:     DebugDir,
			Default: DebugDefault,
		}
	} else {
		assets = &dist.BindataAssets{
			Prefix:             Dir,
			Default:            Default,
			DefaultContentType: DefaultContentType,
		}
	}

	assets.Handler().ServeHTTP(w, r)
}
<commit_msg>fix(http): fix chronograf build asset paths<commit_after>package http

import (
	"net/http"

	// TODO: use platform version of the code
	"github.com/influxdata/platform/chronograf"
	"github.com/influxdata/platform/chronograf/dist"
)

const (
	// Dir is prefix of the assets in the bindata
	Dir = "../chronograf/ui/build"
	// Default is the default item to load if 404
	Default = "../chronograf/ui/build/index.html"
	// DebugDir is the prefix of the assets in development mode
	DebugDir = "chronograf/ui/build"
	// DebugDefault is the default item to load if 404
	DebugDefault = "chronograf/ui/build/index.html"
	// DefaultContentType is the content-type to return for the Default file
	DefaultContentType = "text/html; charset=utf-8"
)

// AssetHandler is an http handler for serving chronograf assets.
type AssetHandler struct {
	Develop bool
}

// NewAssetHandler is the constructor an asset handler.
func NewAssetHandler() *AssetHandler {
	return &AssetHandler{
		Develop: true,
	}
}

// ServeHTTP implements the http handler interface for serving assets.
func (h *AssetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var assets chronograf.Assets
	if h.Develop {
		assets = &dist.DebugAssets{
			Dir:     DebugDir,
			Default: DebugDefault,
		}
	} else {
		assets = &dist.BindataAssets{
			Prefix:             Dir,
			Default:            Default,
			DefaultContentType: DefaultContentType,
		}
	}

	assets.Handler().ServeHTTP(w, r)
}
<|endoftext|>