mirror/forgejo
1
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2024-11-28 12:16:17 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Quieter Integration Tests (#6513)

Browse source 
* Rename BaseLogger to WriterLogger to help the creation of other providers

* Don't export ColorBytes and ResetBytes from ColoredValues

* Make integration tests only print logs if they fail

* check can color before coloring

* I always forget about MSSQL

* Oh and use LEVEL in sqlite.ini

* Make the test logger log at info - as it means you see the router

* Remove empty expected changes

* Make the migrations quieter too

* Don't display SQL on error - it can be looked at in the file logs if necessary

* Fix skip when using onGiteaRun
This commit is contained in:
zeripath 2019-04-07 01:25:14 +01:00 committed by Lauris BH
parent 7ed65a98e8
commit 5422f23ed8
23 changed files with 607 additions and 409 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
docs/content/doc/advanced/logging-documentation.en-us.md
View file

@ -357,10 +357,10 @@ attributes should be cached if this is a commonly used log message.
of bytes representing the color.
These functions will not double wrap a `log.ColoredValue`. They will
also set the ResetBytes to the cached resetBytes.
also set the `resetBytes` to the cached `resetBytes`.
Be careful not to change the contents of resetBytes or boldBytes as this
will break rendering of logging elsewhere. You have been warned.
The `colorBytes` and `resetBytes` are not exported to prevent
accidental overwriting of internal values.
## Log Spoofing protection

2
integrations/git_helper_for_declarative_test.go
View file

@ -50,7 +50,7 @@ func createSSHUrl(gitPath string, u *url.URL) *url.URL {
}
func onGiteaRun(t *testing.T, callback func(*testing.T, *url.URL)) {
prepareTestEnv(t)
prepareTestEnv(t, 1)
s := http.Server{
Handler: mac,
}

25
integrations/git_test.go
View file

@ -38,6 +38,7 @@ func testGit(t *testing.T, u *url.URL) {
u.Path = baseAPITestContext.GitPath()
t.Run("HTTP", func(t *testing.T) {
PrintCurrentTest(t)
httpContext := baseAPITestContext
httpContext.Reponame = "repo-tmp-17"
@ -47,6 +48,7 @@ func testGit(t *testing.T, u *url.URL) {
assert.NoError(t, err)
defer os.RemoveAll(dstPath)
t.Run("Standard", func(t *testing.T) {
PrintCurrentTest(t)
ensureAnonymousClone(t, u)
t.Run("CreateRepo", doAPICreateRepository(httpContext, false))
@ -57,16 +59,21 @@ func testGit(t *testing.T, u *url.URL) {
t.Run("Clone", doGitClone(dstPath, u))
t.Run("PushCommit", func(t *testing.T) {
PrintCurrentTest(t)
t.Run("Little", func(t *testing.T) {
PrintCurrentTest(t)
little = commitAndPush(t, littleSize, dstPath)
})
t.Run("Big", func(t *testing.T) {
PrintCurrentTest(t)
big = commitAndPush(t, bigSize, dstPath)
})
})
})
t.Run("LFS", func(t *testing.T) {
PrintCurrentTest(t)
t.Run("PushCommit", func(t *testing.T) {
PrintCurrentTest(t)
//Setup git LFS
_, err = git.NewCommand("lfs").AddArguments("install").RunInDir(dstPath)
assert.NoError(t, err)
@ -76,17 +83,21 @@ func testGit(t *testing.T, u *url.URL) {
assert.NoError(t, err)
t.Run("Little", func(t *testing.T) {
PrintCurrentTest(t)
littleLFS = commitAndPush(t, littleSize, dstPath)
})
t.Run("Big", func(t *testing.T) {
PrintCurrentTest(t)
bigLFS = commitAndPush(t, bigSize, dstPath)
})
})
t.Run("Locks", func(t *testing.T) {
PrintCurrentTest(t)
lockTest(t, u.String(), dstPath)
})
})
t.Run("Raw", func(t *testing.T) {
PrintCurrentTest(t)
session := loginUser(t, "user2")
// Request raw paths
@ -110,6 +121,7 @@ func testGit(t *testing.T, u *url.URL) {
})
t.Run("Media", func(t *testing.T) {
PrintCurrentTest(t)
session := loginUser(t, "user2")
// Request media paths
@ -132,12 +144,14 @@ func testGit(t *testing.T, u *url.URL) {
})
t.Run("SSH", func(t *testing.T) {
PrintCurrentTest(t)
sshContext := baseAPITestContext
sshContext.Reponame = "repo-tmp-18"
keyname := "my-testing-key"
//Setup key the user ssh key
withKeyFile(t, keyname, func(keyFile string) {
t.Run("CreateUserKey", doAPICreateUserKey(sshContext, "test-key", keyFile))
PrintCurrentTest(t)
//Setup remote link
sshURL := createSSHUrl(sshContext.GitPath(), u)
@ -149,6 +163,7 @@ func testGit(t *testing.T, u *url.URL) {
var little, big, littleLFS, bigLFS string
t.Run("Standard", func(t *testing.T) {
PrintCurrentTest(t)
t.Run("CreateRepo", doAPICreateRepository(sshContext, false))
//TODO get url from api
@ -156,16 +171,21 @@ func testGit(t *testing.T, u *url.URL) {
//time.Sleep(5 * time.Minute)
t.Run("PushCommit", func(t *testing.T) {
PrintCurrentTest(t)
t.Run("Little", func(t *testing.T) {
PrintCurrentTest(t)
little = commitAndPush(t, littleSize, dstPath)
})
t.Run("Big", func(t *testing.T) {
PrintCurrentTest(t)
big = commitAndPush(t, bigSize, dstPath)
})
})
})
t.Run("LFS", func(t *testing.T) {
PrintCurrentTest(t)
t.Run("PushCommit", func(t *testing.T) {
PrintCurrentTest(t)
//Setup git LFS
_, err = git.NewCommand("lfs").AddArguments("install").RunInDir(dstPath)
assert.NoError(t, err)
@ -175,17 +195,21 @@ func testGit(t *testing.T, u *url.URL) {
assert.NoError(t, err)
t.Run("Little", func(t *testing.T) {
PrintCurrentTest(t)
littleLFS = commitAndPush(t, littleSize, dstPath)
})
t.Run("Big", func(t *testing.T) {
PrintCurrentTest(t)
bigLFS = commitAndPush(t, bigSize, dstPath)
})
})
t.Run("Locks", func(t *testing.T) {
PrintCurrentTest(t)
lockTest(t, u.String(), dstPath)
})
})
t.Run("Raw", func(t *testing.T) {
PrintCurrentTest(t)
session := loginUser(t, "user2")
// Request raw paths
@ -209,6 +233,7 @@ func testGit(t *testing.T, u *url.URL) {
})
t.Run("Media", func(t *testing.T) {
PrintCurrentTest(t)
session := loginUser(t, "user2")
// Request media paths

7
integrations/integration_test.go
View file

@ -165,7 +165,12 @@ func initIntegrationTest() {
routers.GlobalInit()
}
func prepareTestEnv(t testing.TB) {
func prepareTestEnv(t testing.TB, skip ...int) {
ourSkip := 2
if len(skip) > 0 {
ourSkip += skip[0]
}
PrintCurrentTest(t, ourSkip)
assert.NoError(t, models.LoadFixtures())
assert.NoError(t, os.RemoveAll(setting.RepoRootPath))
assert.NoError(t, os.RemoveAll(models.LocalCopyPath()))

28
integrations/migration-test/migration_test.go
View file

@ -9,13 +9,13 @@ import (
"database/sql"
"fmt"
"io/ioutil"
"log"
"os"
"path"
"regexp"
"sort"
"testing"
"code.gitea.io/gitea/integrations"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/migrations"
"code.gitea.io/gitea/modules/setting"
@ -26,21 +26,22 @@ import (
var currentEngine *xorm.Engine
func initMigrationTest() {
func initMigrationTest(t *testing.T) {
integrations.PrintCurrentTest(t, 2)
giteaRoot := os.Getenv("GITEA_ROOT")
if giteaRoot == "" {
fmt.Println("Environment variable $GITEA_ROOT not set")
integrations.Printf("Environment variable $GITEA_ROOT not set\n")
os.Exit(1)
}
setting.AppPath = path.Join(giteaRoot, "gitea")
if _, err := os.Stat(setting.AppPath); err != nil {
fmt.Printf("Could not find gitea binary at %s\n", setting.AppPath)
integrations.Printf("Could not find gitea binary at %s\n", setting.AppPath)
os.Exit(1)
}
giteaConf := os.Getenv("GITEA_CONF")
if giteaConf == "" {
fmt.Println("Environment variable $GITEA_CONF not set")
integrations.Printf("Environment variable $GITEA_CONF not set\n")
os.Exit(1)
} else if !path.IsAbs(giteaConf) {
setting.CustomConf = path.Join(giteaRoot, giteaConf)
@ -51,6 +52,7 @@ func initMigrationTest() {
setting.NewContext()
setting.CheckLFSVersion()
models.LoadConfigs()
setting.NewLogServices(true)
}
func getDialect() string {
@ -125,7 +127,7 @@ func restoreOldDB(t *testing.T, version string) bool {
data, err := readSQLFromFile(version)
assert.NoError(t, err)
if len(data) == 0 {
log.Printf("No db found to restore for %s version: %s\n", models.DbCfg.Type, version)
integrations.Printf("No db found to restore for %s version: %s\n", models.DbCfg.Type, version)
return false
}
@ -212,7 +214,8 @@ func wrappedMigrate(x *xorm.Engine) error {
}
func doMigrationTest(t *testing.T, version string) {
log.Printf("Performing migration test for %s version: %s", models.DbCfg.Type, version)
integrations.PrintCurrentTest(t)
integrations.Printf("Performing migration test for %s version: %s\n", models.DbCfg.Type, version)
if !restoreOldDB(t, version) {
return
}
@ -227,19 +230,22 @@ func doMigrationTest(t *testing.T, version string) {
}
func TestMigrations(t *testing.T) {
initMigrationTest()
initMigrationTest(t)
dialect := models.DbCfg.Type
versions, err := availableVersions()
assert.NoError(t, err)
if len(versions) == 0 {
log.Printf("No old database versions available to migration test for %s\n", dialect)
integrations.Printf("No old database versions available to migration test for %s\n", dialect)
return
}
log.Printf("Preparing to test %d migrations for %s\n", len(versions), dialect)
integrations.Printf("Preparing to test %d migrations for %s\n", len(versions), dialect)
for _, version := range versions {
doMigrationTest(t, version)
t.Run(fmt.Sprintf("Migrate-%s-%s", dialect, version), func(t *testing.T) {
doMigrationTest(t, version)
})
}
}

15
integrations/mssql.ini.tmpl
View file

@ -60,14 +60,19 @@ PROVIDER = file
PROVIDER_CONFIG = data/sessions-mssql
[log]
MODE = console,file
ROOT_PATH = mssql-log
MODE = test,file
ROOT_PATH = sqlite-log
REDIRECT_MACARON_LOG = true
ROUTER = ,
MACARON = ,
XORM = file
[log.console]
LEVEL = Warn
[log.test]
LEVEL = Info
COLORIZE = true
[log.file]
LEVEL = Debug
LEVEL = Debug
[security]
INSTALL_LOCK = true

15
integrations/mysql.ini.tmpl
View file

@ -60,14 +60,19 @@ PROVIDER = file
PROVIDER_CONFIG = data/sessions-mysql
[log]
MODE = console,file
ROOT_PATH = mysql-log
MODE = test,file
ROOT_PATH = sqlite-log
REDIRECT_MACARON_LOG = true
ROUTER = ,
MACARON = ,
XORM = file
[log.console]
LEVEL = Warn
[log.test]
LEVEL = Info
COLORIZE = true
[log.file]
LEVEL = Debug
LEVEL = Debug
[security]
INSTALL_LOCK = true

16
integrations/mysql8.ini.tmpl
View file

@ -57,14 +57,20 @@ PROVIDER = file
PROVIDER_CONFIG = data/sessions-mysql8
[log]
MODE = console,file
ROOT_PATH = mysql8-log
MODE = test,file
ROOT_PATH = sqlite-log
REDIRECT_MACARON_LOG = true
ROUTER = ,
MACARON = ,
XORM = file
[log.console]
LEVEL = Warn
[log.test]
LEVEL = Info
COLORIZE = true
[log.file]
LEVEL = Debug
LEVEL = Debug
[security]
INSTALL_LOCK = true

15
integrations/pgsql.ini.tmpl
View file

@ -60,14 +60,19 @@ PROVIDER = file
PROVIDER_CONFIG = data/sessions-pgsql
[log]
MODE = console,file
ROOT_PATH = pgsql-log
MODE = test,file
ROOT_PATH = sqlite-log
REDIRECT_MACARON_LOG = true
ROUTER = ,
MACARON = ,
XORM = file
[log.console]
LEVEL = Warn
[log.test]
LEVEL = Info
COLORIZE = true
[log.file]
LEVEL = Debug
LEVEL = Debug
[security]
INSTALL_LOCK = true

29
integrations/sqlite.ini
View file

@ -33,18 +33,19 @@ APP_DATA_PATH = integrations/gitea-integration-sqlite/data
ENABLE_GZIP = true
[mailer]
ENABLED = false
ENABLED = true
MAILER_TYPE = dummy
FROM = sqlite-integration-test@gitea.io
[service]
REGISTER_EMAIL_CONFIRM = false
ENABLE_NOTIFY_MAIL = false
ENABLE_NOTIFY_MAIL = true
DISABLE_REGISTRATION = false
ENABLE_CAPTCHA = false
REQUIRE_SIGNIN_VIEW = false
DEFAULT_KEEP_EMAIL_PRIVATE = false
DEFAULT_ALLOW_CREATE_ORGANIZATION = true
NO_REPLY_ADDRESS = noreply.example.org
ENABLE_NOTIFY_MAIL = true
[picture]
DISABLE_GRAVATAR = false
@ -54,21 +55,25 @@ ENABLE_FEDERATED_AVATAR = false
PROVIDER = file
[log]
MODE = console,file
ROOT_PATH = sqlite-log
MODE = test,file
ROOT_PATH = sqlite-log
REDIRECT_MACARON_LOG = true
ROUTER = ,
MACARON = ,
XORM = file
[log.console]
LEVEL = Warn
[log.test]
LEVEL = Info
COLORIZE = true
[log.file]
LEVEL = Debug
LEVEL = Debug
[security]
INSTALL_LOCK = true
SECRET_KEY = 9pCviYTWSb
INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTI3OTU5ODN9.OQkH5UmzID2XBdwQ9TAI6Jj2t1X-wElVTjbE7aoN4I8
[mailer]
ENABLED = true
MAILER_TYPE = dummy
FROM = sqlite-integration-test@gitea.io
[oauth2]
JWT_SECRET = KZb_QLUd4fYVyxetjxC4eZkrBgWM2SndOOWDNtgUUko

104
integrations/testlogger.go Normal file
View file

@ -0,0 +1,104 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package integrations
import (
"encoding/json"
"fmt"
"os"
"runtime"
"strings"
"testing"
"code.gitea.io/gitea/modules/log"
)
var prefix string
// TestLogger is a logger which will write to the testing log
type TestLogger struct {
log.WriterLogger
}
var writerCloser = &testLoggerWriterCloser{}
type testLoggerWriterCloser struct {
t testing.TB
}
func (w *testLoggerWriterCloser) Write(p []byte) (int, error) {
if w.t != nil {
if len(p) > 0 && p[len(p)-1] == '\n' {
p = p[:len(p)-1]
}
w.t.Log(string(p))
return len(p), nil
}
return len(p), nil
}
func (w *testLoggerWriterCloser) Close() error {
return nil
}
// PrintCurrentTest prints the current test to os.Stdout
func PrintCurrentTest(t testing.TB, skip ...int) {
actualSkip := 1
if len(skip) > 0 {
actualSkip = skip[0]
}
_, filename, line, _ := runtime.Caller(actualSkip)
if log.CanColorStdout {
fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", log.NewColoredValue(t.Name()), strings.TrimPrefix(filename, prefix), line)
} else {
fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", t.Name(), strings.TrimPrefix(filename, prefix), line)
}
writerCloser.t = t
}
// Printf takes a format and args and prints the string to os.Stdout
func Printf(format string, args ...interface{}) {
if log.CanColorStdout {
for i := 0; i < len(args); i++ {
args[i] = log.NewColoredValue(args[i])
}
}
fmt.Fprintf(os.Stdout, "\t"+format, args...)
}
// NewTestLogger creates a TestLogger as a log.LoggerProvider
func NewTestLogger() log.LoggerProvider {
logger := &TestLogger{}
logger.Colorize = log.CanColorStdout
logger.Level = log.TRACE
return logger
}
// Init inits connection writer with json config.
// json config only need key "level".
func (log *TestLogger) Init(config string) error {
err := json.Unmarshal([]byte(config), log)
if err != nil {
return err
}
log.NewWriterLogger(writerCloser)
return nil
}
// Flush when log should be flushed
func (log *TestLogger) Flush() {
}
// GetName returns the default name for this implementation
func (log *TestLogger) GetName() string {
return "test"
}
func init() {
log.Register("test", NewTestLogger)
_, filename, _, _ := runtime.Caller(0)
prefix = strings.TrimSuffix(filename, "integrations/testlogger.go")
}

328
modules/log/base.go
View file

@ -1,328 +0,0 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package log
import (
"bytes"
"fmt"
"io"
"regexp"
"strings"
"sync"
)
// These flags define which text to prefix to each log entry generated
// by the Logger. Bits are or'ed together to control what's printed.
// There is no control over the order they appear (the order listed
// here) or the format they present (as described in the comments).
// The prefix is followed by a colon only if more than time is stated
// is specified. For example, flags Ldate | Ltime
// produce, 2009/01/23 01:23:23 message.
// The standard is:
// 2009/01/23 01:23:23 ...a/b/c/d.go:23:runtime.Caller() [I]: message
const (
Ldate = 1 << iota // the date in the local time zone: 2009/01/23
Ltime // the time in the local time zone: 01:23:23
Lmicroseconds // microsecond resolution: 01:23:23.123123. assumes Ltime.
Llongfile // full file name and line number: /a/b/c/d.go:23
Lshortfile // final file name element and line number: d.go:23. overrides Llongfile
Lfuncname // function name of the caller: runtime.Caller()
Lshortfuncname // last part of the function name
LUTC // if Ldate or Ltime is set, use UTC rather than the local time zone
Llevelinitial // Initial character of the provided level in brackets eg. [I] for info
Llevel // Provided level in brackets [INFO]
// Last 20 characters of the filename
Lmedfile = Lshortfile | Llongfile
// LstdFlags is the initial value for the standard logger
LstdFlags = Ldate | Ltime | Lmedfile | Lshortfuncname | Llevelinitial
)
var flagFromString = map[string]int{
"none": 0,
"date": Ldate,
"time": Ltime,
"microseconds": Lmicroseconds,
"longfile": Llongfile,
"shortfile": Lshortfile,
"funcname": Lfuncname,
"shortfuncname": Lshortfuncname,
"utc": LUTC,
"levelinitial": Llevelinitial,
"level": Llevel,
"medfile": Lmedfile,
"stdflags": LstdFlags,
}
// FlagsFromString takes a comma separated list of flags and returns
// the flags for this string
func FlagsFromString(from string) int {
flags := 0
for _, flag := range strings.Split(strings.ToLower(from), ",") {
f, ok := flagFromString[strings.TrimSpace(flag)]
if ok {
flags = flags | f
}
}
return flags
}
type byteArrayWriter []byte
func (b *byteArrayWriter) Write(p []byte) (int, error) {
*b = append(*b, p...)
return len(p), nil
}
// BaseLogger represent a basic logger for Gitea
type BaseLogger struct {
out io.WriteCloser
mu sync.Mutex
Level Level `json:"level"`
StacktraceLevel Level `json:"stacktraceLevel"`
Flags int `json:"flags"`
Prefix string `json:"prefix"`
Colorize bool `json:"colorize"`
Expression string `json:"expression"`
regexp *regexp.Regexp
}
func (b *BaseLogger) createLogger(out io.WriteCloser, level ...Level) {
b.mu.Lock()
defer b.mu.Unlock()
b.out = out
switch b.Flags {
case 0:
b.Flags = LstdFlags
case -1:
b.Flags = 0
}
if len(level) > 0 {
b.Level = level[0]
}
b.createExpression()
}
func (b *BaseLogger) createExpression() {
if len(b.Expression) > 0 {
var err error
b.regexp, err = regexp.Compile(b.Expression)
if err != nil {
b.regexp = nil
}
}
}
// GetLevel returns the logging level for this logger
func (b *BaseLogger) GetLevel() Level {
return b.Level
}
// GetStacktraceLevel returns the stacktrace logging level for this logger
func (b *BaseLogger) GetStacktraceLevel() Level {
return b.StacktraceLevel
}
// Copy of cheap integer to fixed-width decimal to ascii from logger.
func itoa(buf *[]byte, i int, wid int) {
var b [20]byte
bp := len(b) - 1
for i >= 10 || wid > 1 {
wid--
q := i / 10
b[bp] = byte('0' + i - q*10)
bp--
i = q
}
// i < 10
b[bp] = byte('0' + i)
*buf = append(*buf, b[bp:]...)
}
func (b *BaseLogger) createMsg(buf *[]byte, event *Event) {
*buf = append(*buf, b.Prefix...)
t := event.time
if b.Flags&(Ldate|Ltime|Lmicroseconds) != 0 {
if b.Colorize {
*buf = append(*buf, fgCyanBytes...)
}
if b.Flags&LUTC != 0 {
t = t.UTC()
}
if b.Flags&Ldate != 0 {
year, month, day := t.Date()
itoa(buf, year, 4)
*buf = append(*buf, '/')
itoa(buf, int(month), 2)
*buf = append(*buf, '/')
itoa(buf, day, 2)
*buf = append(*buf, ' ')
}
if b.Flags&(Ltime|Lmicroseconds) != 0 {
hour, min, sec := t.Clock()
itoa(buf, hour, 2)
*buf = append(*buf, ':')
itoa(buf, min, 2)
*buf = append(*buf, ':')
itoa(buf, sec, 2)
if b.Flags&Lmicroseconds != 0 {
*buf = append(*buf, '.')
itoa(buf, t.Nanosecond()/1e3, 6)
}
*buf = append(*buf, ' ')
}
if b.Colorize {
*buf = append(*buf, resetBytes...)
}
}
if b.Flags&(Lshortfile|Llongfile) != 0 {
if b.Colorize {
*buf = append(*buf, fgGreenBytes...)
}
file := event.filename
if b.Flags&Lmedfile == Lmedfile {
startIndex := len(file) - 20
if startIndex > 0 {
file = "..." + file[startIndex:]
}
} else if b.Flags&Lshortfile != 0 {
startIndex := strings.LastIndexByte(file, '/')
if startIndex > 0 && startIndex < len(file) {
file = file[startIndex+1:]
}
}
*buf = append(*buf, file...)
*buf = append(*buf, ':')
itoa(buf, event.line, -1)
if b.Flags&(Lfuncname|Lshortfuncname) != 0 {
*buf = append(*buf, ':')
} else {
if b.Colorize {
*buf = append(*buf, resetBytes...)
}
*buf = append(*buf, ' ')
}
}
if b.Flags&(Lfuncname|Lshortfuncname) != 0 {
if b.Colorize {
*buf = append(*buf, fgGreenBytes...)
}
funcname := event.caller
if b.Flags&Lshortfuncname != 0 {
lastIndex := strings.LastIndexByte(funcname, '.')
if lastIndex > 0 && len(funcname) > lastIndex+1 {
funcname = funcname[lastIndex+1:]
}
}
*buf = append(*buf, funcname...)
if b.Colorize {
*buf = append(*buf, resetBytes...)
}
*buf = append(*buf, ' ')
}
if b.Flags&(Llevel|Llevelinitial) != 0 {
level := strings.ToUpper(event.level.String())
if b.Colorize {
*buf = append(*buf, levelToColor[event.level]...)
}
*buf = append(*buf, '[')
if b.Flags&Llevelinitial != 0 {
*buf = append(*buf, level[0])
} else {
*buf = append(*buf, level...)
}
*buf = append(*buf, ']')
if b.Colorize {
*buf = append(*buf, resetBytes...)
}
*buf = append(*buf, ' ')
}
var msg = []byte(event.msg)
if len(msg) > 0 && msg[len(msg)-1] == '\n' {
msg = msg[:len(msg)-1]
}
pawMode := allowColor
if !b.Colorize {
pawMode = removeColor
}
baw := byteArrayWriter(*buf)
(&protectedANSIWriter{
w: &baw,
mode: pawMode,
}).Write([]byte(msg))
*buf = baw
if event.stacktrace != "" && b.StacktraceLevel <= event.level {
lines := bytes.Split([]byte(event.stacktrace), []byte("\n"))
if len(lines) > 1 {
for _, line := range lines {
*buf = append(*buf, "\n\t"...)
*buf = append(*buf, line...)
}
}
*buf = append(*buf, '\n')
}
*buf = append(*buf, '\n')
}
// LogEvent logs the event to the internal writer
func (b *BaseLogger) LogEvent(event *Event) error {
if b.Level > event.level {
return nil
}
b.mu.Lock()
defer b.mu.Unlock()
if !b.Match(event) {
return nil
}
var buf []byte
b.createMsg(&buf, event)
_, err := b.out.Write(buf)
return err
}
// Match checks if the given event matches the logger's regexp expression
func (b *BaseLogger) Match(event *Event) bool {
if b.regexp == nil {
return true
}
if b.regexp.Match([]byte(fmt.Sprintf("%s:%d:%s", event.filename, event.line, event.caller))) {
return true
}
// Match on the non-colored msg - therefore strip out colors
var msg []byte
baw := byteArrayWriter(msg)
(&protectedANSIWriter{
w: &baw,
mode: removeColor,
}).Write([]byte(event.msg))
msg = baw
if b.regexp.Match(msg) {
return true
}
return false
}
// Close the base logger
func (b *BaseLogger) Close() {
b.mu.Lock()
defer b.mu.Unlock()
if b.out != nil {
b.out.Close()
}
}
// GetName returns empty for these provider loggers
func (b *BaseLogger) GetName() string {
return ""
}

40
modules/log/colors.go
View file

@ -268,8 +268,8 @@ normalLoop:
// ColoredValue will Color the provided value
type ColoredValue struct {
ColorBytes *[]byte
ResetBytes *[]byte
colorBytes *[]byte
resetBytes *[]byte
Value *interface{}
}
@ -290,14 +290,14 @@ func NewColoredValuePointer(value *interface{}, color ...ColorAttribute) *Colore
if len(color) > 0 {
bytes := ColorBytes(color...)
return &ColoredValue{
ColorBytes: &bytes,
ResetBytes: &resetBytes,
colorBytes: &bytes,
resetBytes: &resetBytes,
Value: value,
}
}
return &ColoredValue{
ColorBytes: &fgBoldBytes,
ResetBytes: &resetBytes,
colorBytes: &fgBoldBytes,
resetBytes: &resetBytes,
Value: value,
}
@ -310,17 +310,37 @@ func NewColoredValueBytes(value interface{}, colorBytes *[]byte) *ColoredValue {
return val
}
return &ColoredValue{
ColorBytes: colorBytes,
ResetBytes: &resetBytes,
colorBytes: colorBytes,
resetBytes: &resetBytes,
Value: &value,
}
}
// Format will format the provided value and protect against ANSI spoofing within the value
func (cv *ColoredValue) Format(s fmt.State, c rune) {
s.Write([]byte(*cv.ColorBytes))
s.Write([]byte(*cv.colorBytes))
fmt.Fprintf(&protectedANSIWriter{w: s}, fmtString(s, c), *(cv.Value))
s.Write([]byte(*cv.ResetBytes))
s.Write([]byte(*cv.resetBytes))
}
// SetColorBytes will allow a user to set the colorBytes of a colored value
func (cv *ColoredValue) SetColorBytes(colorBytes []byte) {
cv.colorBytes = &colorBytes
}
// SetColorBytesPointer will allow a user to set the colorBytes pointer of a colored value
func (cv *ColoredValue) SetColorBytesPointer(colorBytes *[]byte) {
cv.colorBytes = colorBytes
}
// SetResetBytes will allow a user to set the resetBytes pointer of a colored value
func (cv *ColoredValue) SetResetBytes(resetBytes []byte) {
cv.resetBytes = &resetBytes
}
// SetResetBytesPointer will allow a user to set the resetBytes pointer of a colored value
func (cv *ColoredValue) SetResetBytesPointer(resetBytes *[]byte) {
cv.resetBytes = resetBytes
}
func fmtString(s fmt.State, c rune) string {

4
modules/log/conn.go
View file

@ -77,7 +77,7 @@ func (i *connWriter) connect() error {
// ConnLogger implements LoggerProvider.
// it writes messages in keep-live tcp connection.
type ConnLogger struct {
BaseLogger
WriterLogger
ReconnectOnMsg bool `json:"reconnectOnMsg"`
Reconnect bool `json:"reconnect"`
Net string `json:"net"`
@ -98,7 +98,7 @@ func (log *ConnLogger) Init(jsonconfig string) error {
if err != nil {
return err
}
log.createLogger(&connWriter{
log.NewWriterLogger(&connWriter{
ReconnectOnMsg: log.ReconnectOnMsg,
Reconnect: log.Reconnect,
Net: log.Net,

8
modules/log/console.go
View file

@ -34,14 +34,14 @@ func (n *nopWriteCloser) Close() error {
// ConsoleLogger implements LoggerProvider and writes messages to terminal.
type ConsoleLogger struct {
BaseLogger
WriterLogger
Stderr bool `json:"stderr"`
}
// NewConsoleLogger create ConsoleLogger returning as LoggerProvider.
func NewConsoleLogger() LoggerProvider {
log := &ConsoleLogger{}
log.createLogger(&nopWriteCloser{
log.NewWriterLogger(&nopWriteCloser{
w: os.Stdout,
})
return log
@ -55,11 +55,11 @@ func (log *ConsoleLogger) Init(config string) error {
return err
}
if log.Stderr {
log.createLogger(&nopWriteCloser{
log.NewWriterLogger(&nopWriteCloser{
w: os.Stderr,
})
} else {
log.createLogger(log.out)
log.NewWriterLogger(log.out)
}
return nil
}

4
modules/log/file.go
View file

@ -20,7 +20,7 @@ import (
// FileLogger implements LoggerProvider.
// It writes messages by lines limit, file size limit, or time frequency.
type FileLogger struct {
BaseLogger
WriterLogger
mw *MuxWriter
// The opened file
Filename string `json:"filename"`
@ -106,7 +106,7 @@ func (log *FileLogger) Init(config string) error {
return errors.New("config must have filename")
}
// set MuxWriter as Logger's io.Writer
log.createLogger(log.mw)
log.NewWriterLogger(log.mw)
return log.StartLogger()
}

2
modules/log/file_test.go
View file

@ -89,7 +89,6 @@ func TestFileLogger(t *testing.T) {
assert.Equal(t, expected, string(logData))
event.level = DEBUG
expected = expected + ""
fileLogger.LogEvent(&event)
fileLogger.Flush()
logData, err = ioutil.ReadFile(filename)
@ -97,7 +96,6 @@ func TestFileLogger(t *testing.T) {
assert.Equal(t, expected, string(logData))
event.level = TRACE
expected = expected + ""
fileLogger.LogEvent(&event)
fileLogger.Flush()
logData, err = ioutil.ReadFile(filename)

64
modules/log/flags.go Normal file
View file

@ -0,0 +1,64 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package log
import "strings"
// These flags define which text to prefix to each log entry generated
// by the Logger. Bits are or'ed together to control what's printed.
// There is no control over the order they appear (the order listed
// here) or the format they present (as described in the comments).
// The prefix is followed by a colon only if more than time is stated
// is specified. For example, flags Ldate | Ltime
// produce, 2009/01/23 01:23:23 message.
// The standard is:
// 2009/01/23 01:23:23 ...a/logger/c/d.go:23:runtime.Caller() [I]: message
const (
Ldate = 1 << iota // the date in the local time zone: 2009/01/23
Ltime // the time in the local time zone: 01:23:23
Lmicroseconds // microsecond resolution: 01:23:23.123123. assumes Ltime.
Llongfile // full file name and line number: /a/logger/c/d.go:23
Lshortfile // final file name element and line number: d.go:23. overrides Llongfile
Lfuncname // function name of the caller: runtime.Caller()
Lshortfuncname // last part of the function name
LUTC // if Ldate or Ltime is set, use UTC rather than the local time zone
Llevelinitial // Initial character of the provided level in brackets eg. [I] for info
Llevel // Provided level in brackets [INFO]
// Last 20 characters of the filename
Lmedfile = Lshortfile | Llongfile
// LstdFlags is the initial value for the standard logger
LstdFlags = Ldate | Ltime | Lmedfile | Lshortfuncname | Llevelinitial
)
var flagFromString = map[string]int{
"none": 0,
"date": Ldate,
"time": Ltime,
"microseconds": Lmicroseconds,
"longfile": Llongfile,
"shortfile": Lshortfile,
"funcname": Lfuncname,
"shortfuncname": Lshortfuncname,
"utc": LUTC,
"levelinitial": Llevelinitial,
"level": Llevel,
"medfile": Lmedfile,
"stdflags": LstdFlags,
}
// FlagsFromString takes a comma separated list of flags and returns
// the flags for this string
func FlagsFromString(from string) int {
flags := 0
for _, flag := range strings.Split(strings.ToLower(from), ",") {
f, ok := flagFromString[strings.TrimSpace(flag)]
if ok {
flags = flags | f
}
}
return flags
}

4
modules/log/smtp.go
View file

@ -31,7 +31,7 @@ func (s *smtpWriter) Close() error {
// SMTPLogger implements LoggerProvider and is used to send emails via given SMTP-server.
type SMTPLogger struct {
BaseLogger
WriterLogger
Username string `json:"Username"`
Password string `json:"password"`
Host string `json:"host"`
@ -63,7 +63,7 @@ func (log *SMTPLogger) Init(jsonconfig string) error {
if err != nil {
return err
}
log.createLogger(&smtpWriter{
log.NewWriterLogger(&smtpWriter{
owner: log,
})
log.sendMailFn = smtp.SendMail

273
modules/log/writer.go Normal file
View file

@ -0,0 +1,273 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package log
import (
"bytes"
"fmt"
"io"
"regexp"
"strings"
"sync"
)
type byteArrayWriter []byte
func (b *byteArrayWriter) Write(p []byte) (int, error) {
*b = append(*b, p...)
return len(p), nil
}
// WriterLogger represent a basic logger for Gitea
type WriterLogger struct {
out io.WriteCloser
mu sync.Mutex
Level Level `json:"level"`
StacktraceLevel Level `json:"stacktraceLevel"`
Flags int `json:"flags"`
Prefix string `json:"prefix"`
Colorize bool `json:"colorize"`
Expression string `json:"expression"`
regexp *regexp.Regexp
}
// NewWriterLogger creates a new WriterLogger from the provided WriteCloser.
// Optionally the level can be changed at the same time.
func (logger *WriterLogger) NewWriterLogger(out io.WriteCloser, level ...Level) {
logger.mu.Lock()
defer logger.mu.Unlock()
logger.out = out
switch logger.Flags {
case 0:
logger.Flags = LstdFlags
case -1:
logger.Flags = 0
}
if len(level) > 0 {
logger.Level = level[0]
}
logger.createExpression()
}
func (logger *WriterLogger) createExpression() {
if len(logger.Expression) > 0 {
var err error
logger.regexp, err = regexp.Compile(logger.Expression)
if err != nil {
logger.regexp = nil
}
}
}
// GetLevel returns the logging level for this logger
func (logger *WriterLogger) GetLevel() Level {
return logger.Level
}
// GetStacktraceLevel returns the stacktrace logging level for this logger
func (logger *WriterLogger) GetStacktraceLevel() Level {
return logger.StacktraceLevel
}
// Copy of cheap integer to fixed-width decimal to ascii from logger.
func itoa(buf *[]byte, i int, wid int) {
var logger [20]byte
bp := len(logger) - 1
for i >= 10 || wid > 1 {
wid--
q := i / 10
logger[bp] = byte('0' + i - q*10)
bp--
i = q
}
// i < 10
logger[bp] = byte('0' + i)
*buf = append(*buf, logger[bp:]...)
}
func (logger *WriterLogger) createMsg(buf *[]byte, event *Event) {
*buf = append(*buf, logger.Prefix...)
t := event.time
if logger.Flags&(Ldate|Ltime|Lmicroseconds) != 0 {
if logger.Colorize {
*buf = append(*buf, fgCyanBytes...)
}
if logger.Flags&LUTC != 0 {
t = t.UTC()
}
if logger.Flags&Ldate != 0 {
year, month, day := t.Date()
itoa(buf, year, 4)
*buf = append(*buf, '/')
itoa(buf, int(month), 2)
*buf = append(*buf, '/')
itoa(buf, day, 2)
*buf = append(*buf, ' ')
}
if logger.Flags&(Ltime|Lmicroseconds) != 0 {
hour, min, sec := t.Clock()
itoa(buf, hour, 2)
*buf = append(*buf, ':')
itoa(buf, min, 2)
*buf = append(*buf, ':')
itoa(buf, sec, 2)
if logger.Flags&Lmicroseconds != 0 {
*buf = append(*buf, '.')
itoa(buf, t.Nanosecond()/1e3, 6)
}
*buf = append(*buf, ' ')
}
if logger.Colorize {
*buf = append(*buf, resetBytes...)
}
}
if logger.Flags&(Lshortfile|Llongfile) != 0 {
if logger.Colorize {
*buf = append(*buf, fgGreenBytes...)
}
file := event.filename
if logger.Flags&Lmedfile == Lmedfile {
startIndex := len(file) - 20
if startIndex > 0 {
file = "..." + file[startIndex:]
}
} else if logger.Flags&Lshortfile != 0 {
startIndex := strings.LastIndexByte(file, '/')
if startIndex > 0 && startIndex < len(file) {
file = file[startIndex+1:]
}
}
*buf = append(*buf, file...)
*buf = append(*buf, ':')
itoa(buf, event.line, -1)
if logger.Flags&(Lfuncname|Lshortfuncname) != 0 {
*buf = append(*buf, ':')
} else {
if logger.Colorize {
*buf = append(*buf, resetBytes...)
}
*buf = append(*buf, ' ')
}
}
if logger.Flags&(Lfuncname|Lshortfuncname) != 0 {
if logger.Colorize {
*buf = append(*buf, fgGreenBytes...)
}
funcname := event.caller
if logger.Flags&Lshortfuncname != 0 {
lastIndex := strings.LastIndexByte(funcname, '.')
if lastIndex > 0 && len(funcname) > lastIndex+1 {
funcname = funcname[lastIndex+1:]
}
}
*buf = append(*buf, funcname...)
if logger.Colorize {
*buf = append(*buf, resetBytes...)
}
*buf = append(*buf, ' ')
}
if logger.Flags&(Llevel|Llevelinitial) != 0 {
level := strings.ToUpper(event.level.String())
if logger.Colorize {
*buf = append(*buf, levelToColor[event.level]...)
}
*buf = append(*buf, '[')
if logger.Flags&Llevelinitial != 0 {
*buf = append(*buf, level[0])
} else {
*buf = append(*buf, level...)
}
*buf = append(*buf, ']')
if logger.Colorize {
*buf = append(*buf, resetBytes...)
}
*buf = append(*buf, ' ')
}
var msg = []byte(event.msg)
if len(msg) > 0 && msg[len(msg)-1] == '\n' {
msg = msg[:len(msg)-1]
}
pawMode := allowColor
if !logger.Colorize {
pawMode = removeColor
}
baw := byteArrayWriter(*buf)
(&protectedANSIWriter{
w: &baw,
mode: pawMode,
}).Write([]byte(msg))
*buf = baw
if event.stacktrace != "" && logger.StacktraceLevel <= event.level {
lines := bytes.Split([]byte(event.stacktrace), []byte("\n"))
if len(lines) > 1 {
for _, line := range lines {
*buf = append(*buf, "\n\t"...)
*buf = append(*buf, line...)
}
}
*buf = append(*buf, '\n')
}
*buf = append(*buf, '\n')
}
// LogEvent logs the event to the internal writer
func (logger *WriterLogger) LogEvent(event *Event) error {
if logger.Level > event.level {
return nil
}
logger.mu.Lock()
defer logger.mu.Unlock()
if !logger.Match(event) {
return nil
}
var buf []byte
logger.createMsg(&buf, event)
_, err := logger.out.Write(buf)
return err
}
// Match checks if the given event matches the logger's regexp expression
func (logger *WriterLogger) Match(event *Event) bool {
if logger.regexp == nil {
return true
}
if logger.regexp.Match([]byte(fmt.Sprintf("%s:%d:%s", event.filename, event.line, event.caller))) {
return true
}
// Match on the non-colored msg - therefore strip out colors
var msg []byte
baw := byteArrayWriter(msg)
(&protectedANSIWriter{
w: &baw,
mode: removeColor,
}).Write([]byte(event.msg))
msg = baw
if logger.regexp.Match(msg) {
return true
}
return false
}
// Close the base logger
func (logger *WriterLogger) Close() {
logger.mu.Lock()
defer logger.mu.Unlock()
if logger.out != nil {
logger.out.Close()
}
}
// GetName returns empty for these provider loggers
func (logger *WriterLogger) GetName() string {
return ""
}

12
modules/log/base_test.go → modules/log/writer_test.go
View file

@ -38,7 +38,7 @@ func TestBaseLogger(t *testing.T) {
},
}
prefix := "TestPrefix "
b := BaseLogger{
b := WriterLogger{
out: c,
Level: INFO,
Flags: LstdFlags | LUTC,
@ -115,7 +115,7 @@ func TestBaseLoggerDated(t *testing.T) {
},
}
prefix := ""
b := BaseLogger{
b := WriterLogger{
out: c,
Level: WARN,
Flags: Ldate | Ltime | Lmicroseconds | Lshortfile | Llevel,
@ -195,14 +195,14 @@ func TestBaseLoggerMultiLineNoFlagsRegexp(t *testing.T) {
},
}
prefix := ""
b := BaseLogger{
b := WriterLogger{
Level: DEBUG,
StacktraceLevel: ERROR,
Flags: -1,
Prefix: prefix,
Expression: "FILENAME",
}
b.createLogger(c)
b.NewWriterLogger(c)
location, _ := time.LoadLocation("EST")
@ -263,14 +263,14 @@ func TestBrokenRegexp(t *testing.T) {
},
}
b := BaseLogger{
b := WriterLogger{
Level: DEBUG,
StacktraceLevel: ERROR,
Flags: -1,
Prefix: prefix,
Expression: "\\",
}
b.createLogger(c)
b.NewWriterLogger(c)
assert.Empty(t, b.regexp)
b.Close()
assert.Equal(t, true, closed)

9
modules/setting/log.go
View file

@ -273,6 +273,15 @@ func newLogService() {
golog.SetOutput(log.NewLoggerAsWriter("INFO", log.GetLogger(log.DEFAULT)))
}
// NewLogServices creates all the log services
func NewLogServices(disableConsole bool) {
newLogService()
newMacaronLogService()
newRouterLogService()
newAccessLogService()
NewXORMLogService(disableConsole)
}
// NewXORMLogService initializes xorm logger service
func NewXORMLogService(disableConsole bool) {
EnableXORMLog = Cfg.Section("log").Key("ENABLE_XORM_LOG").MustBool(true)

6
modules/setting/setting.go
View file

@ -982,11 +982,7 @@ func loadOrGenerateInternalToken(sec *ini.Section) string {
// NewServices initializes the services
func NewServices() {
newService()
newLogService()
newMacaronLogService()
newAccessLogService()
newRouterLogService()
NewXORMLogService(false)
NewLogServices(false)
newCacheService()
newSessionService()
newMailService()