mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2024-12-05 15:43:12 +01:00
60e4a98ab0
The `ToUTF8*` functions were stripping BOM, while BOM is actually valid in UTF8, so the stripping must be optional depending on use case. This does: - Add a options struct to all `ToUTF8*` functions, that by default will strip BOM to preserve existing behaviour - Remove `ToUTF8` function, it was dead code - Rename `ToUTF8WithErr` to `ToUTF8` - Preserve BOM in Monaco Editor - Remove a unnecessary newline in the textarea value. Browsers did ignore it, it seems but it's better not to rely on this behaviour. Fixes: https://github.com/go-gitea/gitea/issues/28743 Related: https://github.com/go-gitea/gitea/issues/6716 which seems to have once introduced a mechanism that strips and re-adds the BOM, but from what I can tell, this mechanism was removed at some point after that PR.
211 lines
6.4 KiB
Go
211 lines
6.4 KiB
Go
// Copyright 2014 The Gogs Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package charset
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io"
|
|
"strings"
|
|
"unicode/utf8"
|
|
|
|
"code.gitea.io/gitea/modules/log"
|
|
"code.gitea.io/gitea/modules/setting"
|
|
"code.gitea.io/gitea/modules/util"
|
|
|
|
"github.com/gogs/chardet"
|
|
"golang.org/x/net/html/charset"
|
|
"golang.org/x/text/transform"
|
|
)
|
|
|
|
// UTF8BOM is the utf-8 byte-order marker
|
|
var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
|
|
|
|
type ConvertOpts struct {
|
|
KeepBOM bool
|
|
}
|
|
|
|
// ToUTF8WithFallbackReader detects the encoding of content and converts to UTF-8 reader if possible
|
|
func ToUTF8WithFallbackReader(rd io.Reader, opts ConvertOpts) io.Reader {
|
|
buf := make([]byte, 2048)
|
|
n, err := util.ReadAtMost(rd, buf)
|
|
if err != nil {
|
|
return io.MultiReader(bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), rd)
|
|
}
|
|
|
|
charsetLabel, err := DetectEncoding(buf[:n])
|
|
if err != nil || charsetLabel == "UTF-8" {
|
|
return io.MultiReader(bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), rd)
|
|
}
|
|
|
|
encoding, _ := charset.Lookup(charsetLabel)
|
|
if encoding == nil {
|
|
return io.MultiReader(bytes.NewReader(buf[:n]), rd)
|
|
}
|
|
|
|
return transform.NewReader(
|
|
io.MultiReader(
|
|
bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)),
|
|
rd,
|
|
),
|
|
encoding.NewDecoder(),
|
|
)
|
|
}
|
|
|
|
// ToUTF8 converts content to UTF8 encoding
|
|
func ToUTF8(content []byte, opts ConvertOpts) (string, error) {
|
|
charsetLabel, err := DetectEncoding(content)
|
|
if err != nil {
|
|
return "", err
|
|
} else if charsetLabel == "UTF-8" {
|
|
return string(MaybeRemoveBOM(content, opts)), nil
|
|
}
|
|
|
|
encoding, _ := charset.Lookup(charsetLabel)
|
|
if encoding == nil {
|
|
return string(content), fmt.Errorf("Unknown encoding: %s", charsetLabel)
|
|
}
|
|
|
|
// If there is an error, we concatenate the nicely decoded part and the
|
|
// original left over. This way we won't lose much data.
|
|
result, n, err := transform.Bytes(encoding.NewDecoder(), content)
|
|
if err != nil {
|
|
result = append(result, content[n:]...)
|
|
}
|
|
|
|
result = MaybeRemoveBOM(result, opts)
|
|
|
|
return string(result), err
|
|
}
|
|
|
|
// ToUTF8WithFallback detects the encoding of content and converts to UTF-8 if possible
|
|
func ToUTF8WithFallback(content []byte, opts ConvertOpts) []byte {
|
|
bs, _ := io.ReadAll(ToUTF8WithFallbackReader(bytes.NewReader(content), opts))
|
|
return bs
|
|
}
|
|
|
|
// ToUTF8DropErrors makes sure the return string is valid utf-8; attempts conversion if possible
|
|
func ToUTF8DropErrors(content []byte, opts ConvertOpts) []byte {
|
|
charsetLabel, err := DetectEncoding(content)
|
|
if err != nil || charsetLabel == "UTF-8" {
|
|
return MaybeRemoveBOM(content, opts)
|
|
}
|
|
|
|
encoding, _ := charset.Lookup(charsetLabel)
|
|
if encoding == nil {
|
|
return content
|
|
}
|
|
|
|
// We ignore any non-decodable parts from the file.
|
|
// Some parts might be lost
|
|
var decoded []byte
|
|
decoder := encoding.NewDecoder()
|
|
idx := 0
|
|
for {
|
|
result, n, err := transform.Bytes(decoder, content[idx:])
|
|
decoded = append(decoded, result...)
|
|
if err == nil {
|
|
break
|
|
}
|
|
decoded = append(decoded, ' ')
|
|
idx = idx + n + 1
|
|
if idx >= len(content) {
|
|
break
|
|
}
|
|
}
|
|
|
|
return MaybeRemoveBOM(decoded, opts)
|
|
}
|
|
|
|
// MaybeRemoveBOM removes a UTF-8 BOM from a []byte when opts.KeepBOM is false
|
|
func MaybeRemoveBOM(content []byte, opts ConvertOpts) []byte {
|
|
if opts.KeepBOM {
|
|
return content
|
|
}
|
|
if len(content) > 2 && bytes.Equal(content[0:3], UTF8BOM) {
|
|
return content[3:]
|
|
}
|
|
return content
|
|
}
|
|
|
|
// DetectEncoding detect the encoding of content
|
|
func DetectEncoding(content []byte) (string, error) {
|
|
// First we check if the content represents valid utf8 content excepting a truncated character at the end.
|
|
|
|
// Now we could decode all the runes in turn but this is not necessarily the cheapest thing to do
|
|
// instead we walk backwards from the end to trim off a the incomplete character
|
|
toValidate := content
|
|
end := len(toValidate) - 1
|
|
|
|
if end < 0 {
|
|
// no-op
|
|
} else if toValidate[end]>>5 == 0b110 {
|
|
// Incomplete 1 byte extension e.g. © <c2><a9> which has been truncated to <c2>
|
|
toValidate = toValidate[:end]
|
|
} else if end > 0 && toValidate[end]>>6 == 0b10 && toValidate[end-1]>>4 == 0b1110 {
|
|
// Incomplete 2 byte extension e.g. ⛔ <e2><9b><94> which has been truncated to <e2><9b>
|
|
toValidate = toValidate[:end-1]
|
|
} else if end > 1 && toValidate[end]>>6 == 0b10 && toValidate[end-1]>>6 == 0b10 && toValidate[end-2]>>3 == 0b11110 {
|
|
// Incomplete 3 byte extension e.g. 💩 <f0><9f><92><a9> which has been truncated to <f0><9f><92>
|
|
toValidate = toValidate[:end-2]
|
|
}
|
|
if utf8.Valid(toValidate) {
|
|
log.Debug("Detected encoding: utf-8 (fast)")
|
|
return "UTF-8", nil
|
|
}
|
|
|
|
textDetector := chardet.NewTextDetector()
|
|
var detectContent []byte
|
|
if len(content) < 1024 {
|
|
// Check if original content is valid
|
|
if _, err := textDetector.DetectBest(content); err != nil {
|
|
return "", err
|
|
}
|
|
times := 1024 / len(content)
|
|
detectContent = make([]byte, 0, times*len(content))
|
|
for i := 0; i < times; i++ {
|
|
detectContent = append(detectContent, content...)
|
|
}
|
|
} else {
|
|
detectContent = content
|
|
}
|
|
|
|
// Now we can't use DetectBest or just results[0] because the result isn't stable - so we need a tie break
|
|
results, err := textDetector.DetectAll(detectContent)
|
|
if err != nil {
|
|
if err == chardet.NotDetectedError && len(setting.Repository.AnsiCharset) > 0 {
|
|
log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)
|
|
return setting.Repository.AnsiCharset, nil
|
|
}
|
|
return "", err
|
|
}
|
|
|
|
topConfidence := results[0].Confidence
|
|
topResult := results[0]
|
|
priority, has := setting.Repository.DetectedCharsetScore[strings.ToLower(strings.TrimSpace(topResult.Charset))]
|
|
for _, result := range results {
|
|
// As results are sorted in confidence order - if we have a different confidence
|
|
// we know it's less than the current confidence and can break out of the loop early
|
|
if result.Confidence != topConfidence {
|
|
break
|
|
}
|
|
|
|
// Otherwise check if this results is earlier in the DetectedCharsetOrder than our current top guess
|
|
resultPriority, resultHas := setting.Repository.DetectedCharsetScore[strings.ToLower(strings.TrimSpace(result.Charset))]
|
|
if resultHas && (!has || resultPriority < priority) {
|
|
topResult = result
|
|
priority = resultPriority
|
|
has = true
|
|
}
|
|
}
|
|
|
|
// FIXME: to properly decouple this function the fallback ANSI charset should be passed as an argument
|
|
if topResult.Charset != "UTF-8" && len(setting.Repository.AnsiCharset) > 0 {
|
|
log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)
|
|
return setting.Repository.AnsiCharset, err
|
|
}
|
|
|
|
log.Debug("Detected encoding: %s", topResult.Charset)
|
|
return topResult.Charset, err
|
|
}
|