toutf8 improved & add max git diff lines

This commit is contained in:
lunnyxiao 2014-09-17 12:03:03 +08:00
parent efb68a0a96
commit ed84adb679
7 changed files with 63 additions and 35 deletions

View file

@ -255,3 +255,6 @@ CONN =
[i18n] [i18n]
LANGS = en-US,zh-CN,de-DE LANGS = en-US,zh-CN,de-DE
NAMES = English,简体中文,Deutsch NAMES = English,简体中文,Deutsch
[git]
MAX_GITDIFF_LINES = 10000

View file

@ -70,7 +70,7 @@ func (diff *Diff) NumFiles() int {
const DIFF_HEAD = "diff --git " const DIFF_HEAD = "diff --git "
func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) { func ParsePatch(pid int64, maxlines int, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
scanner := bufio.NewScanner(reader) scanner := bufio.NewScanner(reader)
var ( var (
curFile *DiffFile curFile *DiffFile
@ -79,6 +79,7 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
} }
leftLine, rightLine int leftLine, rightLine int
isTooLong bool
) )
diff := &Diff{Files: make([]*DiffFile, 0)} diff := &Diff{Files: make([]*DiffFile, 0)}
@ -90,18 +91,19 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
continue continue
} }
i = i + 1
// Diff data too large.
if i == 5000 {
log.Warn("Diff data too large")
return &Diff{}, nil
}
if line == "" { if line == "" {
continue continue
} }
i = i + 1
// Diff data too large, we only show the first about maxlines lines
if i == maxlines {
isTooLong = true
log.Warn("Diff data too large")
//return &Diff{}, nil
}
switch { switch {
case line[0] == ' ': case line[0] == ' ':
diffLine := &DiffLine{Type: DIFF_LINE_PLAIN, Content: line, LeftIdx: leftLine, RightIdx: rightLine} diffLine := &DiffLine{Type: DIFF_LINE_PLAIN, Content: line, LeftIdx: leftLine, RightIdx: rightLine}
@ -110,6 +112,10 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
curSection.Lines = append(curSection.Lines, diffLine) curSection.Lines = append(curSection.Lines, diffLine)
continue continue
case line[0] == '@': case line[0] == '@':
if isTooLong {
return diff, nil
}
curSection = &DiffSection{} curSection = &DiffSection{}
curFile.Sections = append(curFile.Sections, curSection) curFile.Sections = append(curFile.Sections, curSection)
ss := strings.Split(line, "@@") ss := strings.Split(line, "@@")
@ -143,6 +149,10 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
// Get new file. // Get new file.
if strings.HasPrefix(line, DIFF_HEAD) { if strings.HasPrefix(line, DIFF_HEAD) {
if isTooLong {
return diff, nil
}
fs := strings.Split(line[len(DIFF_HEAD):], " ") fs := strings.Split(line[len(DIFF_HEAD):], " ")
a := fs[0] a := fs[0]
@ -174,7 +184,7 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
return diff, nil return diff, nil
} }
func GetDiffRange(repoPath, beforeCommitId string, afterCommitId string) (*Diff, error) { func GetDiffRange(repoPath, beforeCommitId string, afterCommitId string, maxlines int) (*Diff, error) {
repo, err := git.OpenRepository(repoPath) repo, err := git.OpenRepository(repoPath)
if err != nil { if err != nil {
return nil, err return nil, err
@ -228,9 +238,9 @@ func GetDiffRange(repoPath, beforeCommitId string, afterCommitId string) (*Diff,
} }
}() }()
return ParsePatch(pid, cmd, rd) return ParsePatch(pid, maxlines, cmd, rd)
} }
func GetDiffCommit(repoPath, commitId string) (*Diff, error) { func GetDiffCommit(repoPath, commitId string, maxlines int) (*Diff, error) {
return GetDiffRange(repoPath, "", commitId) return GetDiffRange(repoPath, "", commitId, maxlines)
} }

View file

@ -8,13 +8,16 @@ import (
"bytes" "bytes"
"container/list" "container/list"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"html/template" "html/template"
"runtime" "runtime"
"strings" "strings"
"time" "time"
"code.google.com/p/mahonia"
"github.com/gogits/gogs/modules/setting" "github.com/gogits/gogs/modules/setting"
"github.com/saintfish/chardet"
) )
func Str2html(raw string) template.HTML { func Str2html(raw string) template.HTML {
@ -45,6 +48,29 @@ func ShortSha(sha1 string) string {
return sha1 return sha1
} }
func ToUtf8WithErr(content []byte) (error, string) {
detector := chardet.NewTextDetector()
result, err := detector.DetectBest(content)
if err != nil {
return err, ""
}
if result.Charset == "utf8" {
return nil, string(content)
}
decoder := mahonia.NewDecoder(result.Charset)
if decoder != nil {
return nil, decoder.ConvertString(string(content))
}
return errors.New("unknow char decoder"), string(content)
}
func ToUtf8(content string) string {
_, res := ToUtf8WithErr([]byte(content))
return res
}
var mailDomains = map[string]string{ var mailDomains = map[string]string{
"gmail.com": "gmail.com", "gmail.com": "gmail.com",
} }
@ -103,6 +129,7 @@ var TemplateFuncs template.FuncMap = map[string]interface{}{
"ActionContent2Commits": ActionContent2Commits, "ActionContent2Commits": ActionContent2Commits,
"Oauth2Icon": Oauth2Icon, "Oauth2Icon": Oauth2Icon,
"Oauth2Name": Oauth2Name, "Oauth2Name": Oauth2Name,
"ToUtf8": ToUtf8,
} }
type Actioner interface { type Actioner interface {

View file

@ -64,6 +64,7 @@ var (
// Picture settings. // Picture settings.
PictureService string PictureService string
DisableGravatar bool DisableGravatar bool
MaxGitDiffLines int
// Log settings. // Log settings.
LogRootPath string LogRootPath string
@ -241,6 +242,8 @@ func NewConfigContext() {
[]string{"server"}) []string{"server"})
DisableGravatar = Cfg.MustBool("picture", "DISABLE_GRAVATAR") DisableGravatar = Cfg.MustBool("picture", "DISABLE_GRAVATAR")
MaxGitDiffLines = Cfg.MustInt("git", "MAX_GITDIFF_LINES", 5000)
Langs = Cfg.MustValueArray("i18n", "LANGS", ",") Langs = Cfg.MustValueArray("i18n", "LANGS", ",")
Names = Cfg.MustValueArray("i18n", "NAMES", ",") Names = Cfg.MustValueArray("i18n", "NAMES", ",")
} }

View file

@ -12,6 +12,7 @@ import (
"github.com/gogits/gogs/models" "github.com/gogits/gogs/models"
"github.com/gogits/gogs/modules/base" "github.com/gogits/gogs/modules/base"
"github.com/gogits/gogs/modules/middleware" "github.com/gogits/gogs/modules/middleware"
"github.com/gogits/gogs/modules/setting"
) )
const ( const (
@ -114,7 +115,8 @@ func Diff(ctx *middleware.Context) {
commit := ctx.Repo.Commit commit := ctx.Repo.Commit
diff, err := models.GetDiffCommit(models.RepoPath(userName, repoName), commitId) diff, err := models.GetDiffCommit(models.RepoPath(userName, repoName),
commitId, setting.MaxGitDiffLines)
if err != nil { if err != nil {
ctx.Handle(404, "GetDiffCommit", err) ctx.Handle(404, "GetDiffCommit", err)
return return
@ -176,7 +178,8 @@ func CompareDiff(ctx *middleware.Context) {
return return
} }
diff, err := models.GetDiffRange(models.RepoPath(userName, repoName), beforeCommitId, afterCommitId) diff, err := models.GetDiffRange(models.RepoPath(userName, repoName), beforeCommitId,
afterCommitId, setting.MaxGitDiffLines)
if err != nil { if err != nil {
ctx.Handle(404, "GetDiffRange", err) ctx.Handle(404, "GetDiffRange", err)
return return

View file

@ -11,12 +11,9 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
"github.com/saintfish/chardet"
"github.com/gogits/gogs/modules/base" "github.com/gogits/gogs/modules/base"
"github.com/gogits/gogs/modules/git" "github.com/gogits/gogs/modules/git"
"github.com/gogits/gogs/modules/log" "github.com/gogits/gogs/modules/log"
"github.com/gogits/gogs/modules/mahonia"
"github.com/gogits/gogs/modules/middleware" "github.com/gogits/gogs/modules/middleware"
) )
@ -24,21 +21,6 @@ const (
HOME base.TplName = "repo/home" HOME base.TplName = "repo/home"
) )
func toUtf8(content []byte) (error, string) {
detector := chardet.NewTextDetector()
result, err := detector.DetectBest(content)
if err != nil {
return err, ""
}
if result.Charset == "utf8" {
return nil, string(content)
}
decoder := mahonia.NewDecoder(result.Charset)
return nil, decoder.ConvertString(string(content))
}
func Home(ctx *middleware.Context) { func Home(ctx *middleware.Context) {
ctx.Data["Title"] = ctx.Repo.Repository.Name ctx.Data["Title"] = ctx.Repo.Repository.Name
@ -117,7 +99,7 @@ func Home(ctx *middleware.Context) {
if readmeExist { if readmeExist {
ctx.Data["FileContent"] = string(base.RenderMarkdown(buf, "")) ctx.Data["FileContent"] = string(base.RenderMarkdown(buf, ""))
} else { } else {
if err, content := toUtf8(buf); err != nil { if err, content := base.ToUtf8WithErr(buf); err != nil {
if err != nil { if err != nil {
log.Error(4, "Convert content encoding: %s", err) log.Error(4, "Convert content encoding: %s", err)
} }

View file

@ -105,7 +105,7 @@
<span rel="L1">{{if .RightIdx}}{{.RightIdx}}{{end}}</span> <span rel="L1">{{if .RightIdx}}{{.RightIdx}}{{end}}</span>
</td> </td>
<td class="lines-code"> <td class="lines-code">
<pre>{{.Content}}</pre> <pre>{{ToUtf8 .Content}}</pre>
</td> </td>
</tr> </tr>
{{end}} {{end}}