mirror of
https://github.com/42wim/matterbridge.git
synced 2024-11-22 12:09:30 +01:00
Merge branch 'master' into discord-naitive-upload
This commit is contained in:
Joseph Mansy
GitHub
commit
e68ddcf5c1
22
.github/workflows/development.yml
vendored
22
.github/workflows/development.yml
vendored
@ -5,28 +5,28 @@ jobs:
|
||||
name: golangci-lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 20
|
||||
- name: Run golangci-lint
|
||||
uses: golangci/golangci-lint-action@v2
|
||||
uses: golangci/golangci-lint-action@v3
|
||||
with:
|
||||
version: latest
|
||||
args: "-v --new-from-rev HEAD~5"
|
||||
test-build-upload:
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.18.x]
|
||||
go-version: [1.20.x]
|
||||
platform: [ubuntu-latest]
|
||||
runs-on: ${{ matrix.platform }}
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v2
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
stable: false
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Test
|
||||
@ -39,20 +39,20 @@ jobs:
|
||||
CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags "-s -X github.com/42wim/matterbridge/version.GitHash=$(git log --pretty=format:'%h' -n 1)" -o output/win/matterbridge-$VERSION-windows-amd64.exe
|
||||
CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags "-s -X github.com/42wim/matterbridge/version.GitHash=$(git log --pretty=format:'%h' -n 1)" -o output/mac/matterbridge-$VERSION-darwin-amd64
|
||||
- name: Upload linux 64-bit
|
||||
if: startsWith(matrix.go-version,'1.18')
|
||||
uses: actions/upload-artifact@v2
|
||||
if: startsWith(matrix.go-version,'1.20')
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: matterbridge-linux-64bit
|
||||
path: output/lin
|
||||
- name: Upload windows 64-bit
|
||||
if: startsWith(matrix.go-version,'1.18')
|
||||
uses: actions/upload-artifact@v2
|
||||
if: startsWith(matrix.go-version,'1.20')
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: matterbridge-windows-64bit
|
||||
path: output/win
|
||||
- name: Upload darwin 64-bit
|
||||
if: startsWith(matrix.go-version,'1.18')
|
||||
uses: actions/upload-artifact@v2
|
||||
if: startsWith(matrix.go-version,'1.20')
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: matterbridge-darwin-64bit
|
||||
path: output/mac
|
||||
|
||||
@ -114,6 +114,7 @@ And more...
|
||||
|
||||
### 3rd party via matterbridge api
|
||||
|
||||
- [Delta Chat](https://github.com/deltachat-bot/matterdelta)
|
||||
- [Discourse](https://github.com/DeclanHoare/matterbabble)
|
||||
- [Facebook messenger](https://github.com/powerjungle/fbridge-asyncio)
|
||||
- [Facebook messenger](https://github.com/VictorNine/fbridge)
|
||||
|
||||
@ -1,17 +1,20 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gopkg.in/olahol/melody.v1"
|
||||
"github.com/olahol/melody"
|
||||
|
||||
"github.com/42wim/matterbridge/bridge"
|
||||
"github.com/42wim/matterbridge/bridge/config"
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/labstack/echo/v4/middleware"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
ring "github.com/zfjagann/golang-ring"
|
||||
)
|
||||
|
||||
@ -137,6 +140,36 @@ func (b *API) handlePostMessage(c echo.Context) error {
|
||||
message.Account = b.Account
|
||||
message.ID = ""
|
||||
message.Timestamp = time.Now()
|
||||
|
||||
var (
|
||||
fm map[string]interface{}
|
||||
ds string
|
||||
ok bool
|
||||
)
|
||||
|
||||
for i, f := range message.Extra["file"] {
|
||||
fi := config.FileInfo{}
|
||||
if fm, ok = f.(map[string]interface{}); !ok {
|
||||
return echo.NewHTTPError(http.StatusInternalServerError, "invalid format for extra")
|
||||
}
|
||||
err := mapstructure.Decode(fm, &fi)
|
||||
if err != nil {
|
||||
if !strings.Contains(err.Error(), "got string") {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// mapstructure doesn't decode base64 into []byte, so it must be done manually for fi.Data
|
||||
if ds, ok = fm["Data"].(string); !ok {
|
||||
return echo.NewHTTPError(http.StatusInternalServerError, "invalid format for data")
|
||||
}
|
||||
|
||||
data, err := base64.StdEncoding.DecodeString(ds)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fi.Data = &data
|
||||
message.Extra["file"][i] = fi
|
||||
}
|
||||
b.Log.Debugf("Sending message from %s on %s to gateway", message.Username, "api")
|
||||
b.Remote <- message
|
||||
return c.JSON(http.StatusOK, message)
|
||||
@ -166,15 +199,20 @@ func (b *API) handleStream(c echo.Context) error {
|
||||
}
|
||||
c.Response().Flush()
|
||||
for {
|
||||
select {
|
||||
// TODO: this causes issues, messages should be broadcasted to all connected clients
|
||||
msg := b.Messages.Dequeue()
|
||||
if msg != nil {
|
||||
if err := json.NewEncoder(c.Response()).Encode(msg); err != nil {
|
||||
return err
|
||||
default:
|
||||
msg := b.Messages.Dequeue()
|
||||
if msg != nil {
|
||||
if err := json.NewEncoder(c.Response()).Encode(msg); err != nil {
|
||||
return err
|
||||
}
|
||||
c.Response().Flush()
|
||||
}
|
||||
c.Response().Flush()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
case <-c.Request().Context().Done():
|
||||
return nil
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -81,17 +81,6 @@ func (b *Bdiscord) Connect() error {
|
||||
return err
|
||||
}
|
||||
b.Log.Info("Connection succeeded")
|
||||
b.c.AddHandler(b.messageCreate)
|
||||
b.c.AddHandler(b.messageTyping)
|
||||
b.c.AddHandler(b.messageUpdate)
|
||||
b.c.AddHandler(b.messageDelete)
|
||||
b.c.AddHandler(b.messageDeleteBulk)
|
||||
b.c.AddHandler(b.memberAdd)
|
||||
b.c.AddHandler(b.memberRemove)
|
||||
b.c.AddHandler(b.memberUpdate)
|
||||
if b.GetInt("debuglevel") == 1 {
|
||||
b.c.AddHandler(b.messageEvent)
|
||||
}
|
||||
// Add privileged intent for guild member tracking. This is needed to track nicks
|
||||
// for display names and @mention translation
|
||||
b.c.Identify.Intents = discordgo.MakeIntent(discordgo.IntentsAllWithoutPrivileged |
|
||||
@ -233,6 +222,19 @@ func (b *Bdiscord) Connect() error {
|
||||
b.nickMemberMap[member.Nick] = member
|
||||
}
|
||||
}
|
||||
|
||||
b.c.AddHandler(b.messageCreate)
|
||||
b.c.AddHandler(b.messageTyping)
|
||||
b.c.AddHandler(b.messageUpdate)
|
||||
b.c.AddHandler(b.messageDelete)
|
||||
b.c.AddHandler(b.messageDeleteBulk)
|
||||
b.c.AddHandler(b.memberAdd)
|
||||
b.c.AddHandler(b.memberRemove)
|
||||
b.c.AddHandler(b.memberUpdate)
|
||||
if b.GetInt("debuglevel") == 1 {
|
||||
b.c.AddHandler(b.messageEvent)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@ -47,8 +47,9 @@ func (b *Bdiscord) maybeGetLocalAvatar(msg *config.Message) string {
|
||||
// Returns messageID and error.
|
||||
func (b *Bdiscord) webhookSend(msg *config.Message, channelID string) (*discordgo.Message, error) {
|
||||
var (
|
||||
res *discordgo.Message
|
||||
err error
|
||||
res *discordgo.Message
|
||||
res2 *discordgo.Message
|
||||
err error
|
||||
)
|
||||
|
||||
// If avatar is unset, mutate the message to include the local avatar (but only if settings say we should do this)
|
||||
@ -84,7 +85,7 @@ func (b *Bdiscord) webhookSend(msg *config.Message, channelID string) (*discordg
|
||||
}
|
||||
content := fi.Comment
|
||||
|
||||
_, e2 := b.transmitter.Send(
|
||||
res2, err = b.transmitter.Send(
|
||||
channelID,
|
||||
&discordgo.WebhookParams{
|
||||
Username: msg.Username,
|
||||
@ -94,11 +95,16 @@ func (b *Bdiscord) webhookSend(msg *config.Message, channelID string) (*discordg
|
||||
AllowedMentions: b.getAllowedMentions(),
|
||||
},
|
||||
)
|
||||
if e2 != nil {
|
||||
b.Log.Errorf("Could not send file %#v for message %#v: %s", file, msg, e2)
|
||||
if err != nil {
|
||||
b.Log.Errorf("Could not send file %#v for message %#v: %s", file, msg, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if msg.Text == "" {
|
||||
res = res2
|
||||
}
|
||||
|
||||
return res, err
|
||||
}
|
||||
|
||||
|
||||
@ -161,7 +161,7 @@ func (b *Bmattermost) Send(msg config.Message) (string, error) {
|
||||
if err != nil {
|
||||
b.Log.Errorf("getting post %s failed: %s", msg.ParentID, err)
|
||||
}
|
||||
if post.RootId != "" {
|
||||
if post != nil && post.RootId != "" {
|
||||
msg.ParentID = post.RootId
|
||||
}
|
||||
}
|
||||
|
||||
@ -42,7 +42,14 @@ func (b *Bmumble) handleTextMessage(event *gumble.TextMessageEvent) {
|
||||
if part.Image == nil {
|
||||
rmsg.Text = part.Text
|
||||
} else {
|
||||
fname := b.Account + "_" + strconv.FormatInt(now.UnixNano(), 10) + "_" + strconv.Itoa(i) + part.FileExtension
|
||||
fileExt := part.FileExtension
|
||||
if fileExt == ".jfif" {
|
||||
fileExt = ".jpg"
|
||||
}
|
||||
if fileExt == ".jpe" {
|
||||
fileExt = ".jpg"
|
||||
}
|
||||
fname := b.Account + "_" + strconv.FormatInt(now.UnixNano(), 10) + "_" + strconv.Itoa(i) + fileExt
|
||||
rmsg.Extra = make(map[string][]interface{})
|
||||
if err = helper.HandleDownloadSize(b.Log, &rmsg, fname, int64(len(part.Image)), b.General); err != nil {
|
||||
b.Log.WithError(err).Warn("not including image in message")
|
||||
@ -62,7 +69,6 @@ func (b *Bmumble) handleConnect(event *gumble.ConnectEvent) {
|
||||
}
|
||||
// No need to talk or listen
|
||||
event.Client.Self.SetSelfDeafened(true)
|
||||
event.Client.Self.SetSelfMuted(true)
|
||||
// if the Channel variable is set, this is a reconnect -> rejoin channel
|
||||
if b.Channel != nil {
|
||||
if err := b.doJoin(event.Client, *b.Channel); err != nil {
|
||||
|
||||
@ -250,7 +250,12 @@ func (b *Bmumble) processMessage(msg *config.Message) {
|
||||
// If there is a maximum message length, split and truncate the lines
|
||||
var msgLines []string
|
||||
if maxLength := b.serverConfig.MaximumMessageLength; maxLength != nil {
|
||||
msgLines = helper.GetSubLines(msg.Text, *maxLength-len(msg.Username), b.GetString("MessageClipped"))
|
||||
if *maxLength != 0 { // Some servers will have unlimited message lengths.
|
||||
// Not doing this makes underflows happen.
|
||||
msgLines = helper.GetSubLines(msg.Text, *maxLength-len(msg.Username), b.GetString("MessageClipped"))
|
||||
} else {
|
||||
msgLines = helper.GetSubLines(msg.Text, 0, b.GetString("MessageClipped"))
|
||||
}
|
||||
} else {
|
||||
msgLines = helper.GetSubLines(msg.Text, 0, b.GetString("MessageClipped"))
|
||||
}
|
||||
|
||||
@ -11,7 +11,7 @@ import (
|
||||
"github.com/42wim/matterbridge/bridge/config"
|
||||
"github.com/42wim/matterbridge/bridge/helper"
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api/v5"
|
||||
tgbotapi "github.com/matterbridge/telegram-bot-api/v6"
|
||||
)
|
||||
|
||||
func (b *Btelegram) handleUpdate(rmsg *config.Message, message, posted, edited *tgbotapi.Message) *tgbotapi.Message {
|
||||
@ -20,6 +20,11 @@ func (b *Btelegram) handleUpdate(rmsg *config.Message, message, posted, edited *
|
||||
if posted.Text == "/chatId" {
|
||||
chatID := strconv.FormatInt(posted.Chat.ID, 10)
|
||||
|
||||
// Handle chat topics
|
||||
if posted.IsTopicMessage {
|
||||
chatID = chatID + "/" + strconv.Itoa(posted.MessageThreadID)
|
||||
}
|
||||
|
||||
_, err := b.Send(config.Message{
|
||||
Channel: chatID,
|
||||
Text: fmt.Sprintf("ID of this chat: %s", chatID),
|
||||
@ -91,7 +96,8 @@ func (b *Btelegram) handleForwarded(rmsg *config.Message, message *tgbotapi.Mess
|
||||
|
||||
// handleQuoting handles quoting of previous messages
|
||||
func (b *Btelegram) handleQuoting(rmsg *config.Message, message *tgbotapi.Message) {
|
||||
if message.ReplyToMessage != nil {
|
||||
// Used to check if the message was a reply to the root topic
|
||||
if message.ReplyToMessage != nil && !(message.ReplyToMessage.MessageID == message.MessageThreadID) { //nolint:nestif
|
||||
usernameReply := ""
|
||||
if message.ReplyToMessage.From != nil {
|
||||
if b.GetBool("UseFirstName") {
|
||||
@ -128,7 +134,9 @@ func (b *Btelegram) handleUsername(rmsg *config.Message, message *tgbotapi.Messa
|
||||
rmsg.Username = message.From.FirstName
|
||||
}
|
||||
if b.GetBool("UseFullName") {
|
||||
rmsg.Username = message.From.FirstName + " " + message.From.LastName
|
||||
if message.From.FirstName != "" && message.From.LastName != "" {
|
||||
rmsg.Username = message.From.FirstName + " " + message.From.LastName
|
||||
}
|
||||
}
|
||||
if rmsg.Username == "" {
|
||||
rmsg.Username = message.From.UserName
|
||||
@ -148,7 +156,9 @@ func (b *Btelegram) handleUsername(rmsg *config.Message, message *tgbotapi.Messa
|
||||
rmsg.Username = message.SenderChat.FirstName
|
||||
}
|
||||
if b.GetBool("UseFullName") {
|
||||
rmsg.Username = message.SenderChat.FirstName + " " + message.SenderChat.LastName
|
||||
if message.SenderChat.FirstName != "" && message.SenderChat.LastName != "" {
|
||||
rmsg.Username = message.SenderChat.FirstName + " " + message.SenderChat.LastName
|
||||
}
|
||||
}
|
||||
|
||||
if rmsg.Username == "" || rmsg.Username == "Channel_Bot" {
|
||||
@ -164,6 +174,11 @@ func (b *Btelegram) handleUsername(rmsg *config.Message, message *tgbotapi.Messa
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback on author signature (used in "channel" type of chat)
|
||||
if rmsg.Username == "" && message.AuthorSignature != "" {
|
||||
rmsg.Username = message.AuthorSignature
|
||||
}
|
||||
|
||||
// if we really didn't find a username, set it to unknown
|
||||
if rmsg.Username == "" {
|
||||
rmsg.Username = unknownUser
|
||||
@ -202,9 +217,14 @@ func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) {
|
||||
// set the ID's from the channel or group message
|
||||
rmsg.ID = strconv.Itoa(message.MessageID)
|
||||
rmsg.Channel = strconv.FormatInt(message.Chat.ID, 10)
|
||||
if message.MessageThreadID != 0 {
|
||||
rmsg.Channel += "/" + strconv.Itoa(message.MessageThreadID)
|
||||
}
|
||||
|
||||
// preserve threading from telegram reply
|
||||
if message.ReplyToMessage != nil {
|
||||
if message.ReplyToMessage != nil &&
|
||||
// Used to check if the message was a reply to the root topic
|
||||
!(message.ReplyToMessage.MessageID == message.MessageThreadID) {
|
||||
rmsg.ParentID = strconv.Itoa(message.ReplyToMessage.MessageID)
|
||||
}
|
||||
|
||||
@ -317,12 +337,12 @@ func (b *Btelegram) maybeConvertWebp(name *string, data *[]byte) {
|
||||
|
||||
// handleDownloadFile handles file download
|
||||
func (b *Btelegram) handleDownload(rmsg *config.Message, message *tgbotapi.Message) error {
|
||||
size := 0
|
||||
size := int64(0)
|
||||
var url, name, text string
|
||||
switch {
|
||||
case message.Sticker != nil:
|
||||
text, name, url = b.getDownloadInfo(message.Sticker.FileID, ".webp", true)
|
||||
size = message.Sticker.FileSize
|
||||
size = int64(message.Sticker.FileSize)
|
||||
case message.Voice != nil:
|
||||
text, name, url = b.getDownloadInfo(message.Voice.FileID, ".ogg", true)
|
||||
size = message.Voice.FileSize
|
||||
@ -339,7 +359,7 @@ func (b *Btelegram) handleDownload(rmsg *config.Message, message *tgbotapi.Messa
|
||||
text = " " + message.Document.FileName + " : " + url
|
||||
case message.Photo != nil:
|
||||
photos := message.Photo
|
||||
size = photos[len(photos)-1].FileSize
|
||||
size = int64(photos[len(photos)-1].FileSize)
|
||||
text, name, url = b.getDownloadInfo(photos[len(photos)-1].FileID, "", true)
|
||||
}
|
||||
|
||||
@ -443,7 +463,7 @@ func (b *Btelegram) handleEdit(msg *config.Message, chatid int64) (string, error
|
||||
}
|
||||
|
||||
// handleUploadFile handles native upload of files
|
||||
func (b *Btelegram) handleUploadFile(msg *config.Message, chatid int64, parentID int) (string, error) {
|
||||
func (b *Btelegram) handleUploadFile(msg *config.Message, chatid int64, threadid int, parentID int) (string, error) {
|
||||
var media []interface{}
|
||||
for _, f := range msg.Extra["file"] {
|
||||
fi := f.(config.FileInfo)
|
||||
@ -493,7 +513,7 @@ func (b *Btelegram) handleUploadFile(msg *config.Message, chatid int64, parentID
|
||||
}
|
||||
}
|
||||
|
||||
return b.sendMediaFiles(msg, chatid, parentID, media)
|
||||
return b.sendMediaFiles(msg, chatid, threadid, parentID, media)
|
||||
}
|
||||
|
||||
func (b *Btelegram) handleQuote(message, quoteNick, quoteMessage string) string {
|
||||
|
||||
@ -10,7 +10,7 @@ import (
|
||||
"github.com/42wim/matterbridge/bridge"
|
||||
"github.com/42wim/matterbridge/bridge/config"
|
||||
"github.com/42wim/matterbridge/bridge/helper"
|
||||
tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api/v5"
|
||||
tgbotapi "github.com/matterbridge/telegram-bot-api/v6"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -86,11 +86,41 @@ func TGGetParseMode(b *Btelegram, username string, text string) (textout string,
|
||||
return textout, parsemode
|
||||
}
|
||||
|
||||
func (b *Btelegram) getIds(channel string) (int64, int, error) {
|
||||
var chatid int64
|
||||
topicid := 0
|
||||
|
||||
// get the chatid
|
||||
if strings.Contains(channel, "/") { //nolint:nestif
|
||||
s := strings.Split(channel, "/")
|
||||
if len(s) < 2 {
|
||||
b.Log.Errorf("Invalid channel format: %#v\n", channel)
|
||||
return 0, 0, nil
|
||||
}
|
||||
id, err := strconv.ParseInt(s[0], 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
chatid = id
|
||||
tid, err := strconv.Atoi(s[1])
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
topicid = tid
|
||||
} else {
|
||||
id, err := strconv.ParseInt(channel, 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
chatid = id
|
||||
}
|
||||
return chatid, topicid, nil
|
||||
}
|
||||
|
||||
func (b *Btelegram) Send(msg config.Message) (string, error) {
|
||||
b.Log.Debugf("=> Receiving %#v", msg)
|
||||
|
||||
// get the chatid
|
||||
chatid, err := strconv.ParseInt(msg.Channel, 10, 64)
|
||||
chatid, topicid, err := b.getIds(msg.Channel)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@ -123,13 +153,13 @@ func (b *Btelegram) Send(msg config.Message) (string, error) {
|
||||
// Upload a file if it exists
|
||||
if msg.Extra != nil {
|
||||
for _, rmsg := range helper.HandleExtra(&msg, b.General) {
|
||||
if _, msgErr := b.sendMessage(chatid, rmsg.Username, rmsg.Text, parentID); msgErr != nil {
|
||||
if _, msgErr := b.sendMessage(chatid, topicid, rmsg.Username, rmsg.Text, parentID); msgErr != nil {
|
||||
b.Log.Errorf("sendMessage failed: %s", msgErr)
|
||||
}
|
||||
}
|
||||
// check if we have files to upload (from slack, telegram or mattermost)
|
||||
if len(msg.Extra["file"]) > 0 {
|
||||
return b.handleUploadFile(&msg, chatid, parentID)
|
||||
return b.handleUploadFile(&msg, chatid, topicid, parentID)
|
||||
}
|
||||
}
|
||||
|
||||
@ -143,7 +173,7 @@ func (b *Btelegram) Send(msg config.Message) (string, error) {
|
||||
// Ignore empty text field needs for prevent double messages from whatsapp to telegram
|
||||
// when sending media with text caption
|
||||
if msg.Text != "" {
|
||||
return b.sendMessage(chatid, msg.Username, msg.Text, parentID)
|
||||
return b.sendMessage(chatid, topicid, msg.Username, msg.Text, parentID)
|
||||
}
|
||||
|
||||
return "", nil
|
||||
@ -157,9 +187,12 @@ func (b *Btelegram) getFileDirectURL(id string) string {
|
||||
return res
|
||||
}
|
||||
|
||||
func (b *Btelegram) sendMessage(chatid int64, username, text string, parentID int) (string, error) {
|
||||
func (b *Btelegram) sendMessage(chatid int64, topicid int, username, text string, parentID int) (string, error) {
|
||||
m := tgbotapi.NewMessage(chatid, "")
|
||||
m.Text, m.ParseMode = TGGetParseMode(b, username, text)
|
||||
if topicid != 0 {
|
||||
m.BaseChat.MessageThreadID = topicid
|
||||
}
|
||||
m.ReplyToMessageID = parentID
|
||||
m.DisableWebPagePreview = b.GetBool("DisableWebPagePreview")
|
||||
|
||||
@ -171,11 +204,19 @@ func (b *Btelegram) sendMessage(chatid int64, username, text string, parentID in
|
||||
}
|
||||
|
||||
// sendMediaFiles native upload media files via media group
|
||||
func (b *Btelegram) sendMediaFiles(msg *config.Message, chatid int64, parentID int, media []interface{}) (string, error) {
|
||||
func (b *Btelegram) sendMediaFiles(msg *config.Message, chatid int64, threadid int, parentID int, media []interface{}) (string, error) {
|
||||
if len(media) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
mg := tgbotapi.MediaGroupConfig{ChatID: chatid, ChannelUsername: msg.Username, Media: media, ReplyToMessageID: parentID}
|
||||
mg := tgbotapi.MediaGroupConfig{
|
||||
BaseChat: tgbotapi.BaseChat{
|
||||
ChatID: chatid,
|
||||
MessageThreadID: threadid,
|
||||
ChannelUsername: msg.Username,
|
||||
ReplyToMessageID: parentID,
|
||||
},
|
||||
Media: media,
|
||||
}
|
||||
messages, err := b.c.SendMediaGroup(mg)
|
||||
if err != nil {
|
||||
return "", err
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
//go:build whatsappmulti
|
||||
// +build whatsappmulti
|
||||
|
||||
package bwhatsapp
|
||||
@ -20,9 +21,82 @@ func (b *Bwhatsapp) eventHandler(evt interface{}) {
|
||||
switch e := evt.(type) {
|
||||
case *events.Message:
|
||||
b.handleMessage(e)
|
||||
case *events.GroupInfo:
|
||||
b.handleGroupInfo(e)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Bwhatsapp) handleGroupInfo(event *events.GroupInfo) {
|
||||
|
||||
b.Log.Debugf("Receiving event %#v", event)
|
||||
|
||||
switch {
|
||||
case event.Join != nil:
|
||||
b.handleUserJoin(event)
|
||||
case event.Leave != nil:
|
||||
b.handleUserLeave(event)
|
||||
case event.Topic != nil:
|
||||
b.handleTopicChange(event)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Bwhatsapp) handleUserJoin(event *events.GroupInfo) {
|
||||
for _, joinedJid := range event.Join {
|
||||
senderName := b.getSenderNameFromJID(joinedJid)
|
||||
|
||||
rmsg := config.Message{
|
||||
UserID: joinedJid.String(),
|
||||
Username: senderName,
|
||||
Channel: event.JID.String(),
|
||||
Account: b.Account,
|
||||
Protocol: b.Protocol,
|
||||
Event: config.EventJoinLeave,
|
||||
Text: "joined chat",
|
||||
}
|
||||
|
||||
b.Remote <- rmsg
|
||||
}
|
||||
}
|
||||
func (b *Bwhatsapp) handleUserLeave(event *events.GroupInfo) {
|
||||
for _, leftJid := range event.Leave {
|
||||
senderName := b.getSenderNameFromJID(leftJid)
|
||||
|
||||
rmsg := config.Message{
|
||||
UserID: leftJid.String(),
|
||||
Username: senderName,
|
||||
Channel: event.JID.String(),
|
||||
Account: b.Account,
|
||||
Protocol: b.Protocol,
|
||||
Event: config.EventJoinLeave,
|
||||
Text: "left chat",
|
||||
}
|
||||
|
||||
b.Remote <- rmsg
|
||||
}
|
||||
}
|
||||
func (b *Bwhatsapp) handleTopicChange(event *events.GroupInfo) {
|
||||
msg := event.Topic
|
||||
senderJid := msg.TopicSetBy
|
||||
senderName := b.getSenderNameFromJID(senderJid)
|
||||
|
||||
text := msg.Topic
|
||||
if text == "" {
|
||||
text = "removed topic"
|
||||
}
|
||||
|
||||
rmsg := config.Message{
|
||||
UserID: senderJid.String(),
|
||||
Username: senderName,
|
||||
Channel: event.JID.String(),
|
||||
Account: b.Account,
|
||||
Protocol: b.Protocol,
|
||||
Event: config.EventTopicChange,
|
||||
Text: "Topic changed: " + text,
|
||||
}
|
||||
|
||||
b.Remote <- rmsg
|
||||
}
|
||||
|
||||
func (b *Bwhatsapp) handleMessage(message *events.Message) {
|
||||
msg := message.Message
|
||||
switch {
|
||||
@ -30,7 +104,7 @@ func (b *Bwhatsapp) handleMessage(message *events.Message) {
|
||||
return
|
||||
}
|
||||
|
||||
b.Log.Infof("Receiving message %#v", msg)
|
||||
b.Log.Debugf("Receiving message %#v", msg)
|
||||
|
||||
switch {
|
||||
case msg.Conversation != nil || msg.ExtendedTextMessage != nil:
|
||||
@ -43,6 +117,8 @@ func (b *Bwhatsapp) handleMessage(message *events.Message) {
|
||||
b.handleDocumentMessage(message)
|
||||
case msg.ImageMessage != nil:
|
||||
b.handleImageMessage(message)
|
||||
case msg.ProtocolMessage != nil && *msg.ProtocolMessage.Type == proto.ProtocolMessage_REVOKE:
|
||||
b.handleDelete(msg.ProtocolMessage)
|
||||
}
|
||||
}
|
||||
|
||||
@ -63,6 +139,10 @@ func (b *Bwhatsapp) handleTextMessage(messageInfo types.MessageInfo, msg *proto.
|
||||
// nolint:nestif
|
||||
if msg.GetExtendedTextMessage() == nil {
|
||||
text = msg.GetConversation()
|
||||
} else if msg.GetExtendedTextMessage().GetContextInfo() == nil {
|
||||
// Handle pure text message with a link preview
|
||||
// A pure text message with a link preview acts as an extended text message but will not contain any context info
|
||||
text = msg.GetExtendedTextMessage().GetText()
|
||||
} else {
|
||||
text = msg.GetExtendedTextMessage().GetText()
|
||||
ci := msg.GetExtendedTextMessage().GetContextInfo()
|
||||
@ -85,6 +165,12 @@ func (b *Bwhatsapp) handleTextMessage(messageInfo types.MessageInfo, msg *proto.
|
||||
}
|
||||
}
|
||||
|
||||
parentID := ""
|
||||
if msg.GetExtendedTextMessage() != nil {
|
||||
ci := msg.GetExtendedTextMessage().GetContextInfo()
|
||||
parentID = getParentIdFromCtx(ci)
|
||||
}
|
||||
|
||||
rmsg := config.Message{
|
||||
UserID: senderJID.String(),
|
||||
Username: senderName,
|
||||
@ -93,8 +179,8 @@ func (b *Bwhatsapp) handleTextMessage(messageInfo types.MessageInfo, msg *proto.
|
||||
Account: b.Account,
|
||||
Protocol: b.Protocol,
|
||||
Extra: make(map[string][]interface{}),
|
||||
// ParentID: TODO, // TODO handle thread replies // map from Info.QuotedMessageID string
|
||||
ID: messageInfo.ID,
|
||||
ID: getMessageIdFormat(senderJID, messageInfo.ID),
|
||||
ParentID: parentID,
|
||||
}
|
||||
|
||||
if avatarURL, exists := b.userAvatars[senderJID.String()]; exists {
|
||||
@ -126,7 +212,8 @@ func (b *Bwhatsapp) handleImageMessage(msg *events.Message) {
|
||||
Account: b.Account,
|
||||
Protocol: b.Protocol,
|
||||
Extra: make(map[string][]interface{}),
|
||||
ID: msg.Info.ID,
|
||||
ID: getMessageIdFormat(senderJID, msg.Info.ID),
|
||||
ParentID: getParentIdFromCtx(ci),
|
||||
}
|
||||
|
||||
if avatarURL, exists := b.userAvatars[senderJID.String()]; exists {
|
||||
@ -189,7 +276,8 @@ func (b *Bwhatsapp) handleVideoMessage(msg *events.Message) {
|
||||
Account: b.Account,
|
||||
Protocol: b.Protocol,
|
||||
Extra: make(map[string][]interface{}),
|
||||
ID: msg.Info.ID,
|
||||
ID: getMessageIdFormat(senderJID, msg.Info.ID),
|
||||
ParentID: getParentIdFromCtx(ci),
|
||||
}
|
||||
|
||||
if avatarURL, exists := b.userAvatars[senderJID.String()]; exists {
|
||||
@ -207,7 +295,16 @@ func (b *Bwhatsapp) handleVideoMessage(msg *events.Message) {
|
||||
fileExt = append(fileExt, ".mp4")
|
||||
}
|
||||
|
||||
filename := fmt.Sprintf("%v%v", msg.Info.ID, fileExt[0])
|
||||
// Prefer .mp4 extension, otherwise fallback to first index
|
||||
fileExtIndex := 0
|
||||
for i, n := range fileExt {
|
||||
if ".mp4" == n {
|
||||
fileExtIndex = i
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
filename := fmt.Sprintf("%v%v", msg.Info.ID, fileExt[fileExtIndex])
|
||||
|
||||
b.Log.Debugf("Trying to download %s with size %#v and type %s", filename, imsg.GetFileLength(), imsg.GetMimetype())
|
||||
|
||||
@ -238,7 +335,6 @@ func (b *Bwhatsapp) handleAudioMessage(msg *events.Message) {
|
||||
if senderJID == (types.JID{}) && ci.Participant != nil {
|
||||
senderJID = types.NewJID(ci.GetParticipant(), types.DefaultUserServer)
|
||||
}
|
||||
|
||||
rmsg := config.Message{
|
||||
UserID: senderJID.String(),
|
||||
Username: senderName,
|
||||
@ -246,7 +342,8 @@ func (b *Bwhatsapp) handleAudioMessage(msg *events.Message) {
|
||||
Account: b.Account,
|
||||
Protocol: b.Protocol,
|
||||
Extra: make(map[string][]interface{}),
|
||||
ID: msg.Info.ID,
|
||||
ID: getMessageIdFormat(senderJID, msg.Info.ID),
|
||||
ParentID: getParentIdFromCtx(ci),
|
||||
}
|
||||
|
||||
if avatarURL, exists := b.userAvatars[senderJID.String()]; exists {
|
||||
@ -303,7 +400,8 @@ func (b *Bwhatsapp) handleDocumentMessage(msg *events.Message) {
|
||||
Account: b.Account,
|
||||
Protocol: b.Protocol,
|
||||
Extra: make(map[string][]interface{}),
|
||||
ID: msg.Info.ID,
|
||||
ID: getMessageIdFormat(senderJID, msg.Info.ID),
|
||||
ParentID: getParentIdFromCtx(ci),
|
||||
}
|
||||
|
||||
if avatarURL, exists := b.userAvatars[senderJID.String()]; exists {
|
||||
@ -336,3 +434,20 @@ func (b *Bwhatsapp) handleDocumentMessage(msg *events.Message) {
|
||||
|
||||
b.Remote <- rmsg
|
||||
}
|
||||
|
||||
func (b *Bwhatsapp) handleDelete(messageInfo *proto.ProtocolMessage) {
|
||||
sender, _ := types.ParseJID(*messageInfo.Key.Participant)
|
||||
|
||||
rmsg := config.Message{
|
||||
Account: b.Account,
|
||||
Protocol: b.Protocol,
|
||||
ID: getMessageIdFormat(sender, *messageInfo.Key.Id),
|
||||
Event: config.EventMsgDelete,
|
||||
Text: config.EventMsgDelete,
|
||||
Channel: *messageInfo.Key.RemoteJid,
|
||||
}
|
||||
|
||||
b.Log.Debugf("<= Sending message from %s to gateway", b.Account)
|
||||
b.Log.Debugf("<= Message is %#v", rmsg)
|
||||
b.Remote <- rmsg
|
||||
}
|
||||
|
||||
@ -7,7 +7,10 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
goproto "google.golang.org/protobuf/proto"
|
||||
|
||||
"go.mau.fi/whatsmeow"
|
||||
"go.mau.fi/whatsmeow/binary/proto"
|
||||
"go.mau.fi/whatsmeow/store"
|
||||
"go.mau.fi/whatsmeow/store/sqlstore"
|
||||
"go.mau.fi/whatsmeow/types"
|
||||
@ -61,6 +64,29 @@ func (b *Bwhatsapp) getSenderName(info types.MessageInfo) string {
|
||||
return "Someone"
|
||||
}
|
||||
|
||||
func (b *Bwhatsapp) getSenderNameFromJID(senderJid types.JID) string {
|
||||
sender, exists := b.contacts[senderJid]
|
||||
|
||||
if !exists || (sender.FullName == "" && sender.FirstName == "") {
|
||||
b.reloadContacts() // Contacts may need to be reloaded
|
||||
sender, exists = b.contacts[senderJid]
|
||||
}
|
||||
|
||||
if exists && sender.FullName != "" {
|
||||
return sender.FullName
|
||||
}
|
||||
|
||||
if exists && sender.FirstName != "" {
|
||||
return sender.FirstName
|
||||
}
|
||||
|
||||
if sender.PushName != "" {
|
||||
return sender.PushName
|
||||
}
|
||||
|
||||
return "Someone"
|
||||
}
|
||||
|
||||
func (b *Bwhatsapp) getSenderNotify(senderJid types.JID) string {
|
||||
sender, exists := b.contacts[senderJid]
|
||||
|
||||
@ -122,3 +148,63 @@ func (b *Bwhatsapp) getDevice() (*store.Device, error) {
|
||||
|
||||
return device, nil
|
||||
}
|
||||
|
||||
func (b *Bwhatsapp) getNewReplyContext(parentID string) (*proto.ContextInfo, error) {
|
||||
replyInfo, err := b.parseMessageID(parentID)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sender := fmt.Sprintf("%s@%s", replyInfo.Sender.User, replyInfo.Sender.Server)
|
||||
ctx := &proto.ContextInfo{
|
||||
StanzaId: &replyInfo.MessageID,
|
||||
Participant: &sender,
|
||||
QuotedMessage: &proto.Message{Conversation: goproto.String("")},
|
||||
}
|
||||
|
||||
return ctx, nil
|
||||
}
|
||||
|
||||
func (b *Bwhatsapp) parseMessageID(id string) (*Replyable, error) {
|
||||
// No message ID in case action is executed on a message sent before the bridge was started
|
||||
// and then the bridge cache doesn't have this message ID mapped
|
||||
if id == "" {
|
||||
return &Replyable{MessageID: id}, nil
|
||||
}
|
||||
|
||||
replyInfo := strings.Split(id, "/")
|
||||
|
||||
if len(replyInfo) == 2 {
|
||||
sender, err := types.ParseJID(replyInfo[0])
|
||||
|
||||
if err == nil {
|
||||
return &Replyable{
|
||||
MessageID: types.MessageID(replyInfo[1]),
|
||||
Sender: sender,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
err := fmt.Errorf("MessageID does not match format of {senderJID}:{messageID} : \"%s\"", id)
|
||||
|
||||
return &Replyable{MessageID: id}, err
|
||||
}
|
||||
|
||||
func getParentIdFromCtx(ci *proto.ContextInfo) string {
|
||||
if ci != nil && ci.StanzaId != nil {
|
||||
senderJid, err := types.ParseJID(*ci.Participant)
|
||||
|
||||
if err == nil {
|
||||
return getMessageIdFormat(senderJid, *ci.StanzaId)
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func getMessageIdFormat(jid types.JID, messageID string) string {
|
||||
// we're crafting our own JID str as AD JID format messes with how stuff looks on a webclient
|
||||
jidStr := fmt.Sprintf("%s@%s", jid.User, jid.Server)
|
||||
return fmt.Sprintf("%s/%s", jidStr, messageID)
|
||||
}
|
||||
|
||||
@ -35,11 +35,17 @@ const (
|
||||
type Bwhatsapp struct {
|
||||
*bridge.Config
|
||||
|
||||
startedAt time.Time
|
||||
wc *whatsmeow.Client
|
||||
contacts map[types.JID]types.ContactInfo
|
||||
users map[string]types.ContactInfo
|
||||
userAvatars map[string]string
|
||||
startedAt time.Time
|
||||
wc *whatsmeow.Client
|
||||
contacts map[types.JID]types.ContactInfo
|
||||
users map[string]types.ContactInfo
|
||||
userAvatars map[string]string
|
||||
joinedGroups []*types.GroupInfo
|
||||
}
|
||||
|
||||
type Replyable struct {
|
||||
MessageID types.MessageID
|
||||
Sender types.JID
|
||||
}
|
||||
|
||||
// New Create a new WhatsApp bridge. This will be called for each [whatsapp.<server>] entry you have in the config file
|
||||
@ -121,6 +127,11 @@ func (b *Bwhatsapp) Connect() error {
|
||||
return errors.New("failed to get contacts: " + err.Error())
|
||||
}
|
||||
|
||||
b.joinedGroups, err = b.wc.GetJoinedGroups()
|
||||
if err != nil {
|
||||
return errors.New("failed to get list of joined groups: " + err.Error())
|
||||
}
|
||||
|
||||
b.startedAt = time.Now()
|
||||
|
||||
// map all the users
|
||||
@ -166,11 +177,6 @@ func (b *Bwhatsapp) Disconnect() error {
|
||||
func (b *Bwhatsapp) JoinChannel(channel config.ChannelInfo) error {
|
||||
byJid := isGroupJid(channel.Name)
|
||||
|
||||
groups, err := b.wc.GetJoinedGroups()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// verify if we are member of the given group
|
||||
if byJid {
|
||||
gJID, err := types.ParseJID(channel.Name)
|
||||
@ -178,7 +184,7 @@ func (b *Bwhatsapp) JoinChannel(channel config.ChannelInfo) error {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, group := range groups {
|
||||
for _, group := range b.joinedGroups {
|
||||
if group.JID == gJID {
|
||||
return nil
|
||||
}
|
||||
@ -187,7 +193,7 @@ func (b *Bwhatsapp) JoinChannel(channel config.ChannelInfo) error {
|
||||
|
||||
foundGroups := []string{}
|
||||
|
||||
for _, group := range groups {
|
||||
for _, group := range b.joinedGroups {
|
||||
if group.Name == channel.Name {
|
||||
foundGroups = append(foundGroups, group.Name)
|
||||
}
|
||||
@ -196,7 +202,7 @@ func (b *Bwhatsapp) JoinChannel(channel config.ChannelInfo) error {
|
||||
switch len(foundGroups) {
|
||||
case 0:
|
||||
// didn't match any group - print out possibilites
|
||||
for _, group := range groups {
|
||||
for _, group := range b.joinedGroups {
|
||||
b.Log.Infof("%s %s", group.JID, group.Name)
|
||||
}
|
||||
return fmt.Errorf("please specify group's JID from the list above instead of the name '%s'", channel.Name)
|
||||
@ -222,6 +228,10 @@ func (b *Bwhatsapp) PostDocumentMessage(msg config.Message, filetype string) (st
|
||||
|
||||
// Post document message
|
||||
var message proto.Message
|
||||
var ctx *proto.ContextInfo
|
||||
if msg.ParentID != "" {
|
||||
ctx, _ = b.getNewReplyContext(msg.ParentID)
|
||||
}
|
||||
|
||||
message.DocumentMessage = &proto.DocumentMessage{
|
||||
Title: &fi.Name,
|
||||
@ -233,6 +243,8 @@ func (b *Bwhatsapp) PostDocumentMessage(msg config.Message, filetype string) (st
|
||||
FileSha256: resp.FileSHA256,
|
||||
FileLength: goproto.Uint64(resp.FileLength),
|
||||
Url: &resp.URL,
|
||||
DirectPath: &resp.DirectPath,
|
||||
ContextInfo: ctx,
|
||||
}
|
||||
|
||||
b.Log.Debugf("=> Sending %#v as a document", msg)
|
||||
@ -246,8 +258,6 @@ func (b *Bwhatsapp) PostDocumentMessage(msg config.Message, filetype string) (st
|
||||
// Post an image message from the bridge to WhatsApp
|
||||
// Handle, for sure image/jpeg, image/png and image/gif MIME types
|
||||
func (b *Bwhatsapp) PostImageMessage(msg config.Message, filetype string) (string, error) {
|
||||
groupJID, _ := types.ParseJID(msg.Channel)
|
||||
|
||||
fi := msg.Extra["file"][0].(config.FileInfo)
|
||||
|
||||
caption := msg.Username + fi.Comment
|
||||
@ -258,6 +268,10 @@ func (b *Bwhatsapp) PostImageMessage(msg config.Message, filetype string) (strin
|
||||
}
|
||||
|
||||
var message proto.Message
|
||||
var ctx *proto.ContextInfo
|
||||
if msg.ParentID != "" {
|
||||
ctx, _ = b.getNewReplyContext(msg.ParentID)
|
||||
}
|
||||
|
||||
message.ImageMessage = &proto.ImageMessage{
|
||||
Mimetype: &filetype,
|
||||
@ -267,20 +281,17 @@ func (b *Bwhatsapp) PostImageMessage(msg config.Message, filetype string) (strin
|
||||
FileSha256: resp.FileSHA256,
|
||||
FileLength: goproto.Uint64(resp.FileLength),
|
||||
Url: &resp.URL,
|
||||
DirectPath: &resp.DirectPath,
|
||||
ContextInfo: ctx,
|
||||
}
|
||||
|
||||
b.Log.Debugf("=> Sending %#v as an image", msg)
|
||||
|
||||
ID := whatsmeow.GenerateMessageID()
|
||||
_, err = b.wc.SendMessage(context.TODO(), groupJID, &message, whatsmeow.SendRequestExtra{ID: ID})
|
||||
|
||||
return ID, err
|
||||
return b.sendMessage(msg, &message)
|
||||
}
|
||||
|
||||
// Post a video message from the bridge to WhatsApp
|
||||
func (b *Bwhatsapp) PostVideoMessage(msg config.Message, filetype string) (string, error) {
|
||||
groupJID, _ := types.ParseJID(msg.Channel)
|
||||
|
||||
fi := msg.Extra["file"][0].(config.FileInfo)
|
||||
|
||||
caption := msg.Username + fi.Comment
|
||||
@ -291,6 +302,10 @@ func (b *Bwhatsapp) PostVideoMessage(msg config.Message, filetype string) (strin
|
||||
}
|
||||
|
||||
var message proto.Message
|
||||
var ctx *proto.ContextInfo
|
||||
if msg.ParentID != "" {
|
||||
ctx, _ = b.getNewReplyContext(msg.ParentID)
|
||||
}
|
||||
|
||||
message.VideoMessage = &proto.VideoMessage{
|
||||
Mimetype: &filetype,
|
||||
@ -300,14 +315,13 @@ func (b *Bwhatsapp) PostVideoMessage(msg config.Message, filetype string) (strin
|
||||
FileSha256: resp.FileSHA256,
|
||||
FileLength: goproto.Uint64(resp.FileLength),
|
||||
Url: &resp.URL,
|
||||
DirectPath: &resp.DirectPath,
|
||||
ContextInfo: ctx,
|
||||
}
|
||||
|
||||
b.Log.Debugf("=> Sending %#v as a video", msg)
|
||||
|
||||
ID := whatsmeow.GenerateMessageID()
|
||||
_, err = b.wc.SendMessage(context.TODO(), groupJID, &message, whatsmeow.SendRequestExtra{ID: ID})
|
||||
|
||||
return ID, err
|
||||
return b.sendMessage(msg, &message)
|
||||
}
|
||||
|
||||
// Post audio inline
|
||||
@ -322,6 +336,10 @@ func (b *Bwhatsapp) PostAudioMessage(msg config.Message, filetype string) (strin
|
||||
}
|
||||
|
||||
var message proto.Message
|
||||
var ctx *proto.ContextInfo
|
||||
if msg.ParentID != "" {
|
||||
ctx, _ = b.getNewReplyContext(msg.ParentID)
|
||||
}
|
||||
|
||||
message.AudioMessage = &proto.AudioMessage{
|
||||
Mimetype: &filetype,
|
||||
@ -330,12 +348,13 @@ func (b *Bwhatsapp) PostAudioMessage(msg config.Message, filetype string) (strin
|
||||
FileSha256: resp.FileSHA256,
|
||||
FileLength: goproto.Uint64(resp.FileLength),
|
||||
Url: &resp.URL,
|
||||
DirectPath: &resp.DirectPath,
|
||||
ContextInfo: ctx,
|
||||
}
|
||||
|
||||
b.Log.Debugf("=> Sending %#v as audio", msg)
|
||||
|
||||
ID := whatsmeow.GenerateMessageID()
|
||||
_, err = b.wc.SendMessage(context.TODO(), groupJID, &message, whatsmeow.SendRequestExtra{ID: ID})
|
||||
ID, err := b.sendMessage(msg, &message)
|
||||
|
||||
var captionMessage proto.Message
|
||||
caption := msg.Username + fi.Comment + "\u2B06" // the char on the end is upwards arrow emoji
|
||||
@ -351,6 +370,9 @@ func (b *Bwhatsapp) PostAudioMessage(msg config.Message, filetype string) (strin
|
||||
func (b *Bwhatsapp) Send(msg config.Message) (string, error) {
|
||||
groupJID, _ := types.ParseJID(msg.Channel)
|
||||
|
||||
extendedMsgID, _ := b.parseMessageID(msg.ID)
|
||||
msg.ID = extendedMsgID.MessageID
|
||||
|
||||
b.Log.Debugf("=> Receiving %#v", msg)
|
||||
|
||||
// Delete message
|
||||
@ -400,14 +422,35 @@ func (b *Bwhatsapp) Send(msg config.Message) (string, error) {
|
||||
}
|
||||
}
|
||||
|
||||
var message proto.Message
|
||||
text := msg.Username + msg.Text
|
||||
|
||||
var message proto.Message
|
||||
// If we have a parent ID send an extended message
|
||||
if msg.ParentID != "" {
|
||||
replyContext, err := b.getNewReplyContext(msg.ParentID)
|
||||
|
||||
if err == nil {
|
||||
message = proto.Message{
|
||||
ExtendedTextMessage: &proto.ExtendedTextMessage{
|
||||
Text: &text,
|
||||
ContextInfo: replyContext,
|
||||
},
|
||||
}
|
||||
|
||||
return b.sendMessage(msg, &message)
|
||||
}
|
||||
}
|
||||
|
||||
message.Conversation = &text
|
||||
|
||||
ID := whatsmeow.GenerateMessageID()
|
||||
_, err := b.wc.SendMessage(context.TODO(), groupJID, &message, whatsmeow.SendRequestExtra{ID: ID})
|
||||
|
||||
return ID, err
|
||||
return b.sendMessage(msg, &message)
|
||||
}
|
||||
|
||||
func (b *Bwhatsapp) sendMessage(rmsg config.Message, message *proto.Message) (string, error) {
|
||||
groupJID, _ := types.ParseJID(rmsg.Channel)
|
||||
ID := whatsmeow.GenerateMessageID()
|
||||
|
||||
_, err := b.wc.SendMessage(context.Background(), groupJID, message, whatsmeow.SendRequestExtra{ID: ID})
|
||||
|
||||
return getMessageIdFormat(*b.wc.Store.ID, ID), err
|
||||
}
|
||||
|
||||
54
go.mod
54
go.mod
@ -6,34 +6,36 @@ require (
|
||||
github.com/Benau/tgsconverter v0.0.0-20210809170556-99f4a4f6337f
|
||||
github.com/Philipp15b/go-steam v1.0.1-0.20200727090957-6ae9b3c0a560
|
||||
github.com/Rhymen/go-whatsapp v0.1.2-0.20211102134409-31a2e740845c
|
||||
github.com/SevereCloud/vksdk/v2 v2.15.0
|
||||
github.com/SevereCloud/vksdk/v2 v2.16.0
|
||||
github.com/bwmarrin/discordgo v0.27.0
|
||||
github.com/d5/tengo/v2 v2.13.0
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/fsnotify/fsnotify v1.6.0
|
||||
github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1
|
||||
github.com/gomarkdown/markdown v0.0.0-20221013030248-663e2500819c
|
||||
github.com/google/gops v0.3.26
|
||||
github.com/google/gops v0.3.27
|
||||
github.com/gorilla/schema v1.2.0
|
||||
github.com/gorilla/websocket v1.5.0
|
||||
github.com/harmony-development/shibshib v0.0.0-20220101224523-c98059d09cfa
|
||||
github.com/hashicorp/golang-lru v0.6.0
|
||||
github.com/jpillora/backoff v1.0.0
|
||||
github.com/keybase/go-keybase-chat-bot v0.0.0-20221220212439-e48d9abd2c20
|
||||
github.com/kyokomi/emoji/v2 v2.2.11
|
||||
github.com/labstack/echo/v4 v4.10.0
|
||||
github.com/kyokomi/emoji/v2 v2.2.12
|
||||
github.com/labstack/echo/v4 v4.10.2
|
||||
github.com/lrstanley/girc v0.0.0-20221222153823-a92667a5c9b4
|
||||
github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20211016222428-79310a412696
|
||||
github.com/matterbridge/go-xmpp v0.0.0-20211030125215-791a06c5f1be
|
||||
github.com/matterbridge/gomatrix v0.0.0-20220411225302-271e5088ea27
|
||||
github.com/matterbridge/gozulipbot v0.0.0-20211023205727-a19d6c1f3b75
|
||||
github.com/matterbridge/logrus-prefixed-formatter v0.5.3-0.20200523233437-d971309a77ba
|
||||
github.com/matterbridge/matterclient v0.0.0-20220624224459-272af20c7ddf
|
||||
github.com/matterbridge/matterclient v0.0.0-20221106190440-8bcf49695e0d
|
||||
github.com/matterbridge/telegram-bot-api/v6 v6.5.0
|
||||
github.com/mattermost/mattermost-server/v5 v5.39.3
|
||||
github.com/mattermost/mattermost-server/v6 v6.7.2
|
||||
github.com/mattn/godown v0.0.1
|
||||
github.com/mdp/qrterminal v1.0.1
|
||||
github.com/mitchellh/mapstructure v1.5.0
|
||||
github.com/nelsonken/gomf v0.0.0-20190423072027-c65cc0469e94
|
||||
github.com/olahol/melody v1.1.2
|
||||
github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c
|
||||
github.com/rs/xid v1.4.0
|
||||
github.com/russross/blackfriday v1.6.0
|
||||
@ -47,15 +49,14 @@ require (
|
||||
github.com/writeas/go-strip-markdown v2.0.1+incompatible
|
||||
github.com/yaegashi/msgraph.go v0.1.4
|
||||
github.com/zfjagann/golang-ring v0.0.0-20220330170733-19bcea1b6289
|
||||
go.mau.fi/whatsmeow v0.0.0-20230128195103-dcbc8dd31a22
|
||||
golang.org/x/image v0.3.0
|
||||
golang.org/x/oauth2 v0.4.0
|
||||
golang.org/x/text v0.6.0
|
||||
go.mau.fi/whatsmeow v0.0.0-20230306190159-5caded34a872
|
||||
golang.org/x/image v0.6.0
|
||||
golang.org/x/oauth2 v0.6.0
|
||||
golang.org/x/text v0.8.0
|
||||
gomod.garykim.dev/nc-talk v0.3.0
|
||||
google.golang.org/protobuf v1.28.1
|
||||
gopkg.in/olahol/melody.v1 v1.0.0-20170518105555-d52139073376
|
||||
google.golang.org/protobuf v1.29.0
|
||||
layeh.com/gumble v0.0.0-20221205141517-d1df60a3cc14
|
||||
modernc.org/sqlite v1.20.3
|
||||
modernc.org/sqlite v1.21.0
|
||||
)
|
||||
|
||||
require (
|
||||
@ -80,8 +81,8 @@ require (
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/kettek/apng v0.0.0-20191108220231-414630eed80f // indirect
|
||||
github.com/klauspost/compress v1.15.8 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.0.12 // indirect
|
||||
github.com/klauspost/compress v1.16.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
|
||||
github.com/labstack/gommon v0.4.0 // indirect
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/mattermost/go-i18n v1.11.1-0.20211013152124-5c415071e404 // indirect
|
||||
@ -89,14 +90,13 @@ require (
|
||||
github.com/mattermost/logr v1.0.13 // indirect
|
||||
github.com/mattermost/logr/v2 v2.0.15 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.16 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.13 // indirect
|
||||
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/minio/minio-go/v7 v7.0.24 // indirect
|
||||
github.com/minio/sha256-simd v1.0.0 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/monaco-io/request v1.0.5 // indirect
|
||||
@ -109,7 +109,7 @@ require (
|
||||
github.com/philhofer/fwd v1.1.1 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/rickb777/date v1.12.4 // indirect
|
||||
github.com/rickb777/plural v1.2.0 // indirect
|
||||
github.com/rivo/uniseg v0.2.0 // indirect
|
||||
@ -133,13 +133,13 @@ require (
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
go.uber.org/multierr v1.8.0 // indirect
|
||||
go.uber.org/zap v1.21.0 // indirect
|
||||
golang.org/x/crypto v0.4.0 // indirect
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
|
||||
golang.org/x/net v0.5.0 // indirect
|
||||
golang.org/x/sys v0.4.0 // indirect
|
||||
golang.org/x/term v0.4.0 // indirect
|
||||
golang.org/x/time v0.2.0 // indirect
|
||||
golang.org/x/tools v0.1.12 // indirect
|
||||
golang.org/x/crypto v0.6.0 // indirect
|
||||
golang.org/x/mod v0.8.0 // indirect
|
||||
golang.org/x/net v0.8.0 // indirect
|
||||
golang.org/x/sys v0.6.0 // indirect
|
||||
golang.org/x/term v0.6.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
golang.org/x/tools v0.6.0 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
|
||||
@ -148,9 +148,9 @@ require (
|
||||
lukechampine.com/uint128 v1.2.0 // indirect
|
||||
modernc.org/cc/v3 v3.40.0 // indirect
|
||||
modernc.org/ccgo/v3 v3.16.13 // indirect
|
||||
modernc.org/libc v1.22.2 // indirect
|
||||
modernc.org/libc v1.22.3 // indirect
|
||||
modernc.org/mathutil v1.5.0 // indirect
|
||||
modernc.org/memory v1.4.0 // indirect
|
||||
modernc.org/memory v1.5.0 // indirect
|
||||
modernc.org/opt v0.1.3 // indirect
|
||||
modernc.org/strutil v1.1.3 // indirect
|
||||
modernc.org/token v1.0.1 // indirect
|
||||
|
||||
124
go.sum
124
go.sum
@ -148,8 +148,8 @@ github.com/Rhymen/go-whatsapp v0.1.2-0.20211102134409-31a2e740845c/go.mod h1:DNS
|
||||
github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo=
|
||||
github.com/RoaringBitmap/roaring v0.8.0/go.mod h1:jdT9ykXwHFNdJbEtxePexlFYH9LXucApeS0/+/g+p1I=
|
||||
github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA=
|
||||
github.com/SevereCloud/vksdk/v2 v2.15.0 h1:ywyJvuJzN1sD5+GVcYendwNTpK3R/iBZOlOhulyI9ZQ=
|
||||
github.com/SevereCloud/vksdk/v2 v2.15.0/go.mod h1:0Q20DuofWA78Vdy6aPjZAM6ep1UR6uVEf/fCqdmBYaY=
|
||||
github.com/SevereCloud/vksdk/v2 v2.16.0 h1:DQ90qqwY/yF1X/SWZQs1kQ/Ik+tphK82d+S6Rch46wQ=
|
||||
github.com/SevereCloud/vksdk/v2 v2.16.0/go.mod h1:VN6BH9nFUXcP7Uf0uX74Aht2DQ7+139aG3/Og+jia4w=
|
||||
github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
|
||||
github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
|
||||
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
|
||||
@ -435,7 +435,6 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
@ -585,7 +584,6 @@ github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KE
|
||||
github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
|
||||
github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
|
||||
github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
|
||||
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
|
||||
@ -609,8 +607,6 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB
|
||||
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
|
||||
github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 h1:wG8n/XJQ07TmjbITcGiUaOtXxdrINDz1b0J1w0SzqDc=
|
||||
github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1/go.mod h1:A2S0CWkNylc2phvKXWBBdD3K0iGnDBGbzRpISP2zBl8=
|
||||
github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho=
|
||||
github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
|
||||
github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
|
||||
@ -729,14 +725,13 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
|
||||
github.com/google/go-github/v35 v35.2.0/go.mod h1:s0515YVTI+IMrDoy9Y4pHt9ShGpzHvHO8rZ7L7acgvs=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/gops v0.3.26 h1:Ziyfd8sEhWVbrCIy59c1WOKodI63Jzojwm0JSZbBPS4=
|
||||
github.com/google/gops v0.3.26/go.mod h1:vZ68aOXu2zJoybPyGpaHMmrCyd51DCxJoex4cO3ht/o=
|
||||
github.com/google/gops v0.3.27 h1:BDdWfedShsBbeatZ820oA4DbVOC8yJ4NI8xAlDFWfgI=
|
||||
github.com/google/gops v0.3.27/go.mod h1:lYqabmfnq4Q6UumWNx96Hjup5BDAVc8zmfIy0SkNCSk=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
@ -877,7 +872,6 @@ github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH
|
||||
github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
|
||||
github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI=
|
||||
github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0=
|
||||
github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk=
|
||||
@ -1003,8 +997,8 @@ github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8
|
||||
github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/klauspost/compress v1.15.8 h1:JahtItbkWjf2jzm/T+qgMxkP9EMHsqEUA6vCMGmXvhA=
|
||||
github.com/klauspost/compress v1.15.8/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4=
|
||||
github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
@ -1012,8 +1006,9 @@ github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE=
|
||||
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
|
||||
github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
|
||||
github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
|
||||
github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw=
|
||||
@ -1035,12 +1030,12 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/ktrysmt/go-bitbucket v0.6.4/go.mod h1:9u0v3hsd2rqCHRIpbir1oP7F58uo5dq19sBYvuMoyQ4=
|
||||
github.com/kyokomi/emoji/v2 v2.2.11 h1:Pf/ZWVTbnAVkHOLJLWjPxM/FmgyPe+d85cv/OLP5Yus=
|
||||
github.com/kyokomi/emoji/v2 v2.2.11/go.mod h1:JUcn42DTdsXJo1SWanHh4HKDEyPaR5CqkmoirZZP9qE=
|
||||
github.com/kyokomi/emoji/v2 v2.2.12 h1:sSVA5nH9ebR3Zji1o31wu3yOwD1zKXQA2z0zUyeit60=
|
||||
github.com/kyokomi/emoji/v2 v2.2.12/go.mod h1:JUcn42DTdsXJo1SWanHh4HKDEyPaR5CqkmoirZZP9qE=
|
||||
github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
|
||||
github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y=
|
||||
github.com/labstack/echo/v4 v4.10.0 h1:5CiyngihEO4HXsz3vVsJn7f8xAlWwRr3aY6Ih280ZKA=
|
||||
github.com/labstack/echo/v4 v4.10.0/go.mod h1:S/T/5fy/GigaXnHTkh0ZGe4LpkkQysvRjFMSUTkDRNQ=
|
||||
github.com/labstack/echo/v4 v4.10.2 h1:n1jAhnq/elIFTHr1EYpiYtyKgx4RW9ccVgkqByZaN2M=
|
||||
github.com/labstack/echo/v4 v4.10.2/go.mod h1:OEyqf2//K1DFdE57vw2DRgWY0M7s65IVQO2FzvI4J5k=
|
||||
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
|
||||
github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8=
|
||||
github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM=
|
||||
@ -1060,7 +1055,6 @@ github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lrstanley/girc v0.0.0-20221222153823-a92667a5c9b4 h1:eOJJOM8RTmDcK1F0SqCBX/Ic1vgDnAZfdll6oik0Ups=
|
||||
github.com/lrstanley/girc v0.0.0-20221222153823-a92667a5c9b4/go.mod h1:lgrnhcF8bg/Bd5HA5DOb4Z+uGqUqGnp4skr+J2GwVgI=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
||||
github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
@ -1087,8 +1081,10 @@ github.com/matterbridge/gozulipbot v0.0.0-20211023205727-a19d6c1f3b75 h1:GslZKF7
|
||||
github.com/matterbridge/gozulipbot v0.0.0-20211023205727-a19d6c1f3b75/go.mod h1:yAjnZ34DuDyPHMPHHjOsTk/FefW4JJjoMMCGt/8uuQA=
|
||||
github.com/matterbridge/logrus-prefixed-formatter v0.5.3-0.20200523233437-d971309a77ba h1:XleOY4IjAEIcxAh+IFwT5JT5Ze3RHiYz6m+4ZfZ0rc0=
|
||||
github.com/matterbridge/logrus-prefixed-formatter v0.5.3-0.20200523233437-d971309a77ba/go.mod h1:iXGEotOvwI1R1SjLxRc+BF5rUORTMtE0iMZBT2lxqAU=
|
||||
github.com/matterbridge/matterclient v0.0.0-20220624224459-272af20c7ddf h1:vaiRcLFKSD0fzlcLll53LU8HnpVv8XzP7C0mi8Tfvro=
|
||||
github.com/matterbridge/matterclient v0.0.0-20220624224459-272af20c7ddf/go.mod h1:Zg8PH1P/1CNUxozQ8blnjAV9PA4Qn2qWf33cX5yNKGM=
|
||||
github.com/matterbridge/matterclient v0.0.0-20221106190440-8bcf49695e0d h1:aI0ANEzy3dMv3vEAMQ80AItNie0fBR9ZxE2sAedORmM=
|
||||
github.com/matterbridge/matterclient v0.0.0-20221106190440-8bcf49695e0d/go.mod h1:Zg8PH1P/1CNUxozQ8blnjAV9PA4Qn2qWf33cX5yNKGM=
|
||||
github.com/matterbridge/telegram-bot-api/v6 v6.5.0 h1:wCnHWvt4WGhfognQsuu2OnHyqENBdJRf2mReYTCXggQ=
|
||||
github.com/matterbridge/telegram-bot-api/v6 v6.5.0/go.mod h1:/hSLrs8h/xNsQglQXjwXJ92iZU8XfTGkYUQ7KVDWEVo=
|
||||
github.com/mattermost/go-i18n v1.11.0/go.mod h1:RyS7FDNQlzF1PsjbJWHRI35exqaKGSO9qD4iv8QjE34=
|
||||
github.com/mattermost/go-i18n v1.11.1-0.20211013152124-5c415071e404 h1:Khvh6waxG1cHc4Cz5ef9n3XVCxRWpAKUtqg9PJl5+y8=
|
||||
github.com/mattermost/go-i18n v1.11.1-0.20211013152124-5c415071e404/go.mod h1:RyS7FDNQlzF1PsjbJWHRI35exqaKGSO9qD4iv8QjE34=
|
||||
@ -1129,8 +1125,9 @@ github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcME
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
|
||||
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
@ -1249,6 +1246,8 @@ github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+
|
||||
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
|
||||
github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/olahol/melody v1.1.2 h1:9PZ5kYv/CUy0PRZcJCKja1MUxAh6olVeHkyqaQxO7n0=
|
||||
github.com/olahol/melody v1.1.2/go.mod h1:GgkTl6Y7yWj/HtfD48Q5vLKPVoZOH+Qqgfa7CvJgJM4=
|
||||
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
|
||||
github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
|
||||
github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
|
||||
@ -1365,7 +1364,6 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
|
||||
github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
@ -1416,8 +1414,9 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn
|
||||
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/reflog/dateconstraints v0.2.1/go.mod h1:Ax8AxTBcJc3E/oVS2hd2j7RDM/5MDtuPwuR7lIHtPLo=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/richardlehane/mscfb v1.0.3/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk=
|
||||
github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk=
|
||||
github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
|
||||
@ -1475,7 +1474,6 @@ github.com/shazow/rateio v0.0.0-20200113175441-4461efc8bdc4 h1:zwQ1HBo5FYwn1ksMd
|
||||
github.com/shazow/rateio v0.0.0-20200113175441-4461efc8bdc4/go.mod h1:vt2jWY/3Qw1bIzle5thrJWucsLuuX9iUNnp20CqCciI=
|
||||
github.com/shazow/ssh-chat v1.10.1 h1:ePS+ngEYqm+yUuXegDPutysqLV2WoI22XDOeRgI6CE0=
|
||||
github.com/shazow/ssh-chat v1.10.1/go.mod h1:0+7szsKylcre0vljkVnbuI6q7Odtc+QCDHxa+fFNV54=
|
||||
github.com/shirou/gopsutil/v3 v3.22.10/go.mod h1:QNza6r4YQoydyCfo6rH0blGfKahgibh4dQmV5xdFkQk=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
@ -1554,7 +1552,6 @@ github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHN
|
||||
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
|
||||
github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
|
||||
github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
|
||||
github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
|
||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
|
||||
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
|
||||
@ -1593,7 +1590,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
@ -1627,8 +1623,6 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj
|
||||
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
|
||||
github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
|
||||
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
|
||||
github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
|
||||
github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
|
||||
@ -1698,7 +1692,6 @@ github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf
|
||||
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
|
||||
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEABsc5zNT9+b1CvsJx47JzJ8g=
|
||||
github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM=
|
||||
@ -1719,7 +1712,6 @@ github.com/yuin/goldmark v1.3.8/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
|
||||
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.11/go.mod h1:rmuwmfZ0+bvzB24eSC//bk1R1Zp3hM0OXYv/G2LIilg=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
|
||||
github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
|
||||
github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
|
||||
@ -1738,8 +1730,8 @@ go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3
|
||||
go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
|
||||
go.mau.fi/libsignal v0.1.0 h1:vAKI/nJ5tMhdzke4cTK1fb0idJzz1JuEIpmjprueC+c=
|
||||
go.mau.fi/libsignal v0.1.0/go.mod h1:R8ovrTezxtUNzCQE5PH30StOQWWeBskBsWE55vMfY9I=
|
||||
go.mau.fi/whatsmeow v0.0.0-20230128195103-dcbc8dd31a22 h1:za/zmM0hcfEKTRcLtr2zcUFE4VpUw8CndXNeV+v676c=
|
||||
go.mau.fi/whatsmeow v0.0.0-20230128195103-dcbc8dd31a22/go.mod h1:TrdC8N6SnPFxWo5FiMnDIDFuVyfOLzy5dWDaUPNjcHY=
|
||||
go.mau.fi/whatsmeow v0.0.0-20230306190159-5caded34a872 h1:jrIWy0l9kTxl7bdp3muFofZcyLyI1xxE7BXWeldVKr0=
|
||||
go.mau.fi/whatsmeow v0.0.0-20230306190159-5caded34a872/go.mod h1:zoTtv1CupGEyTew7TOwnBmTbHB4pVad2OzjTf5CVwa0=
|
||||
go.mongodb.org/mongo-driver v1.1.0/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
|
||||
go.mongodb.org/mongo-driver v1.7.0/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8=
|
||||
go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=
|
||||
@ -1826,8 +1818,8 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8=
|
||||
golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
|
||||
golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc=
|
||||
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
@ -1856,8 +1848,8 @@ golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+o
|
||||
golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.0.0-20210622092929-e6eecd499c2c/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20220321031419-a8550c1d254a/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.3.0 h1:HTDXbdK9bjfSWkPzDJIw89W8CAtfFGduujWs33NLLsg=
|
||||
golang.org/x/image v0.3.0/go.mod h1:fXd9211C/0VTlYuAcOhW8dY/RtEJqODXOWBDpmYBf+A=
|
||||
golang.org/x/image v0.6.0 h1:bR8b5okrPI3g/gyZakLZHeWxAR8Dn5CyxXv1hLH5g/4=
|
||||
golang.org/x/image v0.6.0/go.mod h1:MXLdDR43H7cDJq5GEGXEVeeNhPgi+YYEQ2pC1byI1x0=
|
||||
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@ -1885,8 +1877,9 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@ -1969,8 +1962,9 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su
|
||||
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220403103023-749bd193bc2b/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
|
||||
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
|
||||
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
|
||||
golang.org/x/oauth2 v0.0.0-20180227000427-d7d64896b5ff/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
@ -1991,8 +1985,8 @@ golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ
|
||||
golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
|
||||
golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M=
|
||||
golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
|
||||
golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw=
|
||||
golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
|
||||
golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@ -2007,6 +2001,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180224232135-f6cff0780e54/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@ -2099,7 +2095,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@ -2134,20 +2129,22 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220403205710-6acee93ad0eb/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
|
||||
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg=
|
||||
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
|
||||
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@ -2157,16 +2154,17 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
|
||||
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
|
||||
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.2.0 h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE=
|
||||
golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
|
||||
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
@ -2257,8 +2255,9 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
|
||||
golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@ -2437,8 +2436,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
|
||||
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
google.golang.org/protobuf v1.29.0 h1:44S3JjaKmLEE4YIkjzexaP+NzZsudE3Zin5Njn/pYX0=
|
||||
google.golang.org/protobuf v1.29.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
|
||||
@ -2468,8 +2467,6 @@ gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3M
|
||||
gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
|
||||
gopkg.in/olahol/melody.v1 v1.0.0-20170518105555-d52139073376 h1:sY2a+y0j4iDrajJcorb+a0hJIQ6uakU5gybjfLWHlXo=
|
||||
gopkg.in/olahol/melody.v1 v1.0.0-20170518105555-d52139073376/go.mod h1:BHKOc1m5wm8WwQkMqYBoo4vNxhmF7xg8+xhG8L+Cy3M=
|
||||
gopkg.in/olivere/elastic.v6 v6.2.35/go.mod h1:2cTT8Z+/LcArSWpCgvZqBgt3VOqXiy7v00w12Lz8bd4=
|
||||
gopkg.in/olivere/elastic.v6 v6.2.37/go.mod h1:2cTT8Z+/LcArSWpCgvZqBgt3VOqXiy7v00w12Lz8bd4=
|
||||
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
|
||||
@ -2653,8 +2650,8 @@ modernc.org/libc v1.11.98/go.mod h1:ynK5sbjsU77AP+nn61+k+wxUGRx9rOFcIqWYYMaDZ4c=
|
||||
modernc.org/libc v1.11.99/go.mod h1:wLLYgEiY2D17NbBOEp+mIJJJBGSiy7fLL4ZrGGZ+8jI=
|
||||
modernc.org/libc v1.11.101/go.mod h1:wLLYgEiY2D17NbBOEp+mIJJJBGSiy7fLL4ZrGGZ+8jI=
|
||||
modernc.org/libc v1.11.104/go.mod h1:2MH3DaF/gCU8i/UBiVE1VFRos4o523M7zipmwH8SIgQ=
|
||||
modernc.org/libc v1.22.2 h1:4U7v51GyhlWqQmwCHj28Rdq2Yzwk55ovjFrdPjs8Hb0=
|
||||
modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug=
|
||||
modernc.org/libc v1.22.3 h1:D/g6O5ftAfavceqlLOFwaZuA5KYafKwmr30A6iSqoyY=
|
||||
modernc.org/libc v1.22.3/go.mod h1:MQrloYP209xa2zHome2a8HLiLm6k0UT8CoHpV74tOFw=
|
||||
modernc.org/lldb v1.0.0/go.mod h1:jcRvJGWfCGodDZz8BPwiKMJxGJngQ/5DrRapkQnLob8=
|
||||
modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
|
||||
modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
|
||||
@ -2665,8 +2662,8 @@ modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
|
||||
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
|
||||
modernc.org/memory v1.0.4/go.mod h1:nV2OApxradM3/OVbs2/0OsP6nPfakXpi50C7dcoHXlc=
|
||||
modernc.org/memory v1.0.5/go.mod h1:B7OYswTRnfGg+4tDH1t1OeUNnsy2viGTdME4tzd+IjM=
|
||||
modernc.org/memory v1.4.0 h1:crykUfNSnMAXaOJnnxcSzbUGMqkLWjklJKkBK2nwZwk=
|
||||
modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
|
||||
modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
|
||||
modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
|
||||
modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
|
||||
modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
|
||||
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
|
||||
@ -2674,15 +2671,15 @@ modernc.org/ql v1.0.0/go.mod h1:xGVyrLIatPcO2C1JvI/Co8c0sr6y91HKFNy4pt9JXEY=
|
||||
modernc.org/sortutil v1.1.0/go.mod h1:ZyL98OQHJgH9IEfN71VsamvJgrtRX9Dj2gX+vH86L1k=
|
||||
modernc.org/sqlite v1.10.6/go.mod h1:Z9FEjUtZP4qFEg6/SiADg9XCER7aYy9a/j7Pg9P7CPs=
|
||||
modernc.org/sqlite v1.14.3/go.mod h1:xMpicS1i2MJ4C8+Ap0vYBqTwYfpFvdnPE6brbFOtV2Y=
|
||||
modernc.org/sqlite v1.20.3 h1:SqGJMMxjj1PHusLxdYxeQSodg7Jxn9WWkaAQjKrntZs=
|
||||
modernc.org/sqlite v1.20.3/go.mod h1:zKcGyrICaxNTMEHSr1HQ2GUraP0j+845GYw37+EyT6A=
|
||||
modernc.org/sqlite v1.21.0 h1:4aP4MdUf15i3R3M2mx6Q90WHKz3nZLoz96zlB6tNdow=
|
||||
modernc.org/sqlite v1.21.0/go.mod h1:XwQ0wZPIh1iKb5mkvCJ3szzbhk+tykC8ZWqTRTgYRwI=
|
||||
modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
|
||||
modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
|
||||
modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
|
||||
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
|
||||
modernc.org/tcl v1.5.2/go.mod h1:pmJYOLgpiys3oI4AeAafkcUfE+TKKilminxNyU/+Zlo=
|
||||
modernc.org/tcl v1.9.2/go.mod h1:aw7OnlIoiuJgu1gwbTZtrKnGpDqH9wyH++jZcxdqNsg=
|
||||
modernc.org/tcl v1.15.0 h1:oY+JeD11qVVSgVvodMJsu7Edf8tr5E/7tuhF5cNYz34=
|
||||
modernc.org/tcl v1.15.1 h1:mOQwiEK4p7HruMZcwKTZPw/aqtGM4aY00uzWhlKKYws=
|
||||
modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg=
|
||||
modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
@ -2692,7 +2689,6 @@ modernc.org/z v1.2.20/go.mod h1:zU9FiF4PbHdOTUxw+IF8j7ArBMRPsHgq10uVPt6xTzo=
|
||||
modernc.org/z v1.7.0 h1:xkDw/KepgEjeizO2sNco+hqYkU12taxQFqPEmgm1GWE=
|
||||
modernc.org/zappy v1.0.0/go.mod h1:hHe+oGahLVII/aTTyWK/b53VDHMAGCBYYeZ9sn83HC4=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
|
||||
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
|
||||
rsc.io/qr v0.2.0 h1:6vBLea5/NRMVTz8V66gipeLycZMl/+UlFmk8DvqQ6WY=
|
||||
rsc.io/qr v0.2.0/go.mod h1:IF+uZjkb9fqyeF/4tlBoynqmQxUoPfWEKh921coOuXs=
|
||||
|
||||
61
matterbridge.toml.multi
Normal file
61
matterbridge.toml.multi
Normal file
@ -0,0 +1,61 @@
|
||||
#WARNING: as this file contains credentials, be sure to set correct file permissions
|
||||
|
||||
[irc]
|
||||
[irc.foo]
|
||||
Server="irc.myfooserver.com:6667"
|
||||
Nick="matterbot"
|
||||
|
||||
# Can also connect to multiple different servers of the same protocol:
|
||||
[irc]
|
||||
[irc.bar]
|
||||
Server="irc.mybarserver.com:6667"
|
||||
Nick="matterbot"
|
||||
|
||||
[telegram]
|
||||
[telegram.mytelegram]
|
||||
Token="123456789:FillInYourTokenHereThatIsImportant"
|
||||
|
||||
[mattermost]
|
||||
[mattermost.work]
|
||||
#do not prefix it wit http:// or https://
|
||||
Server="yourmattermostserver.domain"
|
||||
Team="yourteam"
|
||||
Login="yourlogin"
|
||||
Password="yourpass"
|
||||
PrefixMessagesWithNick=true
|
||||
|
||||
# Bridge 1: Copy all messages from all rooms to all rooms.
|
||||
# This shows how you can have multiple rooms in a single bridge.
|
||||
[[gateway]]
|
||||
name="cats-are-cool"
|
||||
enable=true
|
||||
[[gateway.inout]]
|
||||
account="irc.foo"
|
||||
channel="#cats-are-cool"
|
||||
[[gateway.inout]]
|
||||
account="irc.bar"
|
||||
channel="#cats-are-cool"
|
||||
[[gateway.inout]]
|
||||
account="telegram.mytelegram"
|
||||
channel="-1234567890123"
|
||||
[[gateway.inout]]
|
||||
account="mattermost.work"
|
||||
channel="cats-are-cool"
|
||||
|
||||
# Bridge 2: Copy some messages from some rooms to some rooms.
|
||||
# This shows how you can have multiple bridges.
|
||||
[[gateway]]
|
||||
name="dog-announcements"
|
||||
enable=true
|
||||
[[gateway.in]]
|
||||
account="irc.foo"
|
||||
channel="#dog-announcements"
|
||||
[[gateway.in]]
|
||||
account="irc.bar"
|
||||
channel="#dog-announcements"
|
||||
[[gateway.out]]
|
||||
account="telegram.mytelegram"
|
||||
channel="-9876543219876"
|
||||
[[gateway.out]]
|
||||
account="mattermost.work"
|
||||
channel="dog-announcements"
|
||||
@ -1956,6 +1956,7 @@ enable=true
|
||||
# nctalk | token | xs25tz5y | The token in the URL when you are in a chat. It will be the last part of the URL.
|
||||
# -------------------------------------------------------------------------------------------------------------------------------------
|
||||
# telegram | chatid | -123456789 | A large negative number. see https://www.linkedin.com/pulse/telegram-bots-beginners-marco-frau
|
||||
# | chatid/topicid | -123456789/12 | A large negative number/number.
|
||||
# -------------------------------------------------------------------------------------------------------------------------------------
|
||||
# vk | peerid | 2000000002 | A number that starts form 2000000000. Use --debug and send any message in chat to get PeerID in the logs
|
||||
# -------------------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
33
vendor/github.com/SevereCloud/vksdk/v2/.golangci.yml
generated
vendored
33
vendor/github.com/SevereCloud/vksdk/v2/.golangci.yml
generated
vendored
@ -1,9 +1,11 @@
|
||||
---
|
||||
run:
|
||||
timeout: 5m
|
||||
|
||||
linters:
|
||||
disable-all: true
|
||||
enable:
|
||||
- bodyclose
|
||||
- deadcode
|
||||
- errcheck
|
||||
- gochecknoglobals
|
||||
- goconst
|
||||
@ -19,13 +21,11 @@ linters:
|
||||
- nakedret
|
||||
- prealloc
|
||||
- staticcheck
|
||||
- structcheck
|
||||
- stylecheck
|
||||
- typecheck
|
||||
- unconvert
|
||||
- unparam
|
||||
- unused
|
||||
- varcheck
|
||||
- whitespace
|
||||
- wsl
|
||||
- godot
|
||||
@ -40,7 +40,6 @@ linters:
|
||||
- makezero
|
||||
- thelper
|
||||
- predeclared
|
||||
- ifshort
|
||||
- revive
|
||||
- durationcheck
|
||||
- gomoddirectives
|
||||
@ -57,9 +56,18 @@ linters:
|
||||
- grouper
|
||||
- decorder
|
||||
- containedctx
|
||||
# - execinquery # FIXME: panic in 1.46.0
|
||||
- nosprintfhostport
|
||||
- usestdlibvars
|
||||
|
||||
- interfacebloat
|
||||
- reassign
|
||||
|
||||
- testableexamples
|
||||
|
||||
- gocheckcompilerdirectives
|
||||
- asasalint
|
||||
|
||||
# - musttag # TODO: need update golangci-lint
|
||||
# - wrapcheck # TODO: v3 Fix
|
||||
# - testpackage # TODO: Fix testpackage
|
||||
# - noctx # TODO: Fix noctx
|
||||
@ -90,11 +98,22 @@ linters:
|
||||
# - errchkjson
|
||||
# - maintidx
|
||||
# - nonamedreturns
|
||||
# - nosnakecase
|
||||
# - execinquery
|
||||
# - logrlint
|
||||
|
||||
# - dupword
|
||||
|
||||
# - ginkgolinter
|
||||
|
||||
# depricated
|
||||
# - maligned
|
||||
# - interfacer
|
||||
# - golint
|
||||
# - ifshort
|
||||
# - deadcode
|
||||
# - structcheck
|
||||
# - varcheck
|
||||
|
||||
issues:
|
||||
exclude-rules:
|
||||
@ -114,4 +133,8 @@ issues:
|
||||
- stylecheck
|
||||
text: "ST1003:.*(Ts|ts).*TS"
|
||||
|
||||
- linters:
|
||||
- gosec
|
||||
text: "G307:"
|
||||
|
||||
exclude-use-default: false
|
||||
|
||||
29
vendor/github.com/SevereCloud/vksdk/v2/CONTRIBUTING.md
generated
vendored
29
vendor/github.com/SevereCloud/vksdk/v2/CONTRIBUTING.md
generated
vendored
@ -6,7 +6,7 @@
|
||||
|
||||
Требования:
|
||||
|
||||
- [Go 1.16+](https://golang.org/doc/install)
|
||||
- [Go 1.18+](https://golang.org/doc/install)
|
||||
- [golangci-lint](https://github.com/golangci/golangci-lint)
|
||||
- [global .gitignore](https://help.github.com/en/articles/ignoring-files#create-a-global-gitignore)
|
||||
|
||||
@ -54,17 +54,17 @@ go test ./...
|
||||
|
||||
```json
|
||||
{
|
||||
"go.testEnvVars": {
|
||||
"SERVICE_TOKEN": "",
|
||||
"WIDGET_TOKEN": "",
|
||||
"MARUSIA_TOKEN": "",
|
||||
"GROUP_TOKEN": "",
|
||||
"CLIENT_SECRET": "",
|
||||
"USER_TOKEN": "",
|
||||
"CLIENT_ID": "123456",
|
||||
"GROUP_ID": "123456",
|
||||
"ACCOUNT_ID": "123456"
|
||||
}
|
||||
"go.testEnvVars": {
|
||||
"SERVICE_TOKEN": "",
|
||||
"WIDGET_TOKEN": "",
|
||||
"MARUSIA_TOKEN": "",
|
||||
"GROUP_TOKEN": "",
|
||||
"CLIENT_SECRET": "",
|
||||
"USER_TOKEN": "",
|
||||
"CLIENT_ID": "123456",
|
||||
"GROUP_ID": "123456",
|
||||
"ACCOUNT_ID": "123456"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@ -88,7 +88,4 @@ git push origin <name_of_your_new_branch>
|
||||
```
|
||||
|
||||
Затем откройте [pull request](https://github.com/SevereCloud/vksdk/pulls)
|
||||
с веткой:
|
||||
|
||||
- `master` если это багфикс
|
||||
- `dev-v1.2.3` если это новая фича
|
||||
с веткой master
|
||||
|
||||
1
vendor/github.com/SevereCloud/vksdk/v2/api/ads.go
generated
vendored
1
vendor/github.com/SevereCloud/vksdk/v2/api/ads.go
generated
vendored
@ -203,7 +203,6 @@ type AdsDeleteCampaignsResponse []ErrorType
|
||||
|
||||
// AdsDeleteCampaigns archives advertising campaigns.
|
||||
//
|
||||
//
|
||||
// Warning! Maximum allowed number of campaigns archived in one request is 100.
|
||||
//
|
||||
// https://vk.com/dev/ads.deleteCampaigns
|
||||
|
||||
4
vendor/github.com/SevereCloud/vksdk/v2/api/api.go
generated
vendored
4
vendor/github.com/SevereCloud/vksdk/v2/api/api.go
generated
vendored
@ -152,7 +152,7 @@ type Params map[string]interface{}
|
||||
// cyrillic symbols will be transliterated automatically.
|
||||
// Numeric format from account.getInfo is supported as well.
|
||||
//
|
||||
// p.Lang(object.LangRU)
|
||||
// p.Lang(object.LangRU)
|
||||
//
|
||||
// See all language code in module object.
|
||||
func (p Params) Lang(v int) Params {
|
||||
@ -248,7 +248,7 @@ func (vk *VK) DefaultHandler(method string, sliceParams ...Params) (Response, er
|
||||
|
||||
rawBody := bytes.NewBufferString(query.Encode())
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", u, rawBody)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, rawBody)
|
||||
if err != nil {
|
||||
return response, err
|
||||
}
|
||||
|
||||
8
vendor/github.com/SevereCloud/vksdk/v2/api/apps.go
generated
vendored
8
vendor/github.com/SevereCloud/vksdk/v2/api/apps.go
generated
vendored
@ -58,7 +58,7 @@ type AppsGetFriendsListResponse struct {
|
||||
|
||||
// AppsGetFriendsList creates friends list for requests and invites in current app.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/apps.getFriendsList
|
||||
func (vk *VK) AppsGetFriendsList(params Params) (response AppsGetFriendsListResponse, err error) {
|
||||
@ -75,7 +75,7 @@ type AppsGetFriendsListExtendedResponse struct {
|
||||
|
||||
// AppsGetFriendsListExtended creates friends list for requests and invites in current app.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/apps.getFriendsList
|
||||
func (vk *VK) AppsGetFriendsListExtended(params Params) (response AppsGetFriendsListExtendedResponse, err error) {
|
||||
@ -92,7 +92,7 @@ type AppsGetLeaderboardResponse struct {
|
||||
|
||||
// AppsGetLeaderboard returns players rating in the game.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/apps.getLeaderboard
|
||||
func (vk *VK) AppsGetLeaderboard(params Params) (response AppsGetLeaderboardResponse, err error) {
|
||||
@ -113,7 +113,7 @@ type AppsGetLeaderboardExtendedResponse struct {
|
||||
|
||||
// AppsGetLeaderboardExtended returns players rating in the game.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/apps.getLeaderboard
|
||||
func (vk *VK) AppsGetLeaderboardExtended(params Params) (response AppsGetLeaderboardExtendedResponse, err error) {
|
||||
|
||||
28
vendor/github.com/SevereCloud/vksdk/v2/api/auth.go
generated
vendored
28
vendor/github.com/SevereCloud/vksdk/v2/api/auth.go
generated
vendored
@ -45,20 +45,20 @@ func (vk *VK) AuthGetProfileInfoBySilentToken(params Params) (response AuthGetPr
|
||||
|
||||
// ExchangeSilentTokenSource call conditions exchangeSilentToken.
|
||||
//
|
||||
// 0 Unknown
|
||||
// 1 Silent authentication
|
||||
// 2 Auth by login and password
|
||||
// 3 Extended registration
|
||||
// 4 Auth by exchange token
|
||||
// 5 Auth by exchange token on reset password
|
||||
// 6 Auth by exchange token on unblock
|
||||
// 7 Auth by exchange token on reset session
|
||||
// 8 Auth by exchange token on change password
|
||||
// 9 Finish phone validation on authentication
|
||||
// 10 Auth by code
|
||||
// 11 Auth by external oauth
|
||||
// 12 Reactivation
|
||||
// 15 Auth by SDK temporary access-token
|
||||
// 0 Unknown
|
||||
// 1 Silent authentication
|
||||
// 2 Auth by login and password
|
||||
// 3 Extended registration
|
||||
// 4 Auth by exchange token
|
||||
// 5 Auth by exchange token on reset password
|
||||
// 6 Auth by exchange token on unblock
|
||||
// 7 Auth by exchange token on reset session
|
||||
// 8 Auth by exchange token on change password
|
||||
// 9 Finish phone validation on authentication
|
||||
// 10 Auth by code
|
||||
// 11 Auth by external oauth
|
||||
// 12 Reactivation
|
||||
// 15 Auth by SDK temporary access-token
|
||||
type ExchangeSilentTokenSource int
|
||||
|
||||
// AuthExchangeSilentAuthTokenResponse struct.
|
||||
|
||||
8
vendor/github.com/SevereCloud/vksdk/v2/api/board.go
generated
vendored
8
vendor/github.com/SevereCloud/vksdk/v2/api/board.go
generated
vendored
@ -78,7 +78,7 @@ type BoardGetCommentsResponse struct {
|
||||
|
||||
// BoardGetComments returns a list of comments on a topic on a community's discussion board.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/board.getComments
|
||||
func (vk *VK) BoardGetComments(params Params) (response BoardGetCommentsResponse, err error) {
|
||||
@ -99,7 +99,7 @@ type BoardGetCommentsExtendedResponse struct {
|
||||
|
||||
// BoardGetCommentsExtended returns a list of comments on a topic on a community's discussion board.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/board.getComments
|
||||
func (vk *VK) BoardGetCommentsExtended(params Params) (response BoardGetCommentsExtendedResponse, err error) {
|
||||
@ -118,7 +118,7 @@ type BoardGetTopicsResponse struct {
|
||||
|
||||
// BoardGetTopics returns a list of topics on a community's discussion board.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/board.getTopics
|
||||
func (vk *VK) BoardGetTopics(params Params) (response BoardGetTopicsResponse, err error) {
|
||||
@ -139,7 +139,7 @@ type BoardGetTopicsExtendedResponse struct {
|
||||
|
||||
// BoardGetTopicsExtended returns a list of topics on a community's discussion board.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/board.getTopics
|
||||
func (vk *VK) BoardGetTopicsExtended(params Params) (response BoardGetTopicsExtendedResponse, err error) {
|
||||
|
||||
23
vendor/github.com/SevereCloud/vksdk/v2/api/calls.go
generated
vendored
Normal file
23
vendor/github.com/SevereCloud/vksdk/v2/api/calls.go
generated
vendored
Normal file
@ -0,0 +1,23 @@
|
||||
package api // import "github.com/SevereCloud/vksdk/v2/api"
|
||||
|
||||
// CallsStartResponse struct.
|
||||
type CallsStartResponse struct {
|
||||
JoinLink string `json:"join_link"`
|
||||
CallID string `json:"call_id"`
|
||||
}
|
||||
|
||||
// CallsStart method.
|
||||
//
|
||||
// https://vk.com/dev/calls.start
|
||||
func (vk *VK) CallsStart(params Params) (response CallsStartResponse, err error) {
|
||||
err = vk.RequestUnmarshal("calls.start", &response, params)
|
||||
return
|
||||
}
|
||||
|
||||
// CallsForceFinish method.
|
||||
//
|
||||
// https://vk.com/dev/calls.forceFinish
|
||||
func (vk *VK) CallsForceFinish(params Params) (response int, err error) {
|
||||
err = vk.RequestUnmarshal("calls.forceFinish", &response, params)
|
||||
return
|
||||
}
|
||||
4
vendor/github.com/SevereCloud/vksdk/v2/api/execute.go
generated
vendored
4
vendor/github.com/SevereCloud/vksdk/v2/api/execute.go
generated
vendored
@ -13,8 +13,8 @@ import (
|
||||
// The Args map variable allows you to retrieve the parameters passed during
|
||||
// the request and avoids code formatting.
|
||||
//
|
||||
// return Args.code; // return parameter "code"
|
||||
// return Args.v; // return parameter "v"
|
||||
// return Args.code; // return parameter "code"
|
||||
// return Args.v; // return parameter "v"
|
||||
//
|
||||
// https://vk.com/dev/execute
|
||||
func (vk *VK) ExecuteWithArgs(code string, params Params, obj interface{}) error {
|
||||
|
||||
4
vendor/github.com/SevereCloud/vksdk/v2/api/fave.go
generated
vendored
4
vendor/github.com/SevereCloud/vksdk/v2/api/fave.go
generated
vendored
@ -79,7 +79,7 @@ type FaveGetResponse struct {
|
||||
|
||||
// FaveGet method.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/fave.get
|
||||
func (vk *VK) FaveGet(params Params) (response FaveGetResponse, err error) {
|
||||
@ -97,7 +97,7 @@ type FaveGetExtendedResponse struct {
|
||||
|
||||
// FaveGetExtended method.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/fave.get
|
||||
func (vk *VK) FaveGetExtended(params Params) (response FaveGetExtendedResponse, err error) {
|
||||
|
||||
4
vendor/github.com/SevereCloud/vksdk/v2/api/friends.go
generated
vendored
4
vendor/github.com/SevereCloud/vksdk/v2/api/friends.go
generated
vendored
@ -169,7 +169,7 @@ func (vk *VK) FriendsGetMutual(params Params) (response FriendsGetMutualResponse
|
||||
|
||||
// FriendsGetOnline returns a list of user IDs of a user's friends who are online.
|
||||
//
|
||||
// online_mobile=0
|
||||
// online_mobile=0
|
||||
//
|
||||
// https://vk.com/dev/friends.getOnline
|
||||
func (vk *VK) FriendsGetOnline(params Params) (response []int, err error) {
|
||||
@ -186,7 +186,7 @@ type FriendsGetOnlineOnlineMobileResponse struct {
|
||||
|
||||
// FriendsGetOnlineOnlineMobile returns a list of user IDs of a user's friends who are online.
|
||||
//
|
||||
// online_mobile=1
|
||||
// online_mobile=1
|
||||
//
|
||||
// https://vk.com/dev/friends.getOnline
|
||||
func (vk *VK) FriendsGetOnlineOnlineMobile(params Params) (response FriendsGetOnlineOnlineMobileResponse, err error) {
|
||||
|
||||
24
vendor/github.com/SevereCloud/vksdk/v2/api/groups.go
generated
vendored
24
vendor/github.com/SevereCloud/vksdk/v2/api/groups.go
generated
vendored
@ -157,7 +157,7 @@ type GroupsGetResponse struct {
|
||||
|
||||
// GroupsGet returns a list of the communities to which a user belongs.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/groups.get
|
||||
func (vk *VK) GroupsGet(params Params) (response GroupsGetResponse, err error) {
|
||||
@ -174,7 +174,7 @@ type GroupsGetExtendedResponse struct {
|
||||
|
||||
// GroupsGetExtended returns a list of the communities to which a user belongs.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/groups.get
|
||||
func (vk *VK) GroupsGetExtended(params Params) (response GroupsGetExtendedResponse, err error) {
|
||||
@ -273,6 +273,8 @@ type GroupsGetCatalogResponse struct {
|
||||
|
||||
// GroupsGetCatalog returns communities list for a catalog category.
|
||||
//
|
||||
// Deprecated: This method is deprecated and may be disabled soon, please avoid
|
||||
//
|
||||
// https://vk.com/dev/groups.getCatalog
|
||||
func (vk *VK) GroupsGetCatalog(params Params) (response GroupsGetCatalogResponse, err error) {
|
||||
err = vk.RequestUnmarshal("groups.getCatalog", &response, params)
|
||||
@ -287,7 +289,7 @@ type GroupsGetCatalogInfoResponse struct {
|
||||
|
||||
// GroupsGetCatalogInfo returns categories list for communities catalog.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/groups.getCatalogInfo
|
||||
func (vk *VK) GroupsGetCatalogInfo(params Params) (response GroupsGetCatalogInfoResponse, err error) {
|
||||
@ -304,7 +306,7 @@ type GroupsGetCatalogInfoExtendedResponse struct {
|
||||
|
||||
// GroupsGetCatalogInfoExtended returns categories list for communities catalog.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/groups.getCatalogInfo
|
||||
func (vk *VK) GroupsGetCatalogInfoExtended(params Params) (response GroupsGetCatalogInfoExtendedResponse, err error) {
|
||||
@ -421,7 +423,7 @@ type GroupsGetMembersFilterManagersResponse struct {
|
||||
|
||||
// GroupsGetMembersFilterManagers returns a list of community members.
|
||||
//
|
||||
// filter=managers
|
||||
// filter=managers
|
||||
//
|
||||
// https://vk.com/dev/groups.getMembers
|
||||
func (vk *VK) GroupsGetMembersFilterManagers(params Params) (
|
||||
@ -522,7 +524,7 @@ func (vk *VK) GroupsInvite(params Params) (response int, err error) {
|
||||
|
||||
// GroupsIsMember returns information specifying whether a user is a member of a community.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/groups.isMember
|
||||
func (vk *VK) GroupsIsMember(params Params) (response int, err error) {
|
||||
@ -542,7 +544,7 @@ type GroupsIsMemberExtendedResponse struct {
|
||||
|
||||
// GroupsIsMemberExtended returns information specifying whether a user is a member of a community.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/groups.isMember
|
||||
func (vk *VK) GroupsIsMemberExtended(params Params) (response GroupsIsMemberExtendedResponse, err error) {
|
||||
@ -556,8 +558,8 @@ type GroupsIsMemberUserIDsExtendedResponse []object.GroupsMemberStatusFull
|
||||
|
||||
// GroupsIsMemberUserIDsExtended returns information specifying whether a user is a member of a community.
|
||||
//
|
||||
// extended=1
|
||||
// need user_ids
|
||||
// extended=1
|
||||
// need user_ids
|
||||
//
|
||||
// https://vk.com/dev/groups.isMember
|
||||
func (vk *VK) GroupsIsMemberUserIDsExtended(params Params) (response GroupsIsMemberUserIDsExtendedResponse, err error) {
|
||||
@ -571,8 +573,8 @@ type GroupsIsMemberUserIDsResponse []object.GroupsMemberStatus
|
||||
|
||||
// GroupsIsMemberUserIDs returns information specifying whether a user is a member of a community.
|
||||
//
|
||||
// extended=0
|
||||
// need user_ids
|
||||
// extended=0
|
||||
// need user_ids
|
||||
//
|
||||
// https://vk.com/dev/groups.isMember
|
||||
func (vk *VK) GroupsIsMemberUserIDs(params Params) (response GroupsIsMemberUserIDsResponse, err error) {
|
||||
|
||||
4
vendor/github.com/SevereCloud/vksdk/v2/api/likes.go
generated
vendored
4
vendor/github.com/SevereCloud/vksdk/v2/api/likes.go
generated
vendored
@ -38,7 +38,7 @@ type LikesGetListResponse struct {
|
||||
|
||||
// LikesGetList likes.getList returns a list of IDs of users who added the specified object to their Likes list.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/likes.getList
|
||||
func (vk *VK) LikesGetList(params Params) (response LikesGetListResponse, err error) {
|
||||
@ -55,7 +55,7 @@ type LikesGetListExtendedResponse struct {
|
||||
|
||||
// LikesGetListExtended likes.getList returns a list of IDs of users who added the specified object to their Likes list.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/likes.getList
|
||||
func (vk *VK) LikesGetListExtended(params Params) (response LikesGetListExtendedResponse, err error) {
|
||||
|
||||
4
vendor/github.com/SevereCloud/vksdk/v2/api/market.go
generated
vendored
4
vendor/github.com/SevereCloud/vksdk/v2/api/market.go
generated
vendored
@ -181,7 +181,7 @@ type MarketGetCommentsResponse struct {
|
||||
|
||||
// MarketGetComments returns comments list for an item.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/market.getComments
|
||||
func (vk *VK) MarketGetComments(params Params) (response MarketGetCommentsResponse, err error) {
|
||||
@ -199,7 +199,7 @@ type MarketGetCommentsExtendedResponse struct {
|
||||
|
||||
// MarketGetCommentsExtended returns comments list for an item.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/market.getComments
|
||||
func (vk *VK) MarketGetCommentsExtended(params Params) (response MarketGetCommentsExtendedResponse, err error) {
|
||||
|
||||
16
vendor/github.com/SevereCloud/vksdk/v2/api/messages.go
generated
vendored
16
vendor/github.com/SevereCloud/vksdk/v2/api/messages.go
generated
vendored
@ -118,6 +118,8 @@ func (vk *VK) MessagesEditChat(params Params) (response int, err error) {
|
||||
|
||||
// MessagesForceCallFinish method.
|
||||
//
|
||||
// Deprecated: Use CallsForceFinish
|
||||
//
|
||||
// https://vk.com/dev/messages.forceCallFinish
|
||||
func (vk *VK) MessagesForceCallFinish(params Params) (response int, err error) {
|
||||
err = vk.RequestUnmarshal("messages.forceCallFinish", &response, params)
|
||||
@ -150,7 +152,7 @@ type MessagesGetByIDResponse struct {
|
||||
|
||||
// MessagesGetByID returns messages by their IDs.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/messages.getById
|
||||
func (vk *VK) MessagesGetByID(params Params) (response MessagesGetByIDResponse, err error) {
|
||||
@ -168,7 +170,7 @@ type MessagesGetByIDExtendedResponse struct {
|
||||
|
||||
// MessagesGetByIDExtended returns messages by their IDs.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/messages.getById
|
||||
func (vk *VK) MessagesGetByIDExtended(params Params) (response MessagesGetByIDExtendedResponse, err error) {
|
||||
@ -268,7 +270,7 @@ type MessagesGetConversationsByIDResponse struct {
|
||||
|
||||
// MessagesGetConversationsByID returns conversations by their IDs.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/messages.getConversationsById
|
||||
func (vk *VK) MessagesGetConversationsByID(params Params) (response MessagesGetConversationsByIDResponse, err error) {
|
||||
@ -286,7 +288,7 @@ type MessagesGetConversationsByIDExtendedResponse struct {
|
||||
|
||||
// MessagesGetConversationsByIDExtended returns conversations by their IDs.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/messages.getConversationsById
|
||||
func (vk *VK) MessagesGetConversationsByIDExtended(params Params) (
|
||||
@ -583,7 +585,7 @@ type MessagesSendUserIDsResponse []struct {
|
||||
|
||||
// MessagesSendPeerIDs sends a message.
|
||||
//
|
||||
// need peer_ids;
|
||||
// need peer_ids;
|
||||
//
|
||||
// https://vk.com/dev/messages.send
|
||||
func (vk *VK) MessagesSendPeerIDs(params Params) (response MessagesSendUserIDsResponse, err error) {
|
||||
@ -593,7 +595,7 @@ func (vk *VK) MessagesSendPeerIDs(params Params) (response MessagesSendUserIDsRe
|
||||
|
||||
// MessagesSendUserIDs sends a message.
|
||||
//
|
||||
// need user_ids or peer_ids;
|
||||
// need user_ids or peer_ids;
|
||||
//
|
||||
// https://vk.com/dev/messages.send
|
||||
//
|
||||
@ -649,6 +651,8 @@ type MessagesStartCallResponse struct {
|
||||
|
||||
// MessagesStartCall method.
|
||||
//
|
||||
// Deprecated: Use CallsStart
|
||||
//
|
||||
// https://vk.com/dev/messages.startCall
|
||||
func (vk *VK) MessagesStartCall(params Params) (response MessagesStartCallResponse, err error) {
|
||||
err = vk.RequestUnmarshal("messages.startCall", &response, params)
|
||||
|
||||
8
vendor/github.com/SevereCloud/vksdk/v2/api/newsfeed.go
generated
vendored
8
vendor/github.com/SevereCloud/vksdk/v2/api/newsfeed.go
generated
vendored
@ -53,7 +53,7 @@ type NewsfeedGetBannedResponse struct {
|
||||
|
||||
// NewsfeedGetBanned returns a list of users and communities banned from the current user's newsfeed.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/newsfeed.getBanned
|
||||
func (vk *VK) NewsfeedGetBanned(params Params) (response NewsfeedGetBannedResponse, err error) {
|
||||
@ -69,7 +69,7 @@ type NewsfeedGetBannedExtendedResponse struct {
|
||||
|
||||
// NewsfeedGetBannedExtended returns a list of users and communities banned from the current user's newsfeed.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/newsfeed.getBanned
|
||||
func (vk *VK) NewsfeedGetBannedExtended(params Params) (response NewsfeedGetBannedExtendedResponse, err error) {
|
||||
@ -183,7 +183,7 @@ type NewsfeedSearchResponse struct {
|
||||
|
||||
// NewsfeedSearch returns search results by statuses.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/newsfeed.search
|
||||
func (vk *VK) NewsfeedSearch(params Params) (response NewsfeedSearchResponse, err error) {
|
||||
@ -204,7 +204,7 @@ type NewsfeedSearchExtendedResponse struct {
|
||||
|
||||
// NewsfeedSearchExtended returns search results by statuses.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/newsfeed.search
|
||||
func (vk *VK) NewsfeedSearchExtended(params Params) (response NewsfeedSearchExtendedResponse, err error) {
|
||||
|
||||
20
vendor/github.com/SevereCloud/vksdk/v2/api/photos.go
generated
vendored
20
vendor/github.com/SevereCloud/vksdk/v2/api/photos.go
generated
vendored
@ -95,7 +95,7 @@ type PhotosGetResponse struct {
|
||||
|
||||
// PhotosGet returns a list of a user's or community's photos.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/photos.get
|
||||
func (vk *VK) PhotosGet(params Params) (response PhotosGetResponse, err error) {
|
||||
@ -112,7 +112,7 @@ type PhotosGetExtendedResponse struct {
|
||||
|
||||
// PhotosGetExtended returns a list of a user's or community's photos.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/photos.get
|
||||
func (vk *VK) PhotosGetExtended(params Params) (response PhotosGetExtendedResponse, err error) {
|
||||
@ -152,7 +152,7 @@ type PhotosGetAllResponse struct {
|
||||
|
||||
// PhotosGetAll returns a list of photos belonging to a user or community, in reverse chronological order.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/photos.getAll
|
||||
func (vk *VK) PhotosGetAll(params Params) (response PhotosGetAllResponse, err error) {
|
||||
@ -170,7 +170,7 @@ type PhotosGetAllExtendedResponse struct {
|
||||
|
||||
// PhotosGetAllExtended returns a list of photos belonging to a user or community, in reverse chronological order.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/photos.getAll
|
||||
func (vk *VK) PhotosGetAllExtended(params Params) (response PhotosGetAllExtendedResponse, err error) {
|
||||
@ -199,7 +199,7 @@ type PhotosGetByIDResponse []object.PhotosPhoto
|
||||
|
||||
// PhotosGetByID returns information about photos by their IDs.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/photos.getById
|
||||
func (vk *VK) PhotosGetByID(params Params) (response PhotosGetByIDResponse, err error) {
|
||||
@ -213,7 +213,7 @@ type PhotosGetByIDExtendedResponse []object.PhotosPhotoFull
|
||||
|
||||
// PhotosGetByIDExtended returns information about photos by their IDs.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/photos.getById
|
||||
func (vk *VK) PhotosGetByIDExtended(params Params) (response PhotosGetByIDExtendedResponse, err error) {
|
||||
@ -244,7 +244,7 @@ type PhotosGetCommentsResponse struct {
|
||||
|
||||
// PhotosGetComments returns a list of comments on a photo.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/photos.getComments
|
||||
func (vk *VK) PhotosGetComments(params Params) (response PhotosGetCommentsResponse, err error) {
|
||||
@ -264,7 +264,7 @@ type PhotosGetCommentsExtendedResponse struct {
|
||||
|
||||
// PhotosGetCommentsExtended returns a list of comments on a photo.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/photos.getComments
|
||||
func (vk *VK) PhotosGetCommentsExtended(params Params) (response PhotosGetCommentsExtendedResponse, err error) {
|
||||
@ -395,7 +395,7 @@ type PhotosGetUserPhotosResponse struct {
|
||||
|
||||
// PhotosGetUserPhotos returns a list of photos in which a user is tagged.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/photos.getUserPhotos
|
||||
func (vk *VK) PhotosGetUserPhotos(params Params) (response PhotosGetUserPhotosResponse, err error) {
|
||||
@ -412,7 +412,7 @@ type PhotosGetUserPhotosExtendedResponse struct {
|
||||
|
||||
// PhotosGetUserPhotosExtended returns a list of photos in which a user is tagged.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/photos.getUserPhotos
|
||||
func (vk *VK) PhotosGetUserPhotosExtended(params Params) (response PhotosGetUserPhotosExtendedResponse, err error) {
|
||||
|
||||
12
vendor/github.com/SevereCloud/vksdk/v2/api/podcasts.go
generated
vendored
12
vendor/github.com/SevereCloud/vksdk/v2/api/podcasts.go
generated
vendored
@ -11,7 +11,7 @@ type PodcastsGetCatalogResponse struct {
|
||||
|
||||
// PodcastsGetCatalog method.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/podcasts.getCatalog
|
||||
func (vk *VK) PodcastsGetCatalog(params Params) (response PodcastsGetCatalogResponse, err error) {
|
||||
@ -28,7 +28,7 @@ type PodcastsGetCatalogExtendedResponse struct {
|
||||
|
||||
// PodcastsGetCatalogExtended method.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/podcasts.getCatalog
|
||||
func (vk *VK) PodcastsGetCatalogExtended(params Params) (response PodcastsGetCatalogExtendedResponse, err error) {
|
||||
@ -70,7 +70,7 @@ type PodcastsGetFeedResponse struct {
|
||||
|
||||
// PodcastsGetFeed method.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/podcasts.getFeed
|
||||
func (vk *VK) PodcastsGetFeed(params Params) (response PodcastsGetFeedResponse, err error) {
|
||||
@ -88,7 +88,7 @@ type PodcastsGetFeedExtendedResponse struct {
|
||||
|
||||
// PodcastsGetFeedExtended method.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/podcasts.getFeed
|
||||
func (vk *VK) PodcastsGetFeedExtended(params Params) (response PodcastsGetFeedExtendedResponse, err error) {
|
||||
@ -116,7 +116,7 @@ type PodcastsGetStartPageResponse struct {
|
||||
|
||||
// PodcastsGetStartPage method.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/podcasts.getStartPage
|
||||
func (vk *VK) PodcastsGetStartPage(params Params) (response PodcastsGetStartPageResponse, err error) {
|
||||
@ -145,7 +145,7 @@ type PodcastsGetStartPageExtendedResponse struct {
|
||||
|
||||
// PodcastsGetStartPageExtended method.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/podcasts.getStartPage
|
||||
func (vk *VK) PodcastsGetStartPageExtended(params Params) (response PodcastsGetStartPageExtendedResponse, err error) {
|
||||
|
||||
22
vendor/github.com/SevereCloud/vksdk/v2/api/stories.go
generated
vendored
22
vendor/github.com/SevereCloud/vksdk/v2/api/stories.go
generated
vendored
@ -30,7 +30,7 @@ type StoriesGetResponse struct {
|
||||
|
||||
// StoriesGet returns stories available for current user.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/stories.get
|
||||
func (vk *VK) StoriesGet(params Params) (response StoriesGetResponse, err error) {
|
||||
@ -50,7 +50,7 @@ type StoriesGetExtendedResponse struct {
|
||||
|
||||
// StoriesGetExtended returns stories available for current user.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/stories.get
|
||||
func (vk *VK) StoriesGetExtended(params Params) (response StoriesGetExtendedResponse, err error) {
|
||||
@ -67,7 +67,7 @@ type StoriesGetBannedResponse struct {
|
||||
|
||||
// StoriesGetBanned returns list of sources hidden from current user's feed.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/stories.getBanned
|
||||
func (vk *VK) StoriesGetBanned(params Params) (response StoriesGetBannedResponse, err error) {
|
||||
@ -85,7 +85,7 @@ type StoriesGetBannedExtendedResponse struct {
|
||||
|
||||
// StoriesGetBannedExtended returns list of sources hidden from current user's feed.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/stories.getBanned
|
||||
func (vk *VK) StoriesGetBannedExtended(params Params) (response StoriesGetBannedExtendedResponse, err error) {
|
||||
@ -102,7 +102,7 @@ type StoriesGetByIDResponse struct {
|
||||
|
||||
// StoriesGetByID returns story by its ID.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/stories.getById
|
||||
func (vk *VK) StoriesGetByID(params Params) (response StoriesGetByIDResponse, err error) {
|
||||
@ -120,7 +120,7 @@ type StoriesGetByIDExtendedResponse struct {
|
||||
|
||||
// StoriesGetByIDExtended returns story by its ID.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/stories.getById
|
||||
func (vk *VK) StoriesGetByIDExtended(params Params) (response StoriesGetByIDExtendedResponse, err error) {
|
||||
@ -152,7 +152,7 @@ type StoriesGetRepliesResponse struct {
|
||||
|
||||
// StoriesGetReplies returns replies to the story.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/stories.getReplies
|
||||
func (vk *VK) StoriesGetReplies(params Params) (response StoriesGetRepliesResponse, err error) {
|
||||
@ -170,7 +170,7 @@ type StoriesGetRepliesExtendedResponse struct {
|
||||
|
||||
// StoriesGetRepliesExtended returns replies to the story.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/stories.getReplies
|
||||
func (vk *VK) StoriesGetRepliesExtended(params Params) (response StoriesGetRepliesExtendedResponse, err error) {
|
||||
@ -213,7 +213,7 @@ type StoriesGetViewersResponse struct {
|
||||
|
||||
// StoriesGetViewers returns a list of story viewers.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/stories.getViewers
|
||||
func (vk *VK) StoriesGetViewers(params Params) (response StoriesGetViewersResponse, err error) {
|
||||
@ -261,7 +261,7 @@ type StoriesSearchResponse struct {
|
||||
|
||||
// StoriesSearch returns search results for stories.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/stories.search
|
||||
func (vk *VK) StoriesSearch(params Params) (response StoriesSearchResponse, err error) {
|
||||
@ -279,7 +279,7 @@ type StoriesSearchExtendedResponse struct {
|
||||
|
||||
// StoriesSearchExtended returns search results for stories.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/stories.search
|
||||
func (vk *VK) StoriesSearchExtended(params Params) (response StoriesSearchExtendedResponse, err error) {
|
||||
|
||||
5
vendor/github.com/SevereCloud/vksdk/v2/api/upload.go
generated
vendored
5
vendor/github.com/SevereCloud/vksdk/v2/api/upload.go
generated
vendored
@ -4,7 +4,6 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"mime/multipart"
|
||||
|
||||
"github.com/SevereCloud/vksdk/v2/object"
|
||||
@ -34,7 +33,7 @@ func (vk *VK) UploadFile(url string, file io.Reader, fieldname, filename string)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
bodyContent, err = ioutil.ReadAll(resp.Body)
|
||||
bodyContent, err = io.ReadAll(resp.Body)
|
||||
|
||||
return
|
||||
}
|
||||
@ -214,7 +213,7 @@ func (vk *VK) uploadOwnerPhoto(params Params, squareCrop string, file io.Reader)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
bodyContent, err := ioutil.ReadAll(resp.Body)
|
||||
bodyContent, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
4
vendor/github.com/SevereCloud/vksdk/v2/api/users.go
generated
vendored
4
vendor/github.com/SevereCloud/vksdk/v2/api/users.go
generated
vendored
@ -24,7 +24,7 @@ type UsersGetFollowersResponse struct {
|
||||
// UsersGetFollowers returns a list of IDs of followers of the user in
|
||||
// question, sorted by date added, most recent first.
|
||||
//
|
||||
// fields="";
|
||||
// fields="";
|
||||
//
|
||||
// https://vk.com/dev/users.getFollowers
|
||||
func (vk *VK) UsersGetFollowers(params Params) (response UsersGetFollowersResponse, err error) {
|
||||
@ -70,7 +70,7 @@ type UsersGetSubscriptionsResponse struct {
|
||||
|
||||
// UsersGetSubscriptions returns a list of IDs of users and public pages followed by the user.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/users.getSubscriptions
|
||||
//
|
||||
|
||||
4
vendor/github.com/SevereCloud/vksdk/v2/api/utils.go
generated
vendored
4
vendor/github.com/SevereCloud/vksdk/v2/api/utils.go
generated
vendored
@ -43,7 +43,7 @@ type UtilsGetLinkStatsResponse object.UtilsLinkStats
|
||||
|
||||
// UtilsGetLinkStats returns stats data for shortened link.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/utils.getLinkStats
|
||||
func (vk *VK) UtilsGetLinkStats(params Params) (response UtilsGetLinkStatsResponse, err error) {
|
||||
@ -57,7 +57,7 @@ type UtilsGetLinkStatsExtendedResponse object.UtilsLinkStatsExtended
|
||||
|
||||
// UtilsGetLinkStatsExtended returns stats data for shortened link.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/utils.getLinkStats
|
||||
func (vk *VK) UtilsGetLinkStatsExtended(params Params) (response UtilsGetLinkStatsExtendedResponse, err error) {
|
||||
|
||||
20
vendor/github.com/SevereCloud/vksdk/v2/api/video.go
generated
vendored
20
vendor/github.com/SevereCloud/vksdk/v2/api/video.go
generated
vendored
@ -97,7 +97,7 @@ type VideoGetResponse struct {
|
||||
|
||||
// VideoGet returns detailed information about videos.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/video.get
|
||||
func (vk *VK) VideoGet(params Params) (response VideoGetResponse, err error) {
|
||||
@ -115,7 +115,7 @@ type VideoGetExtendedResponse struct {
|
||||
|
||||
// VideoGetExtended returns detailed information about videos.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/video.get
|
||||
func (vk *VK) VideoGetExtended(params Params) (response VideoGetExtendedResponse, err error) {
|
||||
@ -143,7 +143,7 @@ type VideoGetAlbumsResponse struct {
|
||||
|
||||
// VideoGetAlbums returns a list of video albums owned by a user or community.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/video.getAlbums
|
||||
func (vk *VK) VideoGetAlbums(params Params) (response VideoGetAlbumsResponse, err error) {
|
||||
@ -160,7 +160,7 @@ type VideoGetAlbumsExtendedResponse struct {
|
||||
|
||||
// VideoGetAlbumsExtended returns a list of video albums owned by a user or community.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/video.getAlbums
|
||||
func (vk *VK) VideoGetAlbumsExtended(params Params) (response VideoGetAlbumsExtendedResponse, err error) {
|
||||
@ -174,7 +174,7 @@ type VideoGetAlbumsByVideoResponse []int
|
||||
|
||||
// VideoGetAlbumsByVideo returns a list of albums in which the video is located.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/video.getAlbumsByVideo
|
||||
func (vk *VK) VideoGetAlbumsByVideo(params Params) (response VideoGetAlbumsByVideoResponse, err error) {
|
||||
@ -191,7 +191,7 @@ type VideoGetAlbumsByVideoExtendedResponse struct {
|
||||
|
||||
// VideoGetAlbumsByVideoExtended returns a list of albums in which the video is located.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/video.getAlbumsByVideo
|
||||
func (vk *VK) VideoGetAlbumsByVideoExtended(params Params) (response VideoGetAlbumsByVideoExtendedResponse, err error) {
|
||||
@ -208,7 +208,7 @@ type VideoGetCommentsResponse struct {
|
||||
|
||||
// VideoGetComments returns a list of comments on a video.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/video.getComments
|
||||
func (vk *VK) VideoGetComments(params Params) (response VideoGetCommentsResponse, err error) {
|
||||
@ -226,7 +226,7 @@ type VideoGetCommentsExtendedResponse struct {
|
||||
|
||||
// VideoGetCommentsExtended returns a list of comments on a video.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/video.getComments
|
||||
func (vk *VK) VideoGetCommentsExtended(params Params) (response VideoGetCommentsExtendedResponse, err error) {
|
||||
@ -321,7 +321,7 @@ type VideoSearchResponse struct {
|
||||
|
||||
// VideoSearch returns a list of videos under the set search criterion.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/video.search
|
||||
func (vk *VK) VideoSearch(params Params) (response VideoSearchResponse, err error) {
|
||||
@ -339,7 +339,7 @@ type VideoSearchExtendedResponse struct {
|
||||
|
||||
// VideoSearchExtended returns a list of videos under the set search criterion.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/video.search
|
||||
func (vk *VK) VideoSearchExtended(params Params) (response VideoSearchExtendedResponse, err error) {
|
||||
|
||||
20
vendor/github.com/SevereCloud/vksdk/v2/api/wall.go
generated
vendored
20
vendor/github.com/SevereCloud/vksdk/v2/api/wall.go
generated
vendored
@ -87,7 +87,7 @@ type WallGetResponse struct {
|
||||
|
||||
// WallGet returns a list of posts on a user wall or community wall.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/wall.get
|
||||
func (vk *VK) WallGet(params Params) (response WallGetResponse, err error) {
|
||||
@ -105,7 +105,7 @@ type WallGetExtendedResponse struct {
|
||||
|
||||
// WallGetExtended returns a list of posts on a user wall or community wall.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/wall.get
|
||||
func (vk *VK) WallGetExtended(params Params) (response WallGetExtendedResponse, err error) {
|
||||
@ -119,7 +119,7 @@ type WallGetByIDResponse []object.WallWallpost
|
||||
|
||||
// WallGetByID returns a list of posts from user or community walls by their IDs.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/wall.getById
|
||||
func (vk *VK) WallGetByID(params Params) (response WallGetByIDResponse, err error) {
|
||||
@ -136,7 +136,7 @@ type WallGetByIDExtendedResponse struct {
|
||||
|
||||
// WallGetByIDExtended returns a list of posts from user or community walls by their IDs.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/wall.getById
|
||||
func (vk *VK) WallGetByIDExtended(params Params) (response WallGetByIDExtendedResponse, err error) {
|
||||
@ -156,7 +156,7 @@ type WallGetCommentResponse struct {
|
||||
|
||||
// WallGetComment allows to obtain wall comment info.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/wall.getComment
|
||||
func (vk *VK) WallGetComment(params Params) (response WallGetCommentResponse, err error) {
|
||||
@ -179,7 +179,7 @@ type WallGetCommentExtendedResponse struct {
|
||||
|
||||
// WallGetCommentExtended allows to obtain wall comment info.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/wall.getComment
|
||||
func (vk *VK) WallGetCommentExtended(params Params) (response WallGetCommentExtendedResponse, err error) {
|
||||
@ -200,7 +200,7 @@ type WallGetCommentsResponse struct {
|
||||
|
||||
// WallGetComments returns a list of comments on a post on a user wall or community wall.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/wall.getComments
|
||||
func (vk *VK) WallGetComments(params Params) (response WallGetCommentsResponse, err error) {
|
||||
@ -222,7 +222,7 @@ type WallGetCommentsExtendedResponse struct {
|
||||
|
||||
// WallGetCommentsExtended returns a list of comments on a post on a user wall or community wall.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/wall.getComments
|
||||
func (vk *VK) WallGetCommentsExtended(params Params) (response WallGetCommentsExtendedResponse, err error) {
|
||||
@ -347,7 +347,7 @@ type WallSearchResponse struct {
|
||||
|
||||
// WallSearch allows to search posts on user or community walls.
|
||||
//
|
||||
// extended=0
|
||||
// extended=0
|
||||
//
|
||||
// https://vk.com/dev/wall.search
|
||||
func (vk *VK) WallSearch(params Params) (response WallSearchResponse, err error) {
|
||||
@ -365,7 +365,7 @@ type WallSearchExtendedResponse struct {
|
||||
|
||||
// WallSearchExtended allows to search posts on user or community walls.
|
||||
//
|
||||
// extended=1
|
||||
// extended=1
|
||||
//
|
||||
// https://vk.com/dev/wall.search
|
||||
func (vk *VK) WallSearchExtended(params Params) (response WallSearchExtendedResponse, err error) {
|
||||
|
||||
2
vendor/github.com/SevereCloud/vksdk/v2/doc.go
generated
vendored
2
vendor/github.com/SevereCloud/vksdk/v2/doc.go
generated
vendored
@ -7,6 +7,6 @@ package vksdk
|
||||
|
||||
// Module constants.
|
||||
const (
|
||||
Version = "2.15.0"
|
||||
Version = "2.16.0"
|
||||
API = "5.131"
|
||||
)
|
||||
|
||||
114
vendor/github.com/SevereCloud/vksdk/v2/events/events.go
generated
vendored
114
vendor/github.com/SevereCloud/vksdk/v2/events/events.go
generated
vendored
@ -156,13 +156,15 @@ func NewFuncList() *FuncList {
|
||||
}
|
||||
|
||||
// Handler group event handler.
|
||||
func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:gocyclo
|
||||
func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { //nolint:gocyclo
|
||||
ctx = context.WithValue(ctx, internal.GroupIDKey, e.GroupID)
|
||||
ctx = context.WithValue(ctx, internal.EventIDKey, e.EventID)
|
||||
ctx = context.WithValue(ctx, internal.EventVersionKey, e.V)
|
||||
|
||||
if sliceFunc, ok := fl.special[e.Type]; ok {
|
||||
for _, f := range sliceFunc {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, e) }()
|
||||
} else {
|
||||
@ -179,6 +181,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.messageNew {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -192,6 +196,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.messageReply {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -205,6 +211,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.messageEdit {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -218,6 +226,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.messageAllow {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -231,6 +241,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.messageDeny {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -244,6 +256,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.messageTypingState {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -257,6 +271,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.messageEvent {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -270,6 +286,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.photoNew {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -283,6 +301,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.photoCommentNew {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -296,6 +316,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.photoCommentEdit {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -309,6 +331,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.photoCommentRestore {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -322,6 +346,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.photoCommentDelete {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -335,6 +361,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.audioNew {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -348,6 +376,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.videoNew {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -361,6 +391,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.videoCommentNew {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -374,6 +406,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.videoCommentEdit {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -387,6 +421,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.videoCommentRestore {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -400,6 +436,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.videoCommentDelete {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -413,6 +451,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.wallPostNew {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -426,6 +466,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.wallRepost {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -439,6 +481,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.wallReplyNew {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -452,6 +496,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.wallReplyEdit {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -465,6 +511,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.wallReplyRestore {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -478,6 +526,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.wallReplyDelete {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -491,6 +541,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.boardPostNew {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -504,6 +556,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.boardPostEdit {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -517,6 +571,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.boardPostRestore {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -530,6 +586,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.boardPostDelete {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -543,6 +601,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.marketCommentNew {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -556,6 +616,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.marketCommentEdit {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -569,6 +631,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.marketCommentRestore {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -582,6 +646,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.marketCommentDelete {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -595,6 +661,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.marketOrderNew {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -608,6 +676,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.marketOrderEdit {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -621,6 +691,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.groupLeave {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -634,6 +706,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.groupJoin {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -647,6 +721,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.userBlock {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -660,6 +736,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.userUnblock {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -673,6 +751,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.pollVoteNew {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -686,6 +766,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.groupOfficersEdit {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -699,6 +781,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.groupChangeSettings {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -712,6 +796,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.groupChangePhoto {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -725,6 +811,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.vkpayTransaction {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -738,6 +826,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.leadFormsNew {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -751,6 +841,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.appPayload {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -764,6 +856,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.messageRead {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -777,6 +871,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.likeAdd {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -790,6 +886,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.likeRemove {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -803,6 +901,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.donutSubscriptionCreate {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -816,6 +916,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.donutSubscriptionProlonged {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -829,6 +931,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.donutSubscriptionExpired {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -842,6 +946,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.donutSubscriptionCancelled {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -855,6 +961,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.donutSubscriptionPriceChanged {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -868,6 +976,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.donutMoneyWithdraw {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
@ -881,6 +991,8 @@ func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:
|
||||
}
|
||||
|
||||
for _, f := range fl.donutMoneyWithdrawError {
|
||||
f := f
|
||||
|
||||
if fl.goroutine {
|
||||
go func() { f(ctx, obj) }()
|
||||
} else {
|
||||
|
||||
12
vendor/github.com/SevereCloud/vksdk/v2/object/stats.go
generated
vendored
12
vendor/github.com/SevereCloud/vksdk/v2/object/stats.go
generated
vendored
@ -66,12 +66,24 @@ type StatsViews struct {
|
||||
|
||||
// StatsWallpostStat struct.
|
||||
type StatsWallpostStat struct {
|
||||
PostID int `json:"post_id"`
|
||||
Hide int `json:"hide"` // Hidings number
|
||||
JoinGroup int `json:"join_group"` // People have joined the group
|
||||
Links int `json:"links"` // Link click-through
|
||||
ReachSubscribers int `json:"reach_subscribers"` // Subscribers reach
|
||||
ReachTotal int `json:"reach_total"` // Total reach
|
||||
ReachViral int `json:"reach_viral"` // Viral reach
|
||||
ReachAds int `json:"reach_ads"` // Advertising reach
|
||||
Report int `json:"report"` // Reports number
|
||||
ToGroup int `json:"to_group"` // Click-through to community
|
||||
Unsubscribe int `json:"unsubscribe"` // Unsubscribed members
|
||||
AdViews int `json:"ad_views"`
|
||||
AdSubscribers int `json:"ad_subscribers"`
|
||||
AdHide int `json:"ad_hide"`
|
||||
AdUnsubscribe int `json:"ad_unsubscribe"`
|
||||
AdLinks int `json:"ad_links"`
|
||||
AdToGroup int `json:"ad_to_group"`
|
||||
AdJoinGroup int `json:"ad_join_group"`
|
||||
AdCoverage int `json:"ad_coverage"`
|
||||
AdReport int `json:"ad_report"`
|
||||
}
|
||||
|
||||
2
vendor/github.com/SevereCloud/vksdk/v2/object/stories.go
generated
vendored
2
vendor/github.com/SevereCloud/vksdk/v2/object/stories.go
generated
vendored
@ -239,7 +239,7 @@ func (cs StoriesClickableStickers) ToJSON() string {
|
||||
}
|
||||
|
||||
// StoriesClickableSticker struct.
|
||||
type StoriesClickableSticker struct { // nolint: maligned
|
||||
type StoriesClickableSticker struct { //nolint: maligned
|
||||
ID int `json:"id"`
|
||||
Type string `json:"type"`
|
||||
ClickableArea []StoriesClickablePoint `json:"clickable_area"`
|
||||
|
||||
8
vendor/github.com/SevereCloud/vksdk/v2/object/users.go
generated
vendored
8
vendor/github.com/SevereCloud/vksdk/v2/object/users.go
generated
vendored
@ -125,6 +125,7 @@ type UsersUser struct {
|
||||
MobilePhone string `json:"mobile_phone"`
|
||||
HomePhone string `json:"home_phone"`
|
||||
FoundWith int `json:"found_with"` // TODO: check it
|
||||
ImageStatus ImageStatusInfo `json:"image_status"`
|
||||
OnlineInfo UsersOnlineInfo `json:"online_info"`
|
||||
Mutual FriendsRequestsMutual `json:"mutual"`
|
||||
TrackCode string `json:"track_code"`
|
||||
@ -138,6 +139,13 @@ func (user UsersUser) ToMention() string {
|
||||
return fmt.Sprintf("[id%d|%s %s]", user.ID, user.FirstName, user.LastName)
|
||||
}
|
||||
|
||||
// ImageStatusInfo struct.
|
||||
type ImageStatusInfo struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Images []BaseImage `json:"images"`
|
||||
}
|
||||
|
||||
// UsersOnlineInfo struct.
|
||||
type UsersOnlineInfo struct {
|
||||
AppID int `json:"app_id"`
|
||||
|
||||
1
vendor/github.com/SevereCloud/vksdk/v2/object/wall.go
generated
vendored
1
vendor/github.com/SevereCloud/vksdk/v2/object/wall.go
generated
vendored
@ -161,6 +161,7 @@ type WallWallpost struct {
|
||||
Edited int `json:"edited"` // Date of editing in Unixtime
|
||||
Copyright WallPostCopyright `json:"copyright"`
|
||||
PostID int `json:"post_id"`
|
||||
PostponedID int `json:"postponed_id"` // ID from scheduled posts
|
||||
ParentsStack []int `json:"parents_stack"`
|
||||
Donut WallWallpostDonut `json:"donut"`
|
||||
ShortTextRate float64 `json:"short_text_rate"`
|
||||
|
||||
3
vendor/github.com/google/gops/agent/agent.go
generated
vendored
3
vendor/github.com/google/gops/agent/agent.go
generated
vendored
@ -12,7 +12,6 @@ import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"os"
|
||||
gosignal "os/signal"
|
||||
@ -115,7 +114,7 @@ func Listen(opts Options) error {
|
||||
}
|
||||
port := listener.Addr().(*net.TCPAddr).Port
|
||||
portfile = filepath.Join(gopsdir, strconv.Itoa(os.Getpid()))
|
||||
err = ioutil.WriteFile(portfile, []byte(strconv.Itoa(port)), os.ModePerm)
|
||||
err = os.WriteFile(portfile, []byte(strconv.Itoa(port)), os.ModePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
12
vendor/github.com/google/gops/internal/internal.go
generated
vendored
12
vendor/github.com/google/gops/internal/internal.go
generated
vendored
@ -6,11 +6,9 @@ package internal
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
@ -26,14 +24,6 @@ func ConfigDir() (string, error) {
|
||||
return filepath.Join(userConfigDir, "gops"), nil
|
||||
}
|
||||
|
||||
if runtime.GOOS == "windows" {
|
||||
return filepath.Join(os.Getenv("APPDATA"), "gops"), nil
|
||||
}
|
||||
|
||||
if xdgConfigDir := os.Getenv("XDG_CONFIG_HOME"); xdgConfigDir != "" {
|
||||
return filepath.Join(xdgConfigDir, "gops"), nil
|
||||
}
|
||||
|
||||
homeDir := guessUnixHomeDir()
|
||||
if homeDir == "" {
|
||||
return "", errors.New("unable to get current user home directory: os/user lookup failed; $HOME is empty")
|
||||
@ -62,7 +52,7 @@ func GetPort(pid int) (string, error) {
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
b, err := ioutil.ReadFile(portfile)
|
||||
b, err := os.ReadFile(portfile)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
2
vendor/github.com/klauspost/compress/.goreleaser.yml
generated
vendored
2
vendor/github.com/klauspost/compress/.goreleaser.yml
generated
vendored
@ -3,7 +3,7 @@
|
||||
before:
|
||||
hooks:
|
||||
- ./gen.sh
|
||||
- go install mvdan.cc/garble@latest
|
||||
- go install mvdan.cc/garble@v0.9.3
|
||||
|
||||
builds:
|
||||
-
|
||||
|
||||
64
vendor/github.com/klauspost/compress/README.md
generated
vendored
64
vendor/github.com/klauspost/compress/README.md
generated
vendored
@ -9,7 +9,6 @@ This package provides various compression algorithms.
|
||||
* [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding.
|
||||
* [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped requests efficiently.
|
||||
* [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation.
|
||||
* [fuzz package](https://github.com/klauspost/compress-fuzz) for fuzz testing all compressors/decompressors here.
|
||||
|
||||
[](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories)
|
||||
[](https://github.com/klauspost/compress/actions/workflows/go.yml)
|
||||
@ -17,6 +16,62 @@ This package provides various compression algorithms.
|
||||
|
||||
# changelog
|
||||
|
||||
* Jan 21st, 2023 (v1.15.15)
|
||||
* deflate: Improve level 7-9 by @klauspost in https://github.com/klauspost/compress/pull/739
|
||||
* zstd: Add delta encoding support by @greatroar in https://github.com/klauspost/compress/pull/728
|
||||
* zstd: Various speed improvements by @greatroar https://github.com/klauspost/compress/pull/741 https://github.com/klauspost/compress/pull/734 https://github.com/klauspost/compress/pull/736 https://github.com/klauspost/compress/pull/744 https://github.com/klauspost/compress/pull/743 https://github.com/klauspost/compress/pull/745
|
||||
* gzhttp: Add SuffixETag() and DropETag() options to prevent ETag collisions on compressed responses by @willbicks in https://github.com/klauspost/compress/pull/740
|
||||
|
||||
* Jan 3rd, 2023 (v1.15.14)
|
||||
|
||||
* flate: Improve speed in big stateless blocks https://github.com/klauspost/compress/pull/718
|
||||
* zstd: Minor speed tweaks by @greatroar in https://github.com/klauspost/compress/pull/716 https://github.com/klauspost/compress/pull/720
|
||||
* export NoGzipResponseWriter for custom ResponseWriter wrappers by @harshavardhana in https://github.com/klauspost/compress/pull/722
|
||||
* s2: Add example for indexing and existing stream https://github.com/klauspost/compress/pull/723
|
||||
|
||||
* Dec 11, 2022 (v1.15.13)
|
||||
* zstd: Add [MaxEncodedSize](https://pkg.go.dev/github.com/klauspost/compress@v1.15.13/zstd#Encoder.MaxEncodedSize) to encoder https://github.com/klauspost/compress/pull/691
|
||||
* zstd: Various tweaks and improvements https://github.com/klauspost/compress/pull/693 https://github.com/klauspost/compress/pull/695 https://github.com/klauspost/compress/pull/696 https://github.com/klauspost/compress/pull/701 https://github.com/klauspost/compress/pull/702 https://github.com/klauspost/compress/pull/703 https://github.com/klauspost/compress/pull/704 https://github.com/klauspost/compress/pull/705 https://github.com/klauspost/compress/pull/706 https://github.com/klauspost/compress/pull/707 https://github.com/klauspost/compress/pull/708
|
||||
|
||||
* Oct 26, 2022 (v1.15.12)
|
||||
|
||||
* zstd: Tweak decoder allocs. https://github.com/klauspost/compress/pull/680
|
||||
* gzhttp: Always delete `HeaderNoCompression` https://github.com/klauspost/compress/pull/683
|
||||
|
||||
* Sept 26, 2022 (v1.15.11)
|
||||
|
||||
* flate: Improve level 1-3 compression https://github.com/klauspost/compress/pull/678
|
||||
* zstd: Improve "best" compression by @nightwolfz in https://github.com/klauspost/compress/pull/677
|
||||
* zstd: Fix+reduce decompression allocations https://github.com/klauspost/compress/pull/668
|
||||
* zstd: Fix non-effective noescape tag https://github.com/klauspost/compress/pull/667
|
||||
|
||||
* Sept 16, 2022 (v1.15.10)
|
||||
|
||||
* zstd: Add [WithDecodeAllCapLimit](https://pkg.go.dev/github.com/klauspost/compress@v1.15.10/zstd#WithDecodeAllCapLimit) https://github.com/klauspost/compress/pull/649
|
||||
* Add Go 1.19 - deprecate Go 1.16 https://github.com/klauspost/compress/pull/651
|
||||
* flate: Improve level 5+6 compression https://github.com/klauspost/compress/pull/656
|
||||
* zstd: Improve "better" compresssion https://github.com/klauspost/compress/pull/657
|
||||
* s2: Improve "best" compression https://github.com/klauspost/compress/pull/658
|
||||
* s2: Improve "better" compression. https://github.com/klauspost/compress/pull/635
|
||||
* s2: Slightly faster non-assembly decompression https://github.com/klauspost/compress/pull/646
|
||||
* Use arrays for constant size copies https://github.com/klauspost/compress/pull/659
|
||||
|
||||
* July 21, 2022 (v1.15.9)
|
||||
|
||||
* zstd: Fix decoder crash on amd64 (no BMI) on invalid input https://github.com/klauspost/compress/pull/645
|
||||
* zstd: Disable decoder extended memory copies (amd64) due to possible crashes https://github.com/klauspost/compress/pull/644
|
||||
* zstd: Allow single segments up to "max decoded size" by @klauspost in https://github.com/klauspost/compress/pull/643
|
||||
|
||||
* July 13, 2022 (v1.15.8)
|
||||
|
||||
* gzip: fix stack exhaustion bug in Reader.Read https://github.com/klauspost/compress/pull/641
|
||||
* s2: Add Index header trim/restore https://github.com/klauspost/compress/pull/638
|
||||
* zstd: Optimize seqdeq amd64 asm by @greatroar in https://github.com/klauspost/compress/pull/636
|
||||
* zstd: Improve decoder memcopy https://github.com/klauspost/compress/pull/637
|
||||
* huff0: Pass a single bitReader pointer to asm by @greatroar in https://github.com/klauspost/compress/pull/634
|
||||
* zstd: Branchless getBits for amd64 w/o BMI2 by @greatroar in https://github.com/klauspost/compress/pull/640
|
||||
* gzhttp: Remove header before writing https://github.com/klauspost/compress/pull/639
|
||||
|
||||
* June 29, 2022 (v1.15.7)
|
||||
|
||||
* s2: Fix absolute forward seeks https://github.com/klauspost/compress/pull/633
|
||||
@ -81,15 +136,15 @@ This package provides various compression algorithms.
|
||||
* gzhttp: Add zstd to transport by @klauspost in [#400](https://github.com/klauspost/compress/pull/400)
|
||||
* gzhttp: Make content-type optional by @klauspost in [#510](https://github.com/klauspost/compress/pull/510)
|
||||
|
||||
<details>
|
||||
<summary>See Details</summary>
|
||||
Both compression and decompression now supports "synchronous" stream operations. This means that whenever "concurrency" is set to 1, they will operate without spawning goroutines.
|
||||
|
||||
Stream decompression is now faster on asynchronous, since the goroutine allocation much more effectively splits the workload. On typical streams this will typically use 2 cores fully for decompression. When a stream has finished decoding no goroutines will be left over, so decoders can now safely be pooled and still be garbage collected.
|
||||
|
||||
While the release has been extensively tested, it is recommended to testing when upgrading.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>See changes to v1.14.x</summary>
|
||||
|
||||
* Feb 22, 2022 (v1.14.4)
|
||||
* flate: Fix rare huffman only (-2) corruption. [#503](https://github.com/klauspost/compress/pull/503)
|
||||
* zip: Update deprecated CreateHeaderRaw to correctly call CreateRaw by @saracen in [#502](https://github.com/klauspost/compress/pull/502)
|
||||
@ -115,6 +170,7 @@ While the release has been extensively tested, it is recommended to testing when
|
||||
* zstd: Performance improvement in [#420]( https://github.com/klauspost/compress/pull/420) [#456](https://github.com/klauspost/compress/pull/456) [#437](https://github.com/klauspost/compress/pull/437) [#467](https://github.com/klauspost/compress/pull/467) [#468](https://github.com/klauspost/compress/pull/468)
|
||||
* zstd: add arm64 xxhash assembly in [#464](https://github.com/klauspost/compress/pull/464)
|
||||
* Add garbled for binaries for s2 in [#445](https://github.com/klauspost/compress/pull/445)
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>See changes to v1.13.x</summary>
|
||||
|
||||
31
vendor/github.com/klauspost/compress/fse/compress.go
generated
vendored
31
vendor/github.com/klauspost/compress/fse/compress.go
generated
vendored
@ -146,54 +146,51 @@ func (s *Scratch) compress(src []byte) error {
|
||||
c1.encodeZero(tt[src[ip-2]])
|
||||
ip -= 2
|
||||
}
|
||||
src = src[:ip]
|
||||
|
||||
// Main compression loop.
|
||||
switch {
|
||||
case !s.zeroBits && s.actualTableLog <= 8:
|
||||
// We can encode 4 symbols without requiring a flush.
|
||||
// We do not need to check if any output is 0 bits.
|
||||
for ip >= 4 {
|
||||
for ; len(src) >= 4; src = src[:len(src)-4] {
|
||||
s.bw.flush32()
|
||||
v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1]
|
||||
v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1]
|
||||
c2.encode(tt[v0])
|
||||
c1.encode(tt[v1])
|
||||
c2.encode(tt[v2])
|
||||
c1.encode(tt[v3])
|
||||
ip -= 4
|
||||
}
|
||||
case !s.zeroBits:
|
||||
// We do not need to check if any output is 0 bits.
|
||||
for ip >= 4 {
|
||||
for ; len(src) >= 4; src = src[:len(src)-4] {
|
||||
s.bw.flush32()
|
||||
v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1]
|
||||
v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1]
|
||||
c2.encode(tt[v0])
|
||||
c1.encode(tt[v1])
|
||||
s.bw.flush32()
|
||||
c2.encode(tt[v2])
|
||||
c1.encode(tt[v3])
|
||||
ip -= 4
|
||||
}
|
||||
case s.actualTableLog <= 8:
|
||||
// We can encode 4 symbols without requiring a flush
|
||||
for ip >= 4 {
|
||||
for ; len(src) >= 4; src = src[:len(src)-4] {
|
||||
s.bw.flush32()
|
||||
v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1]
|
||||
v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1]
|
||||
c2.encodeZero(tt[v0])
|
||||
c1.encodeZero(tt[v1])
|
||||
c2.encodeZero(tt[v2])
|
||||
c1.encodeZero(tt[v3])
|
||||
ip -= 4
|
||||
}
|
||||
default:
|
||||
for ip >= 4 {
|
||||
for ; len(src) >= 4; src = src[:len(src)-4] {
|
||||
s.bw.flush32()
|
||||
v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1]
|
||||
v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1]
|
||||
c2.encodeZero(tt[v0])
|
||||
c1.encodeZero(tt[v1])
|
||||
s.bw.flush32()
|
||||
c2.encodeZero(tt[v2])
|
||||
c1.encodeZero(tt[v3])
|
||||
ip -= 4
|
||||
}
|
||||
}
|
||||
|
||||
@ -459,15 +456,17 @@ func (s *Scratch) countSimple(in []byte) (max int) {
|
||||
for _, v := range in {
|
||||
s.count[v]++
|
||||
}
|
||||
m := uint32(0)
|
||||
m, symlen := uint32(0), s.symbolLen
|
||||
for i, v := range s.count[:] {
|
||||
if v == 0 {
|
||||
continue
|
||||
}
|
||||
if v > m {
|
||||
m = v
|
||||
}
|
||||
if v > 0 {
|
||||
s.symbolLen = uint16(i) + 1
|
||||
}
|
||||
symlen = uint16(i) + 1
|
||||
}
|
||||
s.symbolLen = symlen
|
||||
return int(m)
|
||||
}
|
||||
|
||||
|
||||
8
vendor/github.com/klauspost/compress/huff0/bitreader.go
generated
vendored
8
vendor/github.com/klauspost/compress/huff0/bitreader.go
generated
vendored
@ -67,7 +67,6 @@ func (b *bitReaderBytes) fillFast() {
|
||||
|
||||
// 2 bounds checks.
|
||||
v := b.in[b.off-4 : b.off]
|
||||
v = v[:4]
|
||||
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
|
||||
b.value |= uint64(low) << (b.bitsRead - 32)
|
||||
b.bitsRead -= 32
|
||||
@ -88,8 +87,7 @@ func (b *bitReaderBytes) fill() {
|
||||
return
|
||||
}
|
||||
if b.off > 4 {
|
||||
v := b.in[b.off-4:]
|
||||
v = v[:4]
|
||||
v := b.in[b.off-4 : b.off]
|
||||
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
|
||||
b.value |= uint64(low) << (b.bitsRead - 32)
|
||||
b.bitsRead -= 32
|
||||
@ -179,7 +177,6 @@ func (b *bitReaderShifted) fillFast() {
|
||||
|
||||
// 2 bounds checks.
|
||||
v := b.in[b.off-4 : b.off]
|
||||
v = v[:4]
|
||||
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
|
||||
b.value |= uint64(low) << ((b.bitsRead - 32) & 63)
|
||||
b.bitsRead -= 32
|
||||
@ -200,8 +197,7 @@ func (b *bitReaderShifted) fill() {
|
||||
return
|
||||
}
|
||||
if b.off > 4 {
|
||||
v := b.in[b.off-4:]
|
||||
v = v[:4]
|
||||
v := b.in[b.off-4 : b.off]
|
||||
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
|
||||
b.value |= uint64(low) << ((b.bitsRead - 32) & 63)
|
||||
b.bitsRead -= 32
|
||||
|
||||
114
vendor/github.com/klauspost/compress/huff0/compress.go
generated
vendored
114
vendor/github.com/klauspost/compress/huff0/compress.go
generated
vendored
@ -365,29 +365,29 @@ func (s *Scratch) countSimple(in []byte) (max int, reuse bool) {
|
||||
m := uint32(0)
|
||||
if len(s.prevTable) > 0 {
|
||||
for i, v := range s.count[:] {
|
||||
if v == 0 {
|
||||
continue
|
||||
}
|
||||
if v > m {
|
||||
m = v
|
||||
}
|
||||
if v > 0 {
|
||||
s.symbolLen = uint16(i) + 1
|
||||
if i >= len(s.prevTable) {
|
||||
reuse = false
|
||||
} else {
|
||||
if s.prevTable[i].nBits == 0 {
|
||||
reuse = false
|
||||
}
|
||||
}
|
||||
s.symbolLen = uint16(i) + 1
|
||||
if i >= len(s.prevTable) {
|
||||
reuse = false
|
||||
} else if s.prevTable[i].nBits == 0 {
|
||||
reuse = false
|
||||
}
|
||||
}
|
||||
return int(m), reuse
|
||||
}
|
||||
for i, v := range s.count[:] {
|
||||
if v == 0 {
|
||||
continue
|
||||
}
|
||||
if v > m {
|
||||
m = v
|
||||
}
|
||||
if v > 0 {
|
||||
s.symbolLen = uint16(i) + 1
|
||||
}
|
||||
s.symbolLen = uint16(i) + 1
|
||||
}
|
||||
return int(m), false
|
||||
}
|
||||
@ -484,34 +484,35 @@ func (s *Scratch) buildCTable() error {
|
||||
// Different from reference implementation.
|
||||
huffNode0 := s.nodes[0 : huffNodesLen+1]
|
||||
|
||||
for huffNode[nonNullRank].count == 0 {
|
||||
for huffNode[nonNullRank].count() == 0 {
|
||||
nonNullRank--
|
||||
}
|
||||
|
||||
lowS := int16(nonNullRank)
|
||||
nodeRoot := nodeNb + lowS - 1
|
||||
lowN := nodeNb
|
||||
huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count
|
||||
huffNode[lowS].parent, huffNode[lowS-1].parent = uint16(nodeNb), uint16(nodeNb)
|
||||
huffNode[nodeNb].setCount(huffNode[lowS].count() + huffNode[lowS-1].count())
|
||||
huffNode[lowS].setParent(nodeNb)
|
||||
huffNode[lowS-1].setParent(nodeNb)
|
||||
nodeNb++
|
||||
lowS -= 2
|
||||
for n := nodeNb; n <= nodeRoot; n++ {
|
||||
huffNode[n].count = 1 << 30
|
||||
huffNode[n].setCount(1 << 30)
|
||||
}
|
||||
// fake entry, strong barrier
|
||||
huffNode0[0].count = 1 << 31
|
||||
huffNode0[0].setCount(1 << 31)
|
||||
|
||||
// create parents
|
||||
for nodeNb <= nodeRoot {
|
||||
var n1, n2 int16
|
||||
if huffNode0[lowS+1].count < huffNode0[lowN+1].count {
|
||||
if huffNode0[lowS+1].count() < huffNode0[lowN+1].count() {
|
||||
n1 = lowS
|
||||
lowS--
|
||||
} else {
|
||||
n1 = lowN
|
||||
lowN++
|
||||
}
|
||||
if huffNode0[lowS+1].count < huffNode0[lowN+1].count {
|
||||
if huffNode0[lowS+1].count() < huffNode0[lowN+1].count() {
|
||||
n2 = lowS
|
||||
lowS--
|
||||
} else {
|
||||
@ -519,18 +520,19 @@ func (s *Scratch) buildCTable() error {
|
||||
lowN++
|
||||
}
|
||||
|
||||
huffNode[nodeNb].count = huffNode0[n1+1].count + huffNode0[n2+1].count
|
||||
huffNode0[n1+1].parent, huffNode0[n2+1].parent = uint16(nodeNb), uint16(nodeNb)
|
||||
huffNode[nodeNb].setCount(huffNode0[n1+1].count() + huffNode0[n2+1].count())
|
||||
huffNode0[n1+1].setParent(nodeNb)
|
||||
huffNode0[n2+1].setParent(nodeNb)
|
||||
nodeNb++
|
||||
}
|
||||
|
||||
// distribute weights (unlimited tree height)
|
||||
huffNode[nodeRoot].nbBits = 0
|
||||
huffNode[nodeRoot].setNbBits(0)
|
||||
for n := nodeRoot - 1; n >= startNode; n-- {
|
||||
huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1
|
||||
huffNode[n].setNbBits(huffNode[huffNode[n].parent()].nbBits() + 1)
|
||||
}
|
||||
for n := uint16(0); n <= nonNullRank; n++ {
|
||||
huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1
|
||||
huffNode[n].setNbBits(huffNode[huffNode[n].parent()].nbBits() + 1)
|
||||
}
|
||||
s.actualTableLog = s.setMaxHeight(int(nonNullRank))
|
||||
maxNbBits := s.actualTableLog
|
||||
@ -542,7 +544,7 @@ func (s *Scratch) buildCTable() error {
|
||||
var nbPerRank [tableLogMax + 1]uint16
|
||||
var valPerRank [16]uint16
|
||||
for _, v := range huffNode[:nonNullRank+1] {
|
||||
nbPerRank[v.nbBits]++
|
||||
nbPerRank[v.nbBits()]++
|
||||
}
|
||||
// determine stating value per rank
|
||||
{
|
||||
@ -557,7 +559,7 @@ func (s *Scratch) buildCTable() error {
|
||||
|
||||
// push nbBits per symbol, symbol order
|
||||
for _, v := range huffNode[:nonNullRank+1] {
|
||||
s.cTable[v.symbol].nBits = v.nbBits
|
||||
s.cTable[v.symbol()].nBits = v.nbBits()
|
||||
}
|
||||
|
||||
// assign value within rank, symbol order
|
||||
@ -603,12 +605,12 @@ func (s *Scratch) huffSort() {
|
||||
pos := rank[r].current
|
||||
rank[r].current++
|
||||
prev := nodes[(pos-1)&huffNodesMask]
|
||||
for pos > rank[r].base && c > prev.count {
|
||||
for pos > rank[r].base && c > prev.count() {
|
||||
nodes[pos&huffNodesMask] = prev
|
||||
pos--
|
||||
prev = nodes[(pos-1)&huffNodesMask]
|
||||
}
|
||||
nodes[pos&huffNodesMask] = nodeElt{count: c, symbol: byte(n)}
|
||||
nodes[pos&huffNodesMask] = makeNodeElt(c, byte(n))
|
||||
}
|
||||
}
|
||||
|
||||
@ -617,7 +619,7 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 {
|
||||
huffNode := s.nodes[1 : huffNodesLen+1]
|
||||
//huffNode = huffNode[: huffNodesLen]
|
||||
|
||||
largestBits := huffNode[lastNonNull].nbBits
|
||||
largestBits := huffNode[lastNonNull].nbBits()
|
||||
|
||||
// early exit : no elt > maxNbBits
|
||||
if largestBits <= maxNbBits {
|
||||
@ -627,14 +629,14 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 {
|
||||
baseCost := int(1) << (largestBits - maxNbBits)
|
||||
n := uint32(lastNonNull)
|
||||
|
||||
for huffNode[n].nbBits > maxNbBits {
|
||||
totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits))
|
||||
huffNode[n].nbBits = maxNbBits
|
||||
for huffNode[n].nbBits() > maxNbBits {
|
||||
totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits()))
|
||||
huffNode[n].setNbBits(maxNbBits)
|
||||
n--
|
||||
}
|
||||
// n stops at huffNode[n].nbBits <= maxNbBits
|
||||
|
||||
for huffNode[n].nbBits == maxNbBits {
|
||||
for huffNode[n].nbBits() == maxNbBits {
|
||||
n--
|
||||
}
|
||||
// n end at index of smallest symbol using < maxNbBits
|
||||
@ -655,10 +657,10 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 {
|
||||
{
|
||||
currentNbBits := maxNbBits
|
||||
for pos := int(n); pos >= 0; pos-- {
|
||||
if huffNode[pos].nbBits >= currentNbBits {
|
||||
if huffNode[pos].nbBits() >= currentNbBits {
|
||||
continue
|
||||
}
|
||||
currentNbBits = huffNode[pos].nbBits // < maxNbBits
|
||||
currentNbBits = huffNode[pos].nbBits() // < maxNbBits
|
||||
rankLast[maxNbBits-currentNbBits] = uint32(pos)
|
||||
}
|
||||
}
|
||||
@ -675,8 +677,8 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 {
|
||||
if lowPos == noSymbol {
|
||||
break
|
||||
}
|
||||
highTotal := huffNode[highPos].count
|
||||
lowTotal := 2 * huffNode[lowPos].count
|
||||
highTotal := huffNode[highPos].count()
|
||||
lowTotal := 2 * huffNode[lowPos].count()
|
||||
if highTotal <= lowTotal {
|
||||
break
|
||||
}
|
||||
@ -692,13 +694,14 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 {
|
||||
// this rank is no longer empty
|
||||
rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]
|
||||
}
|
||||
huffNode[rankLast[nBitsToDecrease]].nbBits++
|
||||
huffNode[rankLast[nBitsToDecrease]].setNbBits(1 +
|
||||
huffNode[rankLast[nBitsToDecrease]].nbBits())
|
||||
if rankLast[nBitsToDecrease] == 0 {
|
||||
/* special case, reached largest symbol */
|
||||
rankLast[nBitsToDecrease] = noSymbol
|
||||
} else {
|
||||
rankLast[nBitsToDecrease]--
|
||||
if huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease {
|
||||
if huffNode[rankLast[nBitsToDecrease]].nbBits() != maxNbBits-nBitsToDecrease {
|
||||
rankLast[nBitsToDecrease] = noSymbol /* this rank is now empty */
|
||||
}
|
||||
}
|
||||
@ -706,15 +709,15 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 {
|
||||
|
||||
for totalCost < 0 { /* Sometimes, cost correction overshoot */
|
||||
if rankLast[1] == noSymbol { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */
|
||||
for huffNode[n].nbBits == maxNbBits {
|
||||
for huffNode[n].nbBits() == maxNbBits {
|
||||
n--
|
||||
}
|
||||
huffNode[n+1].nbBits--
|
||||
huffNode[n+1].setNbBits(huffNode[n+1].nbBits() - 1)
|
||||
rankLast[1] = n + 1
|
||||
totalCost++
|
||||
continue
|
||||
}
|
||||
huffNode[rankLast[1]+1].nbBits--
|
||||
huffNode[rankLast[1]+1].setNbBits(huffNode[rankLast[1]+1].nbBits() - 1)
|
||||
rankLast[1]++
|
||||
totalCost++
|
||||
}
|
||||
@ -722,9 +725,26 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 {
|
||||
return maxNbBits
|
||||
}
|
||||
|
||||
type nodeElt struct {
|
||||
count uint32
|
||||
parent uint16
|
||||
symbol byte
|
||||
nbBits uint8
|
||||
// A nodeElt is the fields
|
||||
//
|
||||
// count uint32
|
||||
// parent uint16
|
||||
// symbol byte
|
||||
// nbBits uint8
|
||||
//
|
||||
// in some order, all squashed into an integer so that the compiler
|
||||
// always loads and stores entire nodeElts instead of separate fields.
|
||||
type nodeElt uint64
|
||||
|
||||
func makeNodeElt(count uint32, symbol byte) nodeElt {
|
||||
return nodeElt(count) | nodeElt(symbol)<<48
|
||||
}
|
||||
|
||||
func (e *nodeElt) count() uint32 { return uint32(*e) }
|
||||
func (e *nodeElt) parent() uint16 { return uint16(*e >> 32) }
|
||||
func (e *nodeElt) symbol() byte { return byte(*e >> 48) }
|
||||
func (e *nodeElt) nbBits() uint8 { return uint8(*e >> 56) }
|
||||
|
||||
func (e *nodeElt) setCount(c uint32) { *e = (*e)&0xffffffff00000000 | nodeElt(c) }
|
||||
func (e *nodeElt) setParent(p int16) { *e = (*e)&0xffff0000ffffffff | nodeElt(uint16(p))<<32 }
|
||||
func (e *nodeElt) setNbBits(n uint8) { *e = (*e)&0x00ffffffffffffff | nodeElt(n)<<56 }
|
||||
|
||||
38
vendor/github.com/klauspost/compress/huff0/decompress.go
generated
vendored
38
vendor/github.com/klauspost/compress/huff0/decompress.go
generated
vendored
@ -61,7 +61,7 @@ func ReadTable(in []byte, s *Scratch) (s2 *Scratch, remain []byte, err error) {
|
||||
b, err := fse.Decompress(in[:iSize], s.fse)
|
||||
s.fse.Out = nil
|
||||
if err != nil {
|
||||
return s, nil, err
|
||||
return s, nil, fmt.Errorf("fse decompress returned: %w", err)
|
||||
}
|
||||
if len(b) > 255 {
|
||||
return s, nil, errors.New("corrupt input: output table too large")
|
||||
@ -763,17 +763,20 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
|
||||
d.bufs.Put(buf)
|
||||
return nil, errors.New("corruption detected: stream overrun 1")
|
||||
}
|
||||
copy(out, buf[0][:])
|
||||
copy(out[dstEvery:], buf[1][:])
|
||||
copy(out[dstEvery*2:], buf[2][:])
|
||||
copy(out[dstEvery*3:], buf[3][:])
|
||||
out = out[bufoff:]
|
||||
decoded += bufoff * 4
|
||||
// There must at least be 3 buffers left.
|
||||
if len(out) < dstEvery*3 {
|
||||
if len(out)-bufoff < dstEvery*3 {
|
||||
d.bufs.Put(buf)
|
||||
return nil, errors.New("corruption detected: stream overrun 2")
|
||||
}
|
||||
//copy(out, buf[0][:])
|
||||
//copy(out[dstEvery:], buf[1][:])
|
||||
//copy(out[dstEvery*2:], buf[2][:])
|
||||
*(*[bufoff]byte)(out) = buf[0]
|
||||
*(*[bufoff]byte)(out[dstEvery:]) = buf[1]
|
||||
*(*[bufoff]byte)(out[dstEvery*2:]) = buf[2]
|
||||
*(*[bufoff]byte)(out[dstEvery*3:]) = buf[3]
|
||||
out = out[bufoff:]
|
||||
decoded += bufoff * 4
|
||||
}
|
||||
}
|
||||
if off > 0 {
|
||||
@ -997,17 +1000,22 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) {
|
||||
d.bufs.Put(buf)
|
||||
return nil, errors.New("corruption detected: stream overrun 1")
|
||||
}
|
||||
copy(out, buf[0][:])
|
||||
copy(out[dstEvery:], buf[1][:])
|
||||
copy(out[dstEvery*2:], buf[2][:])
|
||||
copy(out[dstEvery*3:], buf[3][:])
|
||||
out = out[bufoff:]
|
||||
decoded += bufoff * 4
|
||||
// There must at least be 3 buffers left.
|
||||
if len(out) < dstEvery*3 {
|
||||
if len(out)-bufoff < dstEvery*3 {
|
||||
d.bufs.Put(buf)
|
||||
return nil, errors.New("corruption detected: stream overrun 2")
|
||||
}
|
||||
|
||||
//copy(out, buf[0][:])
|
||||
//copy(out[dstEvery:], buf[1][:])
|
||||
//copy(out[dstEvery*2:], buf[2][:])
|
||||
// copy(out[dstEvery*3:], buf[3][:])
|
||||
*(*[bufoff]byte)(out) = buf[0]
|
||||
*(*[bufoff]byte)(out[dstEvery:]) = buf[1]
|
||||
*(*[bufoff]byte)(out[dstEvery*2:]) = buf[2]
|
||||
*(*[bufoff]byte)(out[dstEvery*3:]) = buf[3]
|
||||
out = out[bufoff:]
|
||||
decoded += bufoff * 4
|
||||
}
|
||||
}
|
||||
if off > 0 {
|
||||
|
||||
4
vendor/github.com/klauspost/compress/huff0/decompress_amd64.go
generated
vendored
4
vendor/github.com/klauspost/compress/huff0/decompress_amd64.go
generated
vendored
@ -14,12 +14,14 @@ import (
|
||||
|
||||
// decompress4x_main_loop_x86 is an x86 assembler implementation
|
||||
// of Decompress4X when tablelog > 8.
|
||||
//
|
||||
//go:noescape
|
||||
func decompress4x_main_loop_amd64(ctx *decompress4xContext)
|
||||
|
||||
// decompress4x_8b_loop_x86 is an x86 assembler implementation
|
||||
// of Decompress4X when tablelog <= 8 which decodes 4 entries
|
||||
// per loop.
|
||||
//
|
||||
//go:noescape
|
||||
func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext)
|
||||
|
||||
@ -145,11 +147,13 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) {
|
||||
|
||||
// decompress4x_main_loop_x86 is an x86 assembler implementation
|
||||
// of Decompress1X when tablelog > 8.
|
||||
//
|
||||
//go:noescape
|
||||
func decompress1x_main_loop_amd64(ctx *decompress1xContext)
|
||||
|
||||
// decompress4x_main_loop_x86 is an x86 with BMI2 assembler implementation
|
||||
// of Decompress1X when tablelog > 8.
|
||||
//
|
||||
//go:noescape
|
||||
func decompress1x_main_loop_bmi2(ctx *decompress1xContext)
|
||||
|
||||
|
||||
585
vendor/github.com/klauspost/compress/huff0/decompress_amd64.s
generated
vendored
585
vendor/github.com/klauspost/compress/huff0/decompress_amd64.s
generated
vendored
@ -1,364 +1,352 @@
|
||||
// Code generated by command: go run gen.go -out ../decompress_amd64.s -pkg=huff0. DO NOT EDIT.
|
||||
|
||||
//go:build amd64 && !appengine && !noasm && gc
|
||||
// +build amd64,!appengine,!noasm,gc
|
||||
|
||||
// func decompress4x_main_loop_amd64(ctx *decompress4xContext)
|
||||
TEXT ·decompress4x_main_loop_amd64(SB), $0-8
|
||||
XORQ DX, DX
|
||||
|
||||
// Preload values
|
||||
MOVQ ctx+0(FP), AX
|
||||
MOVBQZX 8(AX), DI
|
||||
MOVQ 16(AX), SI
|
||||
MOVQ 48(AX), BX
|
||||
MOVQ 24(AX), R9
|
||||
MOVQ 32(AX), R10
|
||||
MOVQ (AX), R11
|
||||
MOVQ 16(AX), BX
|
||||
MOVQ 48(AX), SI
|
||||
MOVQ 24(AX), R8
|
||||
MOVQ 32(AX), R9
|
||||
MOVQ (AX), R10
|
||||
|
||||
// Main loop
|
||||
main_loop:
|
||||
MOVQ SI, R8
|
||||
CMPQ R8, BX
|
||||
XORL DX, DX
|
||||
CMPQ BX, SI
|
||||
SETGE DL
|
||||
|
||||
// br0.fillFast32()
|
||||
MOVQ 32(R11), R12
|
||||
MOVBQZX 40(R11), R13
|
||||
CMPQ R13, $0x20
|
||||
MOVQ 32(R10), R11
|
||||
MOVBQZX 40(R10), R12
|
||||
CMPQ R12, $0x20
|
||||
JBE skip_fill0
|
||||
MOVQ 24(R11), AX
|
||||
SUBQ $0x20, R13
|
||||
MOVQ 24(R10), AX
|
||||
SUBQ $0x20, R12
|
||||
SUBQ $0x04, AX
|
||||
MOVQ (R11), R14
|
||||
MOVQ (R10), R13
|
||||
|
||||
// b.value |= uint64(low) << (b.bitsRead & 63)
|
||||
MOVL (AX)(R14*1), R14
|
||||
MOVQ R13, CX
|
||||
SHLQ CL, R14
|
||||
MOVQ AX, 24(R11)
|
||||
ORQ R14, R12
|
||||
MOVL (AX)(R13*1), R13
|
||||
MOVQ R12, CX
|
||||
SHLQ CL, R13
|
||||
MOVQ AX, 24(R10)
|
||||
ORQ R13, R11
|
||||
|
||||
// exhausted = exhausted || (br0.off < 4)
|
||||
CMPQ AX, $0x04
|
||||
SETLT AL
|
||||
ORB AL, DL
|
||||
// exhausted += (br0.off < 4)
|
||||
CMPQ AX, $0x04
|
||||
ADCB $+0, DL
|
||||
|
||||
skip_fill0:
|
||||
// val0 := br0.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v0 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br0.advance(uint8(v0.entry)
|
||||
MOVB CH, AL
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
|
||||
// val1 := br0.peekTopBits(peekBits)
|
||||
MOVQ DI, CX
|
||||
MOVQ R12, R14
|
||||
SHRQ CL, R14
|
||||
MOVQ R11, R13
|
||||
SHRQ CL, R13
|
||||
|
||||
// v1 := table[val1&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br0.advance(uint8(v1.entry))
|
||||
MOVB CH, AH
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
|
||||
// these two writes get coalesced
|
||||
// out[id * dstEvery + 0] = uint8(v0.entry >> 8)
|
||||
// out[id * dstEvery + 1] = uint8(v1.entry >> 8)
|
||||
MOVW AX, (R8)
|
||||
MOVW AX, (BX)
|
||||
|
||||
// update the bitreader structure
|
||||
MOVQ R12, 32(R11)
|
||||
MOVB R13, 40(R11)
|
||||
ADDQ R9, R8
|
||||
MOVQ R11, 32(R10)
|
||||
MOVB R12, 40(R10)
|
||||
|
||||
// br1.fillFast32()
|
||||
MOVQ 80(R11), R12
|
||||
MOVBQZX 88(R11), R13
|
||||
CMPQ R13, $0x20
|
||||
MOVQ 80(R10), R11
|
||||
MOVBQZX 88(R10), R12
|
||||
CMPQ R12, $0x20
|
||||
JBE skip_fill1
|
||||
MOVQ 72(R11), AX
|
||||
SUBQ $0x20, R13
|
||||
MOVQ 72(R10), AX
|
||||
SUBQ $0x20, R12
|
||||
SUBQ $0x04, AX
|
||||
MOVQ 48(R11), R14
|
||||
MOVQ 48(R10), R13
|
||||
|
||||
// b.value |= uint64(low) << (b.bitsRead & 63)
|
||||
MOVL (AX)(R14*1), R14
|
||||
MOVQ R13, CX
|
||||
SHLQ CL, R14
|
||||
MOVQ AX, 72(R11)
|
||||
ORQ R14, R12
|
||||
MOVL (AX)(R13*1), R13
|
||||
MOVQ R12, CX
|
||||
SHLQ CL, R13
|
||||
MOVQ AX, 72(R10)
|
||||
ORQ R13, R11
|
||||
|
||||
// exhausted = exhausted || (br1.off < 4)
|
||||
CMPQ AX, $0x04
|
||||
SETLT AL
|
||||
ORB AL, DL
|
||||
// exhausted += (br1.off < 4)
|
||||
CMPQ AX, $0x04
|
||||
ADCB $+0, DL
|
||||
|
||||
skip_fill1:
|
||||
// val0 := br1.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v0 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br1.advance(uint8(v0.entry)
|
||||
MOVB CH, AL
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
|
||||
// val1 := br1.peekTopBits(peekBits)
|
||||
MOVQ DI, CX
|
||||
MOVQ R12, R14
|
||||
SHRQ CL, R14
|
||||
MOVQ R11, R13
|
||||
SHRQ CL, R13
|
||||
|
||||
// v1 := table[val1&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br1.advance(uint8(v1.entry))
|
||||
MOVB CH, AH
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
|
||||
// these two writes get coalesced
|
||||
// out[id * dstEvery + 0] = uint8(v0.entry >> 8)
|
||||
// out[id * dstEvery + 1] = uint8(v1.entry >> 8)
|
||||
MOVW AX, (R8)
|
||||
MOVW AX, (BX)(R8*1)
|
||||
|
||||
// update the bitreader structure
|
||||
MOVQ R12, 80(R11)
|
||||
MOVB R13, 88(R11)
|
||||
ADDQ R9, R8
|
||||
MOVQ R11, 80(R10)
|
||||
MOVB R12, 88(R10)
|
||||
|
||||
// br2.fillFast32()
|
||||
MOVQ 128(R11), R12
|
||||
MOVBQZX 136(R11), R13
|
||||
CMPQ R13, $0x20
|
||||
MOVQ 128(R10), R11
|
||||
MOVBQZX 136(R10), R12
|
||||
CMPQ R12, $0x20
|
||||
JBE skip_fill2
|
||||
MOVQ 120(R11), AX
|
||||
SUBQ $0x20, R13
|
||||
MOVQ 120(R10), AX
|
||||
SUBQ $0x20, R12
|
||||
SUBQ $0x04, AX
|
||||
MOVQ 96(R11), R14
|
||||
MOVQ 96(R10), R13
|
||||
|
||||
// b.value |= uint64(low) << (b.bitsRead & 63)
|
||||
MOVL (AX)(R14*1), R14
|
||||
MOVQ R13, CX
|
||||
SHLQ CL, R14
|
||||
MOVQ AX, 120(R11)
|
||||
ORQ R14, R12
|
||||
MOVL (AX)(R13*1), R13
|
||||
MOVQ R12, CX
|
||||
SHLQ CL, R13
|
||||
MOVQ AX, 120(R10)
|
||||
ORQ R13, R11
|
||||
|
||||
// exhausted = exhausted || (br2.off < 4)
|
||||
CMPQ AX, $0x04
|
||||
SETLT AL
|
||||
ORB AL, DL
|
||||
// exhausted += (br2.off < 4)
|
||||
CMPQ AX, $0x04
|
||||
ADCB $+0, DL
|
||||
|
||||
skip_fill2:
|
||||
// val0 := br2.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v0 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br2.advance(uint8(v0.entry)
|
||||
MOVB CH, AL
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
|
||||
// val1 := br2.peekTopBits(peekBits)
|
||||
MOVQ DI, CX
|
||||
MOVQ R12, R14
|
||||
SHRQ CL, R14
|
||||
MOVQ R11, R13
|
||||
SHRQ CL, R13
|
||||
|
||||
// v1 := table[val1&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br2.advance(uint8(v1.entry))
|
||||
MOVB CH, AH
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
|
||||
// these two writes get coalesced
|
||||
// out[id * dstEvery + 0] = uint8(v0.entry >> 8)
|
||||
// out[id * dstEvery + 1] = uint8(v1.entry >> 8)
|
||||
MOVW AX, (R8)
|
||||
MOVW AX, (BX)(R8*2)
|
||||
|
||||
// update the bitreader structure
|
||||
MOVQ R12, 128(R11)
|
||||
MOVB R13, 136(R11)
|
||||
ADDQ R9, R8
|
||||
MOVQ R11, 128(R10)
|
||||
MOVB R12, 136(R10)
|
||||
|
||||
// br3.fillFast32()
|
||||
MOVQ 176(R11), R12
|
||||
MOVBQZX 184(R11), R13
|
||||
CMPQ R13, $0x20
|
||||
MOVQ 176(R10), R11
|
||||
MOVBQZX 184(R10), R12
|
||||
CMPQ R12, $0x20
|
||||
JBE skip_fill3
|
||||
MOVQ 168(R11), AX
|
||||
SUBQ $0x20, R13
|
||||
MOVQ 168(R10), AX
|
||||
SUBQ $0x20, R12
|
||||
SUBQ $0x04, AX
|
||||
MOVQ 144(R11), R14
|
||||
MOVQ 144(R10), R13
|
||||
|
||||
// b.value |= uint64(low) << (b.bitsRead & 63)
|
||||
MOVL (AX)(R14*1), R14
|
||||
MOVQ R13, CX
|
||||
SHLQ CL, R14
|
||||
MOVQ AX, 168(R11)
|
||||
ORQ R14, R12
|
||||
MOVL (AX)(R13*1), R13
|
||||
MOVQ R12, CX
|
||||
SHLQ CL, R13
|
||||
MOVQ AX, 168(R10)
|
||||
ORQ R13, R11
|
||||
|
||||
// exhausted = exhausted || (br3.off < 4)
|
||||
CMPQ AX, $0x04
|
||||
SETLT AL
|
||||
ORB AL, DL
|
||||
// exhausted += (br3.off < 4)
|
||||
CMPQ AX, $0x04
|
||||
ADCB $+0, DL
|
||||
|
||||
skip_fill3:
|
||||
// val0 := br3.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v0 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br3.advance(uint8(v0.entry)
|
||||
MOVB CH, AL
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
|
||||
// val1 := br3.peekTopBits(peekBits)
|
||||
MOVQ DI, CX
|
||||
MOVQ R12, R14
|
||||
SHRQ CL, R14
|
||||
MOVQ R11, R13
|
||||
SHRQ CL, R13
|
||||
|
||||
// v1 := table[val1&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br3.advance(uint8(v1.entry))
|
||||
MOVB CH, AH
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
|
||||
// these two writes get coalesced
|
||||
// out[id * dstEvery + 0] = uint8(v0.entry >> 8)
|
||||
// out[id * dstEvery + 1] = uint8(v1.entry >> 8)
|
||||
MOVW AX, (R8)
|
||||
LEAQ (R8)(R8*2), CX
|
||||
MOVW AX, (BX)(CX*1)
|
||||
|
||||
// update the bitreader structure
|
||||
MOVQ R12, 176(R11)
|
||||
MOVB R13, 184(R11)
|
||||
ADDQ $0x02, SI
|
||||
MOVQ R11, 176(R10)
|
||||
MOVB R12, 184(R10)
|
||||
ADDQ $0x02, BX
|
||||
TESTB DL, DL
|
||||
JZ main_loop
|
||||
MOVQ ctx+0(FP), AX
|
||||
SUBQ 16(AX), SI
|
||||
SHLQ $0x02, SI
|
||||
MOVQ SI, 40(AX)
|
||||
SUBQ 16(AX), BX
|
||||
SHLQ $0x02, BX
|
||||
MOVQ BX, 40(AX)
|
||||
RET
|
||||
|
||||
// func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext)
|
||||
TEXT ·decompress4x_8b_main_loop_amd64(SB), $0-8
|
||||
XORQ DX, DX
|
||||
|
||||
// Preload values
|
||||
MOVQ ctx+0(FP), CX
|
||||
MOVBQZX 8(CX), DI
|
||||
MOVQ 16(CX), BX
|
||||
MOVQ 48(CX), SI
|
||||
MOVQ 24(CX), R9
|
||||
MOVQ 32(CX), R10
|
||||
MOVQ (CX), R11
|
||||
MOVQ 24(CX), R8
|
||||
MOVQ 32(CX), R9
|
||||
MOVQ (CX), R10
|
||||
|
||||
// Main loop
|
||||
main_loop:
|
||||
MOVQ BX, R8
|
||||
CMPQ R8, SI
|
||||
XORL DX, DX
|
||||
CMPQ BX, SI
|
||||
SETGE DL
|
||||
|
||||
// br0.fillFast32()
|
||||
MOVQ 32(R11), R12
|
||||
MOVBQZX 40(R11), R13
|
||||
CMPQ R13, $0x20
|
||||
MOVQ 32(R10), R11
|
||||
MOVBQZX 40(R10), R12
|
||||
CMPQ R12, $0x20
|
||||
JBE skip_fill0
|
||||
MOVQ 24(R11), R14
|
||||
SUBQ $0x20, R13
|
||||
SUBQ $0x04, R14
|
||||
MOVQ (R11), R15
|
||||
MOVQ 24(R10), R13
|
||||
SUBQ $0x20, R12
|
||||
SUBQ $0x04, R13
|
||||
MOVQ (R10), R14
|
||||
|
||||
// b.value |= uint64(low) << (b.bitsRead & 63)
|
||||
MOVL (R14)(R15*1), R15
|
||||
MOVQ R13, CX
|
||||
SHLQ CL, R15
|
||||
MOVQ R14, 24(R11)
|
||||
ORQ R15, R12
|
||||
MOVL (R13)(R14*1), R14
|
||||
MOVQ R12, CX
|
||||
SHLQ CL, R14
|
||||
MOVQ R13, 24(R10)
|
||||
ORQ R14, R11
|
||||
|
||||
// exhausted = exhausted || (br0.off < 4)
|
||||
CMPQ R14, $0x04
|
||||
SETLT AL
|
||||
ORB AL, DL
|
||||
// exhausted += (br0.off < 4)
|
||||
CMPQ R13, $0x04
|
||||
ADCB $+0, DL
|
||||
|
||||
skip_fill0:
|
||||
// val0 := br0.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v0 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br0.advance(uint8(v0.entry)
|
||||
MOVB CH, AL
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
|
||||
// val1 := br0.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v1 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br0.advance(uint8(v1.entry)
|
||||
MOVB CH, AH
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
BSWAPL AX
|
||||
|
||||
// val2 := br0.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v2 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br0.advance(uint8(v2.entry)
|
||||
MOVB CH, AH
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
|
||||
// val3 := br0.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v3 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br0.advance(uint8(v3.entry)
|
||||
MOVB CH, AL
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
BSWAPL AX
|
||||
|
||||
// these four writes get coalesced
|
||||
@ -366,88 +354,86 @@ skip_fill0:
|
||||
// out[id * dstEvery + 1] = uint8(v1.entry >> 8)
|
||||
// out[id * dstEvery + 3] = uint8(v2.entry >> 8)
|
||||
// out[id * dstEvery + 4] = uint8(v3.entry >> 8)
|
||||
MOVL AX, (R8)
|
||||
MOVL AX, (BX)
|
||||
|
||||
// update the bitreader structure
|
||||
MOVQ R12, 32(R11)
|
||||
MOVB R13, 40(R11)
|
||||
ADDQ R9, R8
|
||||
MOVQ R11, 32(R10)
|
||||
MOVB R12, 40(R10)
|
||||
|
||||
// br1.fillFast32()
|
||||
MOVQ 80(R11), R12
|
||||
MOVBQZX 88(R11), R13
|
||||
CMPQ R13, $0x20
|
||||
MOVQ 80(R10), R11
|
||||
MOVBQZX 88(R10), R12
|
||||
CMPQ R12, $0x20
|
||||
JBE skip_fill1
|
||||
MOVQ 72(R11), R14
|
||||
SUBQ $0x20, R13
|
||||
SUBQ $0x04, R14
|
||||
MOVQ 48(R11), R15
|
||||
MOVQ 72(R10), R13
|
||||
SUBQ $0x20, R12
|
||||
SUBQ $0x04, R13
|
||||
MOVQ 48(R10), R14
|
||||
|
||||
// b.value |= uint64(low) << (b.bitsRead & 63)
|
||||
MOVL (R14)(R15*1), R15
|
||||
MOVQ R13, CX
|
||||
SHLQ CL, R15
|
||||
MOVQ R14, 72(R11)
|
||||
ORQ R15, R12
|
||||
MOVL (R13)(R14*1), R14
|
||||
MOVQ R12, CX
|
||||
SHLQ CL, R14
|
||||
MOVQ R13, 72(R10)
|
||||
ORQ R14, R11
|
||||
|
||||
// exhausted = exhausted || (br1.off < 4)
|
||||
CMPQ R14, $0x04
|
||||
SETLT AL
|
||||
ORB AL, DL
|
||||
// exhausted += (br1.off < 4)
|
||||
CMPQ R13, $0x04
|
||||
ADCB $+0, DL
|
||||
|
||||
skip_fill1:
|
||||
// val0 := br1.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v0 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br1.advance(uint8(v0.entry)
|
||||
MOVB CH, AL
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
|
||||
// val1 := br1.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v1 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br1.advance(uint8(v1.entry)
|
||||
MOVB CH, AH
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
BSWAPL AX
|
||||
|
||||
// val2 := br1.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v2 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br1.advance(uint8(v2.entry)
|
||||
MOVB CH, AH
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
|
||||
// val3 := br1.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v3 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br1.advance(uint8(v3.entry)
|
||||
MOVB CH, AL
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
BSWAPL AX
|
||||
|
||||
// these four writes get coalesced
|
||||
@ -455,88 +441,86 @@ skip_fill1:
|
||||
// out[id * dstEvery + 1] = uint8(v1.entry >> 8)
|
||||
// out[id * dstEvery + 3] = uint8(v2.entry >> 8)
|
||||
// out[id * dstEvery + 4] = uint8(v3.entry >> 8)
|
||||
MOVL AX, (R8)
|
||||
MOVL AX, (BX)(R8*1)
|
||||
|
||||
// update the bitreader structure
|
||||
MOVQ R12, 80(R11)
|
||||
MOVB R13, 88(R11)
|
||||
ADDQ R9, R8
|
||||
MOVQ R11, 80(R10)
|
||||
MOVB R12, 88(R10)
|
||||
|
||||
// br2.fillFast32()
|
||||
MOVQ 128(R11), R12
|
||||
MOVBQZX 136(R11), R13
|
||||
CMPQ R13, $0x20
|
||||
MOVQ 128(R10), R11
|
||||
MOVBQZX 136(R10), R12
|
||||
CMPQ R12, $0x20
|
||||
JBE skip_fill2
|
||||
MOVQ 120(R11), R14
|
||||
SUBQ $0x20, R13
|
||||
SUBQ $0x04, R14
|
||||
MOVQ 96(R11), R15
|
||||
MOVQ 120(R10), R13
|
||||
SUBQ $0x20, R12
|
||||
SUBQ $0x04, R13
|
||||
MOVQ 96(R10), R14
|
||||
|
||||
// b.value |= uint64(low) << (b.bitsRead & 63)
|
||||
MOVL (R14)(R15*1), R15
|
||||
MOVQ R13, CX
|
||||
SHLQ CL, R15
|
||||
MOVQ R14, 120(R11)
|
||||
ORQ R15, R12
|
||||
MOVL (R13)(R14*1), R14
|
||||
MOVQ R12, CX
|
||||
SHLQ CL, R14
|
||||
MOVQ R13, 120(R10)
|
||||
ORQ R14, R11
|
||||
|
||||
// exhausted = exhausted || (br2.off < 4)
|
||||
CMPQ R14, $0x04
|
||||
SETLT AL
|
||||
ORB AL, DL
|
||||
// exhausted += (br2.off < 4)
|
||||
CMPQ R13, $0x04
|
||||
ADCB $+0, DL
|
||||
|
||||
skip_fill2:
|
||||
// val0 := br2.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v0 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br2.advance(uint8(v0.entry)
|
||||
MOVB CH, AL
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
|
||||
// val1 := br2.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v1 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br2.advance(uint8(v1.entry)
|
||||
MOVB CH, AH
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
BSWAPL AX
|
||||
|
||||
// val2 := br2.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v2 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br2.advance(uint8(v2.entry)
|
||||
MOVB CH, AH
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
|
||||
// val3 := br2.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v3 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br2.advance(uint8(v3.entry)
|
||||
MOVB CH, AL
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
BSWAPL AX
|
||||
|
||||
// these four writes get coalesced
|
||||
@ -544,88 +528,86 @@ skip_fill2:
|
||||
// out[id * dstEvery + 1] = uint8(v1.entry >> 8)
|
||||
// out[id * dstEvery + 3] = uint8(v2.entry >> 8)
|
||||
// out[id * dstEvery + 4] = uint8(v3.entry >> 8)
|
||||
MOVL AX, (R8)
|
||||
MOVL AX, (BX)(R8*2)
|
||||
|
||||
// update the bitreader structure
|
||||
MOVQ R12, 128(R11)
|
||||
MOVB R13, 136(R11)
|
||||
ADDQ R9, R8
|
||||
MOVQ R11, 128(R10)
|
||||
MOVB R12, 136(R10)
|
||||
|
||||
// br3.fillFast32()
|
||||
MOVQ 176(R11), R12
|
||||
MOVBQZX 184(R11), R13
|
||||
CMPQ R13, $0x20
|
||||
MOVQ 176(R10), R11
|
||||
MOVBQZX 184(R10), R12
|
||||
CMPQ R12, $0x20
|
||||
JBE skip_fill3
|
||||
MOVQ 168(R11), R14
|
||||
SUBQ $0x20, R13
|
||||
SUBQ $0x04, R14
|
||||
MOVQ 144(R11), R15
|
||||
MOVQ 168(R10), R13
|
||||
SUBQ $0x20, R12
|
||||
SUBQ $0x04, R13
|
||||
MOVQ 144(R10), R14
|
||||
|
||||
// b.value |= uint64(low) << (b.bitsRead & 63)
|
||||
MOVL (R14)(R15*1), R15
|
||||
MOVQ R13, CX
|
||||
SHLQ CL, R15
|
||||
MOVQ R14, 168(R11)
|
||||
ORQ R15, R12
|
||||
MOVL (R13)(R14*1), R14
|
||||
MOVQ R12, CX
|
||||
SHLQ CL, R14
|
||||
MOVQ R13, 168(R10)
|
||||
ORQ R14, R11
|
||||
|
||||
// exhausted = exhausted || (br3.off < 4)
|
||||
CMPQ R14, $0x04
|
||||
SETLT AL
|
||||
ORB AL, DL
|
||||
// exhausted += (br3.off < 4)
|
||||
CMPQ R13, $0x04
|
||||
ADCB $+0, DL
|
||||
|
||||
skip_fill3:
|
||||
// val0 := br3.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v0 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br3.advance(uint8(v0.entry)
|
||||
MOVB CH, AL
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
|
||||
// val1 := br3.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v1 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br3.advance(uint8(v1.entry)
|
||||
MOVB CH, AH
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
BSWAPL AX
|
||||
|
||||
// val2 := br3.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v2 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br3.advance(uint8(v2.entry)
|
||||
MOVB CH, AH
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
|
||||
// val3 := br3.peekTopBits(peekBits)
|
||||
MOVQ R12, R14
|
||||
MOVQ R11, R13
|
||||
MOVQ DI, CX
|
||||
SHRQ CL, R14
|
||||
SHRQ CL, R13
|
||||
|
||||
// v3 := table[val0&mask]
|
||||
MOVW (R10)(R14*2), CX
|
||||
MOVW (R9)(R13*2), CX
|
||||
|
||||
// br3.advance(uint8(v3.entry)
|
||||
MOVB CH, AL
|
||||
SHLQ CL, R12
|
||||
ADDB CL, R13
|
||||
SHLQ CL, R11
|
||||
ADDB CL, R12
|
||||
BSWAPL AX
|
||||
|
||||
// these four writes get coalesced
|
||||
@ -633,11 +615,12 @@ skip_fill3:
|
||||
// out[id * dstEvery + 1] = uint8(v1.entry >> 8)
|
||||
// out[id * dstEvery + 3] = uint8(v2.entry >> 8)
|
||||
// out[id * dstEvery + 4] = uint8(v3.entry >> 8)
|
||||
MOVL AX, (R8)
|
||||
LEAQ (R8)(R8*2), CX
|
||||
MOVL AX, (BX)(CX*1)
|
||||
|
||||
// update the bitreader structure
|
||||
MOVQ R12, 176(R11)
|
||||
MOVB R13, 184(R11)
|
||||
MOVQ R11, 176(R10)
|
||||
MOVB R12, 184(R10)
|
||||
ADDQ $0x04, BX
|
||||
TESTB DL, DL
|
||||
JZ main_loop
|
||||
@ -653,7 +636,7 @@ TEXT ·decompress1x_main_loop_amd64(SB), $0-8
|
||||
MOVQ 16(CX), DX
|
||||
MOVQ 24(CX), BX
|
||||
CMPQ BX, $0x04
|
||||
JB error_max_decoded_size_exeeded
|
||||
JB error_max_decoded_size_exceeded
|
||||
LEAQ (DX)(BX*1), BX
|
||||
MOVQ (CX), SI
|
||||
MOVQ (SI), R8
|
||||
@ -668,7 +651,7 @@ main_loop:
|
||||
// Check if we have room for 4 bytes in the output buffer
|
||||
LEAQ 4(DX), CX
|
||||
CMPQ CX, BX
|
||||
JGE error_max_decoded_size_exeeded
|
||||
JGE error_max_decoded_size_exceeded
|
||||
|
||||
// Decode 4 values
|
||||
CMPQ R11, $0x20
|
||||
@ -745,7 +728,7 @@ loop_condition:
|
||||
RET
|
||||
|
||||
// Report error
|
||||
error_max_decoded_size_exeeded:
|
||||
error_max_decoded_size_exceeded:
|
||||
MOVQ ctx+0(FP), AX
|
||||
MOVQ $-1, CX
|
||||
MOVQ CX, 40(AX)
|
||||
@ -758,7 +741,7 @@ TEXT ·decompress1x_main_loop_bmi2(SB), $0-8
|
||||
MOVQ 16(CX), DX
|
||||
MOVQ 24(CX), BX
|
||||
CMPQ BX, $0x04
|
||||
JB error_max_decoded_size_exeeded
|
||||
JB error_max_decoded_size_exceeded
|
||||
LEAQ (DX)(BX*1), BX
|
||||
MOVQ (CX), SI
|
||||
MOVQ (SI), R8
|
||||
@ -773,7 +756,7 @@ main_loop:
|
||||
// Check if we have room for 4 bytes in the output buffer
|
||||
LEAQ 4(DX), CX
|
||||
CMPQ CX, BX
|
||||
JGE error_max_decoded_size_exeeded
|
||||
JGE error_max_decoded_size_exceeded
|
||||
|
||||
// Decode 4 values
|
||||
CMPQ R11, $0x20
|
||||
@ -840,7 +823,7 @@ loop_condition:
|
||||
RET
|
||||
|
||||
// Report error
|
||||
error_max_decoded_size_exeeded:
|
||||
error_max_decoded_size_exceeded:
|
||||
MOVQ ctx+0(FP), AX
|
||||
MOVQ $-1, CX
|
||||
MOVQ CX, 40(AX)
|
||||
|
||||
18
vendor/github.com/klauspost/compress/huff0/decompress_generic.go
generated
vendored
18
vendor/github.com/klauspost/compress/huff0/decompress_generic.go
generated
vendored
@ -122,17 +122,21 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) {
|
||||
d.bufs.Put(buf)
|
||||
return nil, errors.New("corruption detected: stream overrun 1")
|
||||
}
|
||||
copy(out, buf[0][:])
|
||||
copy(out[dstEvery:], buf[1][:])
|
||||
copy(out[dstEvery*2:], buf[2][:])
|
||||
copy(out[dstEvery*3:], buf[3][:])
|
||||
out = out[bufoff:]
|
||||
decoded += bufoff * 4
|
||||
// There must at least be 3 buffers left.
|
||||
if len(out) < dstEvery*3 {
|
||||
if len(out)-bufoff < dstEvery*3 {
|
||||
d.bufs.Put(buf)
|
||||
return nil, errors.New("corruption detected: stream overrun 2")
|
||||
}
|
||||
//copy(out, buf[0][:])
|
||||
//copy(out[dstEvery:], buf[1][:])
|
||||
//copy(out[dstEvery*2:], buf[2][:])
|
||||
//copy(out[dstEvery*3:], buf[3][:])
|
||||
*(*[bufoff]byte)(out) = buf[0]
|
||||
*(*[bufoff]byte)(out[dstEvery:]) = buf[1]
|
||||
*(*[bufoff]byte)(out[dstEvery*2:]) = buf[2]
|
||||
*(*[bufoff]byte)(out[dstEvery*3:]) = buf[3]
|
||||
out = out[bufoff:]
|
||||
decoded += bufoff * 4
|
||||
}
|
||||
}
|
||||
if off > 0 {
|
||||
|
||||
28
vendor/github.com/klauspost/compress/internal/snapref/encode_other.go
generated
vendored
28
vendor/github.com/klauspost/compress/internal/snapref/encode_other.go
generated
vendored
@ -18,6 +18,7 @@ func load64(b []byte, i int) uint64 {
|
||||
// emitLiteral writes a literal chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= len(lit) && len(lit) <= 65536
|
||||
func emitLiteral(dst, lit []byte) int {
|
||||
@ -42,6 +43,7 @@ func emitLiteral(dst, lit []byte) int {
|
||||
// emitCopy writes a copy chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= offset && offset <= 65535
|
||||
// 4 <= length && length <= 65535
|
||||
@ -89,6 +91,7 @@ func emitCopy(dst []byte, offset, length int) int {
|
||||
// src[i:i+k-j] and src[j:k] have the same contents.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// 0 <= i && i < j && j <= len(src)
|
||||
func extendMatch(src []byte, i, j int) int {
|
||||
for ; j < len(src) && src[i] == src[j]; i, j = i+1, j+1 {
|
||||
@ -100,13 +103,36 @@ func hash(u, shift uint32) uint32 {
|
||||
return (u * 0x1e35a7bd) >> shift
|
||||
}
|
||||
|
||||
// EncodeBlockInto exposes encodeBlock but checks dst size.
|
||||
func EncodeBlockInto(dst, src []byte) (d int) {
|
||||
if MaxEncodedLen(len(src)) > len(dst) {
|
||||
return 0
|
||||
}
|
||||
|
||||
// encodeBlock breaks on too big blocks, so split.
|
||||
for len(src) > 0 {
|
||||
p := src
|
||||
src = nil
|
||||
if len(p) > maxBlockSize {
|
||||
p, src = p[:maxBlockSize], p[maxBlockSize:]
|
||||
}
|
||||
if len(p) < minNonLiteralBlockSize {
|
||||
d += emitLiteral(dst[d:], p)
|
||||
} else {
|
||||
d += encodeBlock(dst[d:], p)
|
||||
}
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlock(dst, src []byte) (d int) {
|
||||
// Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive.
|
||||
// The table element type is uint16, as s < sLimit and sLimit < len(src)
|
||||
|
||||
465
vendor/github.com/klauspost/compress/s2/README.md
generated
vendored
465
vendor/github.com/klauspost/compress/s2/README.md
generated
vendored
@ -20,11 +20,12 @@ This is important, so you don't have to worry about spending CPU cycles on alrea
|
||||
* Concurrent stream compression
|
||||
* Faster decompression, even for Snappy compatible content
|
||||
* Concurrent Snappy/S2 stream decompression
|
||||
* Ability to quickly skip forward in compressed stream
|
||||
* Skip forward in compressed stream
|
||||
* Random seeking with indexes
|
||||
* Compatible with reading Snappy compressed content
|
||||
* Smaller block size overhead on incompressible blocks
|
||||
* Block concatenation
|
||||
* Block Dictionary support
|
||||
* Uncompressed stream mode
|
||||
* Automatic stream size padding
|
||||
* Snappy compatible block compression
|
||||
@ -325,35 +326,35 @@ The content compressed in this mode is fully compatible with the standard decode
|
||||
|
||||
Snappy vs S2 **compression** speed on 16 core (32 thread) computer, using all threads and a single thread (1 CPU):
|
||||
|
||||
| File | S2 speed | S2 Throughput | S2 % smaller | S2 "better" | "better" throughput | "better" % smaller |
|
||||
|-----------------------------------------------------------------------------------------------------|----------|---------------|--------------|-------------|---------------------|--------------------|
|
||||
| [rawstudio-mint14.tar](https://files.klauspost.com/compress/rawstudio-mint14.7z) | 12.70x | 10556 MB/s | 7.35% | 4.15x | 3455 MB/s | 12.79% |
|
||||
| (1 CPU) | 1.14x | 948 MB/s | - | 0.42x | 349 MB/s | - |
|
||||
| [github-june-2days-2019.json](https://files.klauspost.com/compress/github-june-2days-2019.json.zst) | 17.13x | 14484 MB/s | 31.60% | 10.09x | 8533 MB/s | 37.71% |
|
||||
| (1 CPU) | 1.33x | 1127 MB/s | - | 0.70x | 589 MB/s | - |
|
||||
| [github-ranks-backup.bin](https://files.klauspost.com/compress/github-ranks-backup.bin.zst) | 15.14x | 12000 MB/s | -5.79% | 6.59x | 5223 MB/s | 5.80% |
|
||||
| (1 CPU) | 1.11x | 877 MB/s | - | 0.47x | 370 MB/s | - |
|
||||
| [consensus.db.10gb](https://files.klauspost.com/compress/consensus.db.10gb.zst) | 14.62x | 12116 MB/s | 15.90% | 5.35x | 4430 MB/s | 16.08% |
|
||||
| (1 CPU) | 1.38x | 1146 MB/s | - | 0.38x | 312 MB/s | - |
|
||||
| [adresser.json](https://files.klauspost.com/compress/adresser.json.zst) | 8.83x | 17579 MB/s | 43.86% | 6.54x | 13011 MB/s | 47.23% |
|
||||
| (1 CPU) | 1.14x | 2259 MB/s | - | 0.74x | 1475 MB/s | - |
|
||||
| [gob-stream](https://files.klauspost.com/compress/gob-stream.7z) | 16.72x | 14019 MB/s | 24.02% | 10.11x | 8477 MB/s | 30.48% |
|
||||
| (1 CPU) | 1.24x | 1043 MB/s | - | 0.70x | 586 MB/s | - |
|
||||
| [10gb.tar](http://mattmahoney.net/dc/10gb.html) | 13.33x | 9254 MB/s | 1.84% | 6.75x | 4686 MB/s | 6.72% |
|
||||
| (1 CPU) | 0.97x | 672 MB/s | - | 0.53x | 366 MB/s | - |
|
||||
| sharnd.out.2gb | 2.11x | 12639 MB/s | 0.01% | 1.98x | 11833 MB/s | 0.01% |
|
||||
| (1 CPU) | 0.93x | 5594 MB/s | - | 1.34x | 8030 MB/s | - |
|
||||
| [enwik9](http://mattmahoney.net/dc/textdata.html) | 19.34x | 8220 MB/s | 3.98% | 7.87x | 3345 MB/s | 15.82% |
|
||||
| (1 CPU) | 1.06x | 452 MB/s | - | 0.50x | 213 MB/s | - |
|
||||
| [silesia.tar](http://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip) | 10.48x | 6124 MB/s | 5.67% | 3.76x | 2197 MB/s | 12.60% |
|
||||
| (1 CPU) | 0.97x | 568 MB/s | - | 0.46x | 271 MB/s | - |
|
||||
| [enwik10](https://encode.su/threads/3315-enwik10-benchmark-results) | 21.07x | 9020 MB/s | 6.36% | 6.91x | 2959 MB/s | 16.95% |
|
||||
| (1 CPU) | 1.07x | 460 MB/s | - | 0.51x | 220 MB/s | - |
|
||||
| File | S2 Speed | S2 Throughput | S2 % smaller | S2 "better" | "better" throughput | "better" % smaller |
|
||||
|---------------------------------------------------------------------------------------------------------|----------|---------------|--------------|-------------|---------------------|--------------------|
|
||||
| [rawstudio-mint14.tar](https://files.klauspost.com/compress/rawstudio-mint14.7z) | 16.33x | 10556 MB/s | 8.0% | 6.04x | 5252 MB/s | 14.7% |
|
||||
| (1 CPU) | 1.08x | 940 MB/s | - | 0.46x | 400 MB/s | - |
|
||||
| [github-june-2days-2019.json](https://files.klauspost.com/compress/github-june-2days-2019.json.zst) | 16.51x | 15224 MB/s | 31.70% | 9.47x | 8734 MB/s | 37.71% |
|
||||
| (1 CPU) | 1.26x | 1157 MB/s | - | 0.60x | 556 MB/s | - |
|
||||
| [github-ranks-backup.bin](https://files.klauspost.com/compress/github-ranks-backup.bin.zst) | 15.14x | 12598 MB/s | -5.76% | 6.23x | 5675 MB/s | 3.62% |
|
||||
| (1 CPU) | 1.02x | 932 MB/s | - | 0.47x | 432 MB/s | - |
|
||||
| [consensus.db.10gb](https://files.klauspost.com/compress/consensus.db.10gb.zst) | 11.21x | 12116 MB/s | 15.95% | 3.24x | 3500 MB/s | 18.00% |
|
||||
| (1 CPU) | 1.05x | 1135 MB/s | - | 0.27x | 292 MB/s | - |
|
||||
| [apache.log](https://files.klauspost.com/compress/apache.log.zst) | 8.55x | 16673 MB/s | 20.54% | 5.85x | 11420 MB/s | 24.97% |
|
||||
| (1 CPU) | 1.91x | 1771 MB/s | - | 0.53x | 1041 MB/s | - |
|
||||
| [gob-stream](https://files.klauspost.com/compress/gob-stream.7z) | 15.76x | 14357 MB/s | 24.01% | 8.67x | 7891 MB/s | 33.68% |
|
||||
| (1 CPU) | 1.17x | 1064 MB/s | - | 0.65x | 595 MB/s | - |
|
||||
| [10gb.tar](http://mattmahoney.net/dc/10gb.html) | 13.33x | 9835 MB/s | 2.34% | 6.85x | 4863 MB/s | 9.96% |
|
||||
| (1 CPU) | 0.97x | 689 MB/s | - | 0.55x | 387 MB/s | - |
|
||||
| sharnd.out.2gb | 9.11x | 13213 MB/s | 0.01% | 1.49x | 9184 MB/s | 0.01% |
|
||||
| (1 CPU) | 0.88x | 5418 MB/s | - | 0.77x | 5417 MB/s | - |
|
||||
| [sofia-air-quality-dataset csv](https://files.klauspost.com/compress/sofia-air-quality-dataset.tar.zst) | 22.00x | 11477 MB/s | 18.73% | 11.15x | 5817 MB/s | 27.88% |
|
||||
| (1 CPU) | 1.23x | 642 MB/s | - | 0.71x | 642 MB/s | - |
|
||||
| [silesia.tar](http://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip) | 11.23x | 6520 MB/s | 5.9% | 5.35x | 3109 MB/s | 15.88% |
|
||||
| (1 CPU) | 1.05x | 607 MB/s | - | 0.52x | 304 MB/s | - |
|
||||
| [enwik9](https://files.klauspost.com/compress/enwik9.zst) | 19.28x | 8440 MB/s | 4.04% | 9.31x | 4076 MB/s | 18.04% |
|
||||
| (1 CPU) | 1.12x | 488 MB/s | - | 0.57x | 250 MB/s | - |
|
||||
|
||||
### Legend
|
||||
|
||||
* `S2 speed`: Speed of S2 compared to Snappy, using 16 cores and 1 core.
|
||||
* `S2 throughput`: Throughput of S2 in MB/s.
|
||||
* `S2 Speed`: Speed of S2 compared to Snappy, using 16 cores and 1 core.
|
||||
* `S2 Throughput`: Throughput of S2 in MB/s.
|
||||
* `S2 % smaller`: How many percent of the Snappy output size is S2 better.
|
||||
* `S2 "better"`: Speed when enabling "better" compression mode in S2 compared to Snappy.
|
||||
* `"better" throughput`: Speed when enabling "better" compression mode in S2 compared to Snappy.
|
||||
@ -361,7 +362,7 @@ Snappy vs S2 **compression** speed on 16 core (32 thread) computer, using all th
|
||||
|
||||
There is a good speedup across the board when using a single thread and a significant speedup when using multiple threads.
|
||||
|
||||
Machine generated data gets by far the biggest compression boost, with size being being reduced by up to 45% of Snappy size.
|
||||
Machine generated data gets by far the biggest compression boost, with size being reduced by up to 35% of Snappy size.
|
||||
|
||||
The "better" compression mode sees a good improvement in all cases, but usually at a performance cost.
|
||||
|
||||
@ -404,15 +405,15 @@ The "better" compression mode will actively look for shorter matches, which is w
|
||||
Without assembly decompression is also very fast; single goroutine decompression speed. No assembly:
|
||||
|
||||
| File | S2 Throughput | S2 throughput |
|
||||
|--------------------------------|--------------|---------------|
|
||||
| consensus.db.10gb.s2 | 1.84x | 2289.8 MB/s |
|
||||
| 10gb.tar.s2 | 1.30x | 867.07 MB/s |
|
||||
| rawstudio-mint14.tar.s2 | 1.66x | 1329.65 MB/s |
|
||||
| github-june-2days-2019.json.s2 | 2.36x | 1831.59 MB/s |
|
||||
| github-ranks-backup.bin.s2 | 1.73x | 1390.7 MB/s |
|
||||
| enwik9.s2 | 1.67x | 681.53 MB/s |
|
||||
| adresser.json.s2 | 3.41x | 4230.53 MB/s |
|
||||
| silesia.tar.s2 | 1.52x | 811.58 |
|
||||
|--------------------------------|---------------|---------------|
|
||||
| consensus.db.10gb.s2 | 1.84x | 2289.8 MB/s |
|
||||
| 10gb.tar.s2 | 1.30x | 867.07 MB/s |
|
||||
| rawstudio-mint14.tar.s2 | 1.66x | 1329.65 MB/s |
|
||||
| github-june-2days-2019.json.s2 | 2.36x | 1831.59 MB/s |
|
||||
| github-ranks-backup.bin.s2 | 1.73x | 1390.7 MB/s |
|
||||
| enwik9.s2 | 1.67x | 681.53 MB/s |
|
||||
| adresser.json.s2 | 3.41x | 4230.53 MB/s |
|
||||
| silesia.tar.s2 | 1.52x | 811.58 |
|
||||
|
||||
Even though S2 typically compresses better than Snappy, decompression speed is always better.
|
||||
|
||||
@ -450,14 +451,14 @@ The most reliable is a wide dataset.
|
||||
For this we use [`webdevdata.org-2015-01-07-subset`](https://files.klauspost.com/compress/webdevdata.org-2015-01-07-4GB-subset.7z),
|
||||
53927 files, total input size: 4,014,735,833 bytes. Single goroutine used.
|
||||
|
||||
| * | Input | Output | Reduction | MB/s |
|
||||
|-------------------|------------|------------|-----------|--------|
|
||||
| S2 | 4014735833 | 1059723369 | 73.60% | **934.34** |
|
||||
| S2 Better | 4014735833 | 969670507 | 75.85% | 532.70 |
|
||||
| S2 Best | 4014735833 | 906625668 | **77.85%** | 46.84 |
|
||||
| Snappy | 4014735833 | 1128706759 | 71.89% | 762.59 |
|
||||
| S2, Snappy Output | 4014735833 | 1093821420 | 72.75% | 908.60 |
|
||||
| LZ4 | 4014735833 | 1079259294 | 73.12% | 526.94 |
|
||||
| * | Input | Output | Reduction | MB/s |
|
||||
|-------------------|------------|------------|------------|------------|
|
||||
| S2 | 4014735833 | 1059723369 | 73.60% | **936.73** |
|
||||
| S2 Better | 4014735833 | 961580539 | 76.05% | 451.10 |
|
||||
| S2 Best | 4014735833 | 899182886 | **77.60%** | 46.84 |
|
||||
| Snappy | 4014735833 | 1128706759 | 71.89% | 790.15 |
|
||||
| S2, Snappy Output | 4014735833 | 1093823291 | 72.75% | 936.60 |
|
||||
| LZ4 | 4014735833 | 1063768713 | 73.50% | 452.02 |
|
||||
|
||||
S2 delivers both the best single threaded throughput with regular mode and the best compression rate with "best".
|
||||
"Better" mode provides the same compression speed as LZ4 with better compression ratio.
|
||||
@ -489,43 +490,24 @@ AMD64 assembly is use for both S2 and Snappy.
|
||||
|
||||
| Absolute Perf | Snappy size | S2 Size | Snappy Speed | S2 Speed | Snappy dec | S2 dec |
|
||||
|-----------------------|-------------|---------|--------------|-------------|-------------|-------------|
|
||||
| html | 22843 | 21111 | 16246 MB/s | 17438 MB/s | 40972 MB/s | 49263 MB/s |
|
||||
| urls.10K | 335492 | 287326 | 7943 MB/s | 9693 MB/s | 22523 MB/s | 26484 MB/s |
|
||||
| fireworks.jpeg | 123034 | 123100 | 349544 MB/s | 273889 MB/s | 718321 MB/s | 827552 MB/s |
|
||||
| fireworks.jpeg (200B) | 146 | 155 | 8869 MB/s | 17773 MB/s | 33691 MB/s | 52421 MB/s |
|
||||
| paper-100k.pdf | 85304 | 84459 | 167546 MB/s | 101263 MB/s | 326905 MB/s | 291944 MB/s |
|
||||
| html_x_4 | 92234 | 21113 | 15194 MB/s | 50670 MB/s | 30843 MB/s | 32217 MB/s |
|
||||
| alice29.txt | 88034 | 85975 | 5936 MB/s | 6139 MB/s | 12882 MB/s | 20044 MB/s |
|
||||
| asyoulik.txt | 77503 | 79650 | 5517 MB/s | 6366 MB/s | 12735 MB/s | 22806 MB/s |
|
||||
| lcet10.txt | 234661 | 220670 | 6235 MB/s | 6067 MB/s | 14519 MB/s | 18697 MB/s |
|
||||
| plrabn12.txt | 319267 | 317985 | 5159 MB/s | 5726 MB/s | 11923 MB/s | 19901 MB/s |
|
||||
| geo.protodata | 23335 | 18690 | 21220 MB/s | 26529 MB/s | 56271 MB/s | 62540 MB/s |
|
||||
| kppkn.gtb | 69526 | 65312 | 9732 MB/s | 8559 MB/s | 18491 MB/s | 18969 MB/s |
|
||||
| alice29.txt (128B) | 80 | 82 | 6691 MB/s | 15489 MB/s | 31883 MB/s | 38874 MB/s |
|
||||
| alice29.txt (1000B) | 774 | 774 | 12204 MB/s | 13000 MB/s | 48056 MB/s | 52341 MB/s |
|
||||
| alice29.txt (10000B) | 6648 | 6933 | 10044 MB/s | 12806 MB/s | 32378 MB/s | 46322 MB/s |
|
||||
| alice29.txt (20000B) | 12686 | 13574 | 7733 MB/s | 11210 MB/s | 30566 MB/s | 58969 MB/s |
|
||||
| html | 22843 | 20868 | 16246 MB/s | 18617 MB/s | 40972 MB/s | 49263 MB/s |
|
||||
| urls.10K | 335492 | 286541 | 7943 MB/s | 10201 MB/s | 22523 MB/s | 26484 MB/s |
|
||||
| fireworks.jpeg | 123034 | 123100 | 349544 MB/s | 303228 MB/s | 718321 MB/s | 827552 MB/s |
|
||||
| fireworks.jpeg (200B) | 146 | 155 | 8869 MB/s | 20180 MB/s | 33691 MB/s | 52421 MB/s |
|
||||
| paper-100k.pdf | 85304 | 84202 | 167546 MB/s | 112988 MB/s | 326905 MB/s | 291944 MB/s |
|
||||
| html_x_4 | 92234 | 20870 | 15194 MB/s | 54457 MB/s | 30843 MB/s | 32217 MB/s |
|
||||
| alice29.txt | 88034 | 85934 | 5936 MB/s | 6540 MB/s | 12882 MB/s | 20044 MB/s |
|
||||
| asyoulik.txt | 77503 | 79575 | 5517 MB/s | 6657 MB/s | 12735 MB/s | 22806 MB/s |
|
||||
| lcet10.txt | 234661 | 220383 | 6235 MB/s | 6303 MB/s | 14519 MB/s | 18697 MB/s |
|
||||
| plrabn12.txt | 319267 | 318196 | 5159 MB/s | 6074 MB/s | 11923 MB/s | 19901 MB/s |
|
||||
| geo.protodata | 23335 | 18606 | 21220 MB/s | 25432 MB/s | 56271 MB/s | 62540 MB/s |
|
||||
| kppkn.gtb | 69526 | 65019 | 9732 MB/s | 8905 MB/s | 18491 MB/s | 18969 MB/s |
|
||||
| alice29.txt (128B) | 80 | 82 | 6691 MB/s | 17179 MB/s | 31883 MB/s | 38874 MB/s |
|
||||
| alice29.txt (1000B) | 774 | 774 | 12204 MB/s | 13273 MB/s | 48056 MB/s | 52341 MB/s |
|
||||
| alice29.txt (10000B) | 6648 | 6933 | 10044 MB/s | 12824 MB/s | 32378 MB/s | 46322 MB/s |
|
||||
| alice29.txt (20000B) | 12686 | 13516 | 7733 MB/s | 12160 MB/s | 30566 MB/s | 58969 MB/s |
|
||||
|
||||
|
||||
| Relative Perf | Snappy size | S2 size improved | S2 Speed | S2 Dec Speed |
|
||||
|-----------------------|-------------|------------------|----------|--------------|
|
||||
| html | 22.31% | 7.58% | 1.07x | 1.20x |
|
||||
| urls.10K | 47.78% | 14.36% | 1.22x | 1.18x |
|
||||
| fireworks.jpeg | 99.95% | -0.05% | 0.78x | 1.15x |
|
||||
| fireworks.jpeg (200B) | 73.00% | -6.16% | 2.00x | 1.56x |
|
||||
| paper-100k.pdf | 83.30% | 0.99% | 0.60x | 0.89x |
|
||||
| html_x_4 | 22.52% | 77.11% | 3.33x | 1.04x |
|
||||
| alice29.txt | 57.88% | 2.34% | 1.03x | 1.56x |
|
||||
| asyoulik.txt | 61.91% | -2.77% | 1.15x | 1.79x |
|
||||
| lcet10.txt | 54.99% | 5.96% | 0.97x | 1.29x |
|
||||
| plrabn12.txt | 66.26% | 0.40% | 1.11x | 1.67x |
|
||||
| geo.protodata | 19.68% | 19.91% | 1.25x | 1.11x |
|
||||
| kppkn.gtb | 37.72% | 6.06% | 0.88x | 1.03x |
|
||||
| alice29.txt (128B) | 62.50% | -2.50% | 2.31x | 1.22x |
|
||||
| alice29.txt (1000B) | 77.40% | 0.00% | 1.07x | 1.09x |
|
||||
| alice29.txt (10000B) | 66.48% | -4.29% | 1.27x | 1.43x |
|
||||
| alice29.txt (20000B) | 63.43% | -7.00% | 1.45x | 1.93x |
|
||||
|
||||
Speed is generally at or above Snappy. Small blocks gets a significant speedup, although at the expense of size.
|
||||
|
||||
Decompression speed is better than Snappy, except in one case.
|
||||
@ -543,43 +525,24 @@ So individual benchmarks should only be seen as a guideline and the overall pict
|
||||
|
||||
| Absolute Perf | Snappy size | Better Size | Snappy Speed | Better Speed | Snappy dec | Better dec |
|
||||
|-----------------------|-------------|-------------|--------------|--------------|-------------|-------------|
|
||||
| html | 22843 | 19833 | 16246 MB/s | 7731 MB/s | 40972 MB/s | 40292 MB/s |
|
||||
| urls.10K | 335492 | 253529 | 7943 MB/s | 3980 MB/s | 22523 MB/s | 20981 MB/s |
|
||||
| fireworks.jpeg | 123034 | 123100 | 349544 MB/s | 9760 MB/s | 718321 MB/s | 823698 MB/s |
|
||||
| fireworks.jpeg (200B) | 146 | 142 | 8869 MB/s | 594 MB/s | 33691 MB/s | 30101 MB/s |
|
||||
| paper-100k.pdf | 85304 | 82915 | 167546 MB/s | 7470 MB/s | 326905 MB/s | 198869 MB/s |
|
||||
| html_x_4 | 92234 | 19841 | 15194 MB/s | 23403 MB/s | 30843 MB/s | 30937 MB/s |
|
||||
| alice29.txt | 88034 | 73218 | 5936 MB/s | 2945 MB/s | 12882 MB/s | 16611 MB/s |
|
||||
| asyoulik.txt | 77503 | 66844 | 5517 MB/s | 2739 MB/s | 12735 MB/s | 14975 MB/s |
|
||||
| lcet10.txt | 234661 | 190589 | 6235 MB/s | 3099 MB/s | 14519 MB/s | 16634 MB/s |
|
||||
| plrabn12.txt | 319267 | 270828 | 5159 MB/s | 2600 MB/s | 11923 MB/s | 13382 MB/s |
|
||||
| geo.protodata | 23335 | 18278 | 21220 MB/s | 11208 MB/s | 56271 MB/s | 57961 MB/s |
|
||||
| kppkn.gtb | 69526 | 61851 | 9732 MB/s | 4556 MB/s | 18491 MB/s | 16524 MB/s |
|
||||
| alice29.txt (128B) | 80 | 81 | 6691 MB/s | 529 MB/s | 31883 MB/s | 34225 MB/s |
|
||||
| alice29.txt (1000B) | 774 | 748 | 12204 MB/s | 1943 MB/s | 48056 MB/s | 42068 MB/s |
|
||||
| alice29.txt (10000B) | 6648 | 6234 | 10044 MB/s | 2949 MB/s | 32378 MB/s | 28813 MB/s |
|
||||
| alice29.txt (20000B) | 12686 | 11584 | 7733 MB/s | 2822 MB/s | 30566 MB/s | 27315 MB/s |
|
||||
| html | 22843 | 18972 | 16246 MB/s | 8621 MB/s | 40972 MB/s | 40292 MB/s |
|
||||
| urls.10K | 335492 | 248079 | 7943 MB/s | 5104 MB/s | 22523 MB/s | 20981 MB/s |
|
||||
| fireworks.jpeg | 123034 | 123100 | 349544 MB/s | 84429 MB/s | 718321 MB/s | 823698 MB/s |
|
||||
| fireworks.jpeg (200B) | 146 | 149 | 8869 MB/s | 7125 MB/s | 33691 MB/s | 30101 MB/s |
|
||||
| paper-100k.pdf | 85304 | 82887 | 167546 MB/s | 11087 MB/s | 326905 MB/s | 198869 MB/s |
|
||||
| html_x_4 | 92234 | 18982 | 15194 MB/s | 29316 MB/s | 30843 MB/s | 30937 MB/s |
|
||||
| alice29.txt | 88034 | 71611 | 5936 MB/s | 3709 MB/s | 12882 MB/s | 16611 MB/s |
|
||||
| asyoulik.txt | 77503 | 65941 | 5517 MB/s | 3380 MB/s | 12735 MB/s | 14975 MB/s |
|
||||
| lcet10.txt | 234661 | 184939 | 6235 MB/s | 3537 MB/s | 14519 MB/s | 16634 MB/s |
|
||||
| plrabn12.txt | 319267 | 264990 | 5159 MB/s | 2960 MB/s | 11923 MB/s | 13382 MB/s |
|
||||
| geo.protodata | 23335 | 17689 | 21220 MB/s | 10859 MB/s | 56271 MB/s | 57961 MB/s |
|
||||
| kppkn.gtb | 69526 | 55398 | 9732 MB/s | 5206 MB/s | 18491 MB/s | 16524 MB/s |
|
||||
| alice29.txt (128B) | 80 | 78 | 6691 MB/s | 7422 MB/s | 31883 MB/s | 34225 MB/s |
|
||||
| alice29.txt (1000B) | 774 | 746 | 12204 MB/s | 5734 MB/s | 48056 MB/s | 42068 MB/s |
|
||||
| alice29.txt (10000B) | 6648 | 6218 | 10044 MB/s | 6055 MB/s | 32378 MB/s | 28813 MB/s |
|
||||
| alice29.txt (20000B) | 12686 | 11492 | 7733 MB/s | 3143 MB/s | 30566 MB/s | 27315 MB/s |
|
||||
|
||||
|
||||
| Relative Perf | Snappy size | Better size | Better Speed | Better dec |
|
||||
|-----------------------|-------------|-------------|--------------|------------|
|
||||
| html | 22.31% | 13.18% | 0.48x | 0.98x |
|
||||
| urls.10K | 47.78% | 24.43% | 0.50x | 0.93x |
|
||||
| fireworks.jpeg | 99.95% | -0.05% | 0.03x | 1.15x |
|
||||
| fireworks.jpeg (200B) | 73.00% | 2.74% | 0.07x | 0.89x |
|
||||
| paper-100k.pdf | 83.30% | 2.80% | 0.07x | 0.61x |
|
||||
| html_x_4 | 22.52% | 78.49% | 0.04x | 1.00x |
|
||||
| alice29.txt | 57.88% | 16.83% | 1.54x | 1.29x |
|
||||
| asyoulik.txt | 61.91% | 13.75% | 0.50x | 1.18x |
|
||||
| lcet10.txt | 54.99% | 18.78% | 0.50x | 1.15x |
|
||||
| plrabn12.txt | 66.26% | 15.17% | 0.50x | 1.12x |
|
||||
| geo.protodata | 19.68% | 21.67% | 0.50x | 1.03x |
|
||||
| kppkn.gtb | 37.72% | 11.04% | 0.53x | 0.89x |
|
||||
| alice29.txt (128B) | 62.50% | -1.25% | 0.47x | 1.07x |
|
||||
| alice29.txt (1000B) | 77.40% | 3.36% | 0.08x | 0.88x |
|
||||
| alice29.txt (10000B) | 66.48% | 6.23% | 0.16x | 0.89x |
|
||||
| alice29.txt (20000B) | 63.43% | 8.69% | 0.29x | 0.89x |
|
||||
|
||||
Except for the mostly incompressible JPEG image compression is better and usually in the
|
||||
double digits in terms of percentage reduction over Snappy.
|
||||
|
||||
@ -605,33 +568,150 @@ Some examples compared on 16 core CPU, amd64 assembly used:
|
||||
|
||||
```
|
||||
* enwik10
|
||||
Default... 10000000000 -> 4761467548 [47.61%]; 1.098s, 8685.6MB/s
|
||||
Better... 10000000000 -> 4219438251 [42.19%]; 1.925s, 4954.2MB/s
|
||||
Best... 10000000000 -> 3627364337 [36.27%]; 43.051s, 221.5MB/s
|
||||
Default... 10000000000 -> 4759950115 [47.60%]; 1.03s, 9263.0MB/s
|
||||
Better... 10000000000 -> 4084706676 [40.85%]; 2.16s, 4415.4MB/s
|
||||
Best... 10000000000 -> 3615520079 [36.16%]; 42.259s, 225.7MB/s
|
||||
|
||||
* github-june-2days-2019.json
|
||||
Default... 6273951764 -> 1043196283 [16.63%]; 431ms, 13882.3MB/s
|
||||
Better... 6273951764 -> 949146808 [15.13%]; 547ms, 10938.4MB/s
|
||||
Best... 6273951764 -> 832855506 [13.27%]; 9.455s, 632.8MB/s
|
||||
Default... 6273951764 -> 1041700255 [16.60%]; 431ms, 13882.3MB/s
|
||||
Better... 6273951764 -> 945841238 [15.08%]; 547ms, 10938.4MB/s
|
||||
Best... 6273951764 -> 826392576 [13.17%]; 9.455s, 632.8MB/s
|
||||
|
||||
* nyc-taxi-data-10M.csv
|
||||
Default... 3325605752 -> 1095998837 [32.96%]; 324ms, 9788.7MB/s
|
||||
Better... 3325605752 -> 954776589 [28.71%]; 491ms, 6459.4MB/s
|
||||
Best... 3325605752 -> 779098746 [23.43%]; 8.29s, 382.6MB/s
|
||||
Default... 3325605752 -> 1093516949 [32.88%]; 324ms, 9788.7MB/s
|
||||
Better... 3325605752 -> 885394158 [26.62%]; 491ms, 6459.4MB/s
|
||||
Best... 3325605752 -> 773681257 [23.26%]; 8.29s, 412.0MB/s
|
||||
|
||||
* 10gb.tar
|
||||
Default... 10065157632 -> 5916578242 [58.78%]; 1.028s, 9337.4MB/s
|
||||
Better... 10065157632 -> 5649207485 [56.13%]; 1.597s, 6010.6MB/s
|
||||
Best... 10065157632 -> 5208719802 [51.75%]; 32.78s, 292.8MB/
|
||||
Default... 10065157632 -> 5915541066 [58.77%]; 1.028s, 9337.4MB/s
|
||||
Better... 10065157632 -> 5453844650 [54.19%]; 1.597s, 4862.7MB/s
|
||||
Best... 10065157632 -> 5192495021 [51.59%]; 32.78s, 308.2MB/
|
||||
|
||||
* consensus.db.10gb
|
||||
Default... 10737418240 -> 4562648848 [42.49%]; 882ms, 11610.0MB/s
|
||||
Better... 10737418240 -> 4542428129 [42.30%]; 1.533s, 6679.7MB/s
|
||||
Best... 10737418240 -> 4244773384 [39.53%]; 42.96s, 238.4MB/s
|
||||
Default... 10737418240 -> 4549762344 [42.37%]; 882ms, 12118.4MB/s
|
||||
Better... 10737418240 -> 4438535064 [41.34%]; 1.533s, 3500.9MB/s
|
||||
Best... 10737418240 -> 4210602774 [39.21%]; 42.96s, 254.4MB/s
|
||||
```
|
||||
|
||||
Decompression speed should be around the same as using the 'better' compression mode.
|
||||
|
||||
## Dictionaries
|
||||
|
||||
*Note: S2 dictionary compression is currently at an early implementation stage, with no assembly for
|
||||
neither encoding nor decoding. Performance improvements can be expected in the future.*
|
||||
|
||||
Adding dictionaries allow providing a custom dictionary that will serve as lookup in the beginning of blocks.
|
||||
|
||||
The same dictionary *must* be used for both encoding and decoding.
|
||||
S2 does not keep track of whether the same dictionary is used,
|
||||
and using the wrong dictionary will most often not result in an error when decompressing.
|
||||
|
||||
Blocks encoded *without* dictionaries can be decompressed seamlessly *with* a dictionary.
|
||||
This means it is possible to switch from an encoding without dictionaries to an encoding with dictionaries
|
||||
and treat the blocks similarly.
|
||||
|
||||
Similar to [zStandard dictionaries](https://github.com/facebook/zstd#the-case-for-small-data-compression),
|
||||
the same usage scenario applies to S2 dictionaries.
|
||||
|
||||
> Training works if there is some correlation in a family of small data samples. The more data-specific a dictionary is, the more efficient it is (there is no universal dictionary). Hence, deploying one dictionary per type of data will provide the greatest benefits. Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file.
|
||||
|
||||
S2 further limits the dictionary to only be enabled on the first 64KB of a block.
|
||||
This will remove any negative (speed) impacts of the dictionaries on bigger blocks.
|
||||
|
||||
### Compression
|
||||
|
||||
Using the [github_users_sample_set](https://github.com/facebook/zstd/releases/download/v1.1.3/github_users_sample_set.tar.zst)
|
||||
and a 64KB dictionary trained with zStandard the following sizes can be achieved.
|
||||
|
||||
| | Default | Better | Best |
|
||||
|--------------------|------------------|------------------|-----------------------|
|
||||
| Without Dictionary | 3362023 (44.92%) | 3083163 (41.19%) | 3057944 (40.86%) |
|
||||
| With Dictionary | 921524 (12.31%) | 873154 (11.67%) | 785503 bytes (10.49%) |
|
||||
|
||||
So for highly repetitive content, this case provides an almost 3x reduction in size.
|
||||
|
||||
For less uniform data we will use the Go source code tree.
|
||||
Compressing First 64KB of all `.go` files in `go/src`, Go 1.19.5, 8912 files, 51253563 bytes input:
|
||||
|
||||
| | Default | Better | Best |
|
||||
|--------------------|-------------------|-------------------|-------------------|
|
||||
| Without Dictionary | 22955767 (44.79%) | 20189613 (39.39% | 19482828 (38.01%) |
|
||||
| With Dictionary | 19654568 (38.35%) | 16289357 (31.78%) | 15184589 (29.63%) |
|
||||
| Saving/file | 362 bytes | 428 bytes | 472 bytes |
|
||||
|
||||
|
||||
### Creating Dictionaries
|
||||
|
||||
There are no tools to create dictionaries in S2.
|
||||
However, there are multiple ways to create a useful dictionary:
|
||||
|
||||
#### Using a Sample File
|
||||
|
||||
If your input is very uniform, you can just use a sample file as the dictionary.
|
||||
|
||||
For example in the `github_users_sample_set` above, the average compression only goes up from
|
||||
10.49% to 11.48% by using the first file as dictionary compared to using a dedicated dictionary.
|
||||
|
||||
```Go
|
||||
// Read a sample
|
||||
sample, err := os.ReadFile("sample.json")
|
||||
|
||||
// Create a dictionary.
|
||||
dict := s2.MakeDict(sample, nil)
|
||||
|
||||
// b := dict.Bytes() will provide a dictionary that can be saved
|
||||
// and reloaded with s2.NewDict(b).
|
||||
|
||||
// To encode:
|
||||
encoded := dict.Encode(nil, file)
|
||||
|
||||
// To decode:
|
||||
decoded, err := dict.Decode(nil, file)
|
||||
```
|
||||
|
||||
#### Using Zstandard
|
||||
|
||||
Zstandard dictionaries can easily be converted to S2 dictionaries.
|
||||
|
||||
This can be helpful to generate dictionaries for files that don't have a fixed structure.
|
||||
|
||||
|
||||
Example, with training set files placed in `./training-set`:
|
||||
|
||||
`λ zstd -r --train-fastcover training-set/* --maxdict=65536 -o name.dict`
|
||||
|
||||
This will create a dictionary of 64KB, that can be converted to a dictionary like this:
|
||||
|
||||
```Go
|
||||
// Decode the Zstandard dictionary.
|
||||
insp, err := zstd.InspectDictionary(zdict)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// We are only interested in the contents.
|
||||
// Assume that files start with "// Copyright (c) 2023".
|
||||
// Search for the longest match for that.
|
||||
// This may save a few bytes.
|
||||
dict := s2.MakeDict(insp.Content(), []byte("// Copyright (c) 2023"))
|
||||
|
||||
// b := dict.Bytes() will provide a dictionary that can be saved
|
||||
// and reloaded with s2.NewDict(b).
|
||||
|
||||
// We can now encode using this dictionary
|
||||
encodedWithDict := dict.Encode(nil, payload)
|
||||
|
||||
// To decode content:
|
||||
decoded, err := dict.Decode(nil, encodedWithDict)
|
||||
```
|
||||
|
||||
It is recommended to save the dictionary returned by ` b:= dict.Bytes()`, since that will contain only the S2 dictionary.
|
||||
|
||||
This dictionary can later be loaded using `s2.NewDict(b)`. The dictionary then no longer requires `zstd` to be initialized.
|
||||
|
||||
Also note how `s2.MakeDict` allows you to search for a common starting sequence of your files.
|
||||
This can be omitted, at the expense of a few bytes.
|
||||
|
||||
# Snappy Compatibility
|
||||
|
||||
S2 now offers full compatibility with Snappy.
|
||||
@ -648,10 +728,10 @@ If you would like more control, you can use the s2 package as described below:
|
||||
Snappy compatible blocks can be generated with the S2 encoder.
|
||||
Compression and speed is typically a bit better `MaxEncodedLen` is also smaller for smaller memory usage. Replace
|
||||
|
||||
| Snappy | S2 replacement |
|
||||
|----------------------------|-------------------------|
|
||||
| snappy.Encode(...) | s2.EncodeSnappy(...) |
|
||||
| snappy.MaxEncodedLen(...) | s2.MaxEncodedLen(...) |
|
||||
| Snappy | S2 replacement |
|
||||
|---------------------------|-----------------------|
|
||||
| snappy.Encode(...) | s2.EncodeSnappy(...) |
|
||||
| snappy.MaxEncodedLen(...) | s2.MaxEncodedLen(...) |
|
||||
|
||||
`s2.EncodeSnappy` can be replaced with `s2.EncodeSnappyBetter` or `s2.EncodeSnappyBest` to get more efficiently compressed snappy compatible output.
|
||||
|
||||
@ -660,12 +740,12 @@ Compression and speed is typically a bit better `MaxEncodedLen` is also smaller
|
||||
Comparison of [`webdevdata.org-2015-01-07-subset`](https://files.klauspost.com/compress/webdevdata.org-2015-01-07-4GB-subset.7z),
|
||||
53927 files, total input size: 4,014,735,833 bytes. amd64, single goroutine used:
|
||||
|
||||
| Encoder | Size | MB/s | Reduction |
|
||||
|-----------------------|------------|------------|------------
|
||||
| snappy.Encode | 1128706759 | 725.59 | 71.89% |
|
||||
| s2.EncodeSnappy | 1093823291 | **899.16** | 72.75% |
|
||||
| s2.EncodeSnappyBetter | 1001158548 | 578.49 | 75.06% |
|
||||
| s2.EncodeSnappyBest | 944507998 | 66.00 | **76.47%**|
|
||||
| Encoder | Size | MB/s | Reduction |
|
||||
|-----------------------|------------|------------|------------|
|
||||
| snappy.Encode | 1128706759 | 725.59 | 71.89% |
|
||||
| s2.EncodeSnappy | 1093823291 | **899.16** | 72.75% |
|
||||
| s2.EncodeSnappyBetter | 1001158548 | 578.49 | 75.06% |
|
||||
| s2.EncodeSnappyBest | 944507998 | 66.00 | **76.47%** |
|
||||
|
||||
## Streams
|
||||
|
||||
@ -835,6 +915,13 @@ This is done using the regular "Skip" function:
|
||||
|
||||
This will ensure that we are at exactly the offset we want, and reading from `dec` will start at the requested offset.
|
||||
|
||||
# Compact storage
|
||||
|
||||
For compact storage [RemoveIndexHeaders](https://pkg.go.dev/github.com/klauspost/compress/s2#RemoveIndexHeaders) can be used to remove any redundant info from
|
||||
a serialized index. If you remove the header it must be restored before [Loading](https://pkg.go.dev/github.com/klauspost/compress/s2#Index.Load).
|
||||
|
||||
This is expected to save 20 bytes. These can be restored using [RestoreIndexHeaders](https://pkg.go.dev/github.com/klauspost/compress/s2#RestoreIndexHeaders). This removes a layer of security, but is the most compact representation. Returns nil if headers contains errors.
|
||||
|
||||
## Index Format:
|
||||
|
||||
Each block is structured as a snappy skippable block, with the chunk ID 0x99.
|
||||
@ -844,20 +931,20 @@ The block can be read from the front, but contains information so it can be read
|
||||
Numbers are stored as fixed size little endian values or [zigzag encoded](https://developers.google.com/protocol-buffers/docs/encoding#signed_integers) [base 128 varints](https://developers.google.com/protocol-buffers/docs/encoding),
|
||||
with un-encoded value length of 64 bits, unless other limits are specified.
|
||||
|
||||
| Content | Format |
|
||||
|---------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|
|
||||
| ID, `[1]byte` | Always 0x99. |
|
||||
| Data Length, `[3]byte` | 3 byte little-endian length of the chunk in bytes, following this. |
|
||||
| Header `[6]byte` | Header, must be `[115, 50, 105, 100, 120, 0]` or in text: "s2idx\x00". |
|
||||
| UncompressedSize, Varint | Total Uncompressed size. |
|
||||
| CompressedSize, Varint | Total Compressed size if known. Should be -1 if unknown. |
|
||||
| EstBlockSize, Varint | Block Size, used for guessing uncompressed offsets. Must be >= 0. |
|
||||
| Entries, Varint | Number of Entries in index, must be < 65536 and >=0. |
|
||||
| HasUncompressedOffsets `byte` | 0 if no uncompressed offsets are present, 1 if present. Other values are invalid. |
|
||||
| UncompressedOffsets, [Entries]VarInt | Uncompressed offsets. See below how to decode. |
|
||||
| CompressedOffsets, [Entries]VarInt | Compressed offsets. See below how to decode. |
|
||||
| Block Size, `[4]byte` | Little Endian total encoded size (including header and trailer). Can be used for searching backwards to start of block. |
|
||||
| Trailer `[6]byte` | Trailer, must be `[0, 120, 100, 105, 50, 115]` or in text: "\x00xdi2s". Can be used for identifying block from end of stream. |
|
||||
| Content | Format |
|
||||
|--------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|
|
||||
| ID, `[1]byte` | Always 0x99. |
|
||||
| Data Length, `[3]byte` | 3 byte little-endian length of the chunk in bytes, following this. |
|
||||
| Header `[6]byte` | Header, must be `[115, 50, 105, 100, 120, 0]` or in text: "s2idx\x00". |
|
||||
| UncompressedSize, Varint | Total Uncompressed size. |
|
||||
| CompressedSize, Varint | Total Compressed size if known. Should be -1 if unknown. |
|
||||
| EstBlockSize, Varint | Block Size, used for guessing uncompressed offsets. Must be >= 0. |
|
||||
| Entries, Varint | Number of Entries in index, must be < 65536 and >=0. |
|
||||
| HasUncompressedOffsets `byte` | 0 if no uncompressed offsets are present, 1 if present. Other values are invalid. |
|
||||
| UncompressedOffsets, [Entries]VarInt | Uncompressed offsets. See below how to decode. |
|
||||
| CompressedOffsets, [Entries]VarInt | Compressed offsets. See below how to decode. |
|
||||
| Block Size, `[4]byte` | Little Endian total encoded size (including header and trailer). Can be used for searching backwards to start of block. |
|
||||
| Trailer `[6]byte` | Trailer, must be `[0, 120, 100, 105, 50, 115]` or in text: "\x00xdi2s". Can be used for identifying block from end of stream. |
|
||||
|
||||
For regular streams the uncompressed offsets are fully predictable,
|
||||
so `HasUncompressedOffsets` allows to specify that compressed blocks all have
|
||||
@ -929,6 +1016,7 @@ To decode from any given uncompressed offset `(wantOffset)`:
|
||||
|
||||
See [using indexes](https://github.com/klauspost/compress/tree/master/s2#using-indexes) for functions that perform the operations with a simpler interface.
|
||||
|
||||
|
||||
# Format Extensions
|
||||
|
||||
* Frame [Stream identifier](https://github.com/google/snappy/blob/master/framing_format.txt#L68) changed from `sNaPpY` to `S2sTwO`.
|
||||
@ -951,13 +1039,80 @@ The length is specified by reading the 3-bit length specified in the tag and dec
|
||||
| 7 | 65540 + read 3 bytes |
|
||||
|
||||
This allows any repeat offset + length to be represented by 2 to 5 bytes.
|
||||
It also allows to emit matches longer than 64 bytes with one copy + one repeat instead of several 64 byte copies.
|
||||
|
||||
Lengths are stored as little endian values.
|
||||
|
||||
The first copy of a block cannot be a repeat offset and the offset is not carried across blocks in streams.
|
||||
The first copy of a block cannot be a repeat offset and the offset is reset on every block in streams.
|
||||
|
||||
Default streaming block size is 1MB.
|
||||
|
||||
# Dictionary Encoding
|
||||
|
||||
Adding dictionaries allow providing a custom dictionary that will serve as lookup in the beginning of blocks.
|
||||
|
||||
A dictionary provides an initial repeat value that can be used to point to a common header.
|
||||
|
||||
Other than that the dictionary contains values that can be used as back-references.
|
||||
|
||||
Often used data should be placed at the *end* of the dictionary since offsets < 2048 bytes will be smaller.
|
||||
|
||||
## Format
|
||||
|
||||
Dictionary *content* must at least 16 bytes and less or equal to 64KiB (65536 bytes).
|
||||
|
||||
Encoding: `[repeat value (uvarint)][dictionary content...]`
|
||||
|
||||
Before the dictionary content, an unsigned base-128 (uvarint) encoded value specifying the initial repeat offset.
|
||||
This value is an offset into the dictionary content and not a back-reference offset,
|
||||
so setting this to 0 will make the repeat value point to the first value of the dictionary.
|
||||
|
||||
The value must be less than the dictionary length-8
|
||||
|
||||
## Encoding
|
||||
|
||||
From the decoder point of view the dictionary content is seen as preceding the encoded content.
|
||||
|
||||
`[dictionary content][decoded output]`
|
||||
|
||||
Backreferences to the dictionary are encoded as ordinary backreferences that have an offset before the start of the decoded block.
|
||||
|
||||
Matches copying from the dictionary are **not** allowed to cross from the dictionary into the decoded data.
|
||||
However, if a copy ends at the end of the dictionary the next repeat will point to the start of the decoded buffer, which is allowed.
|
||||
|
||||
The first match can be a repeat value, which will use the repeat offset stored in the dictionary.
|
||||
|
||||
When 64KB (65536 bytes) has been en/decoded it is no longer allowed to reference the dictionary,
|
||||
neither by a copy nor repeat operations.
|
||||
If the boundary is crossed while copying from the dictionary, the operation should complete,
|
||||
but the next instruction is not allowed to reference the dictionary.
|
||||
|
||||
Valid blocks encoded *without* a dictionary can be decoded with any dictionary.
|
||||
There are no checks whether the supplied dictionary is the correct for a block.
|
||||
Because of this there is no overhead by using a dictionary.
|
||||
|
||||
## Example
|
||||
|
||||
This is the dictionary content. Elements are separated by `[]`.
|
||||
|
||||
Dictionary: `[0x0a][Yesterday 25 bananas were added to Benjamins brown bag]`.
|
||||
|
||||
Initial repeat offset is set at 10, which is the letter `2`.
|
||||
|
||||
Encoded `[LIT "10"][REPEAT len=10][LIT "hich"][MATCH off=50 len=6][MATCH off=31 len=6][MATCH off=61 len=10]`
|
||||
|
||||
Decoded: `[10][ bananas w][hich][ were ][brown ][were added]`
|
||||
|
||||
Output: `10 bananas which were brown were added`
|
||||
|
||||
|
||||
## Streams
|
||||
|
||||
For streams each block can use the dictionary.
|
||||
|
||||
The dictionary cannot not currently be provided on the stream.
|
||||
|
||||
|
||||
# LICENSE
|
||||
|
||||
This code is based on the [Snappy-Go](https://github.com/golang/snappy) implementation.
|
||||
|
||||
504
vendor/github.com/klauspost/compress/s2/decode.go
generated
vendored
504
vendor/github.com/klauspost/compress/s2/decode.go
generated
vendored
@ -11,7 +11,9 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"sync"
|
||||
)
|
||||
|
||||
@ -719,7 +721,11 @@ func (r *Reader) Skip(n int64) error {
|
||||
// decoded[i:j] contains decoded bytes that have not yet been passed on.
|
||||
left := int64(r.j - r.i)
|
||||
if left >= n {
|
||||
r.i += int(n)
|
||||
tmp := int64(r.i) + n
|
||||
if tmp > math.MaxInt32 {
|
||||
return errors.New("s2: internal overflow in skip")
|
||||
}
|
||||
r.i = int(tmp)
|
||||
return nil
|
||||
}
|
||||
n -= int64(r.j - r.i)
|
||||
@ -875,15 +881,20 @@ func (r *Reader) Skip(n int64) error {
|
||||
// See Reader.ReadSeeker
|
||||
type ReadSeeker struct {
|
||||
*Reader
|
||||
readAtMu sync.Mutex
|
||||
}
|
||||
|
||||
// ReadSeeker will return an io.ReadSeeker compatible version of the reader.
|
||||
// ReadSeeker will return an io.ReadSeeker and io.ReaderAt
|
||||
// compatible version of the reader.
|
||||
// If 'random' is specified the returned io.Seeker can be used for
|
||||
// random seeking, otherwise only forward seeking is supported.
|
||||
// Enabling random seeking requires the original input to support
|
||||
// the io.Seeker interface.
|
||||
// A custom index can be specified which will be used if supplied.
|
||||
// When using a custom index, it will not be read from the input stream.
|
||||
// The ReadAt position will affect regular reads and the current position of Seek.
|
||||
// So using Read after ReadAt will continue from where the ReadAt stopped.
|
||||
// No functions should be used concurrently.
|
||||
// The returned ReadSeeker contains a shallow reference to the existing Reader,
|
||||
// meaning changes performed to one is reflected in the other.
|
||||
func (r *Reader) ReadSeeker(random bool, index []byte) (*ReadSeeker, error) {
|
||||
@ -947,44 +958,61 @@ func (r *Reader) ReadSeeker(random bool, index []byte) (*ReadSeeker, error) {
|
||||
// Seek allows seeking in compressed data.
|
||||
func (r *ReadSeeker) Seek(offset int64, whence int) (int64, error) {
|
||||
if r.err != nil {
|
||||
return 0, r.err
|
||||
}
|
||||
if offset == 0 && whence == io.SeekCurrent {
|
||||
return r.blockStart + int64(r.i), nil
|
||||
}
|
||||
if !r.readHeader {
|
||||
// Make sure we read the header.
|
||||
_, r.err = r.Read([]byte{})
|
||||
}
|
||||
rs, ok := r.r.(io.ReadSeeker)
|
||||
if r.index == nil || !ok {
|
||||
if whence == io.SeekCurrent && offset >= 0 {
|
||||
err := r.Skip(offset)
|
||||
return r.blockStart + int64(r.i), err
|
||||
if !errors.Is(r.err, io.EOF) {
|
||||
return 0, r.err
|
||||
}
|
||||
if whence == io.SeekStart && offset >= r.blockStart+int64(r.i) {
|
||||
err := r.Skip(offset - r.blockStart - int64(r.i))
|
||||
return r.blockStart + int64(r.i), err
|
||||
}
|
||||
return 0, ErrUnsupported
|
||||
// Reset on EOF
|
||||
r.err = nil
|
||||
}
|
||||
|
||||
}
|
||||
// Calculate absolute offset.
|
||||
absOffset := offset
|
||||
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
case io.SeekCurrent:
|
||||
offset += r.blockStart + int64(r.i)
|
||||
absOffset = r.blockStart + int64(r.i) + offset
|
||||
case io.SeekEnd:
|
||||
if offset > 0 {
|
||||
return 0, errors.New("seek after end of file")
|
||||
if r.index == nil {
|
||||
return 0, ErrUnsupported
|
||||
}
|
||||
offset = r.index.TotalUncompressed + offset
|
||||
absOffset = r.index.TotalUncompressed + offset
|
||||
default:
|
||||
r.err = ErrUnsupported
|
||||
return 0, r.err
|
||||
}
|
||||
|
||||
if offset < 0 {
|
||||
if absOffset < 0 {
|
||||
return 0, errors.New("seek before start of file")
|
||||
}
|
||||
|
||||
c, u, err := r.index.Find(offset)
|
||||
if !r.readHeader {
|
||||
// Make sure we read the header.
|
||||
_, r.err = r.Read([]byte{})
|
||||
if r.err != nil {
|
||||
return 0, r.err
|
||||
}
|
||||
}
|
||||
|
||||
// If we are inside current block no need to seek.
|
||||
// This includes no offset changes.
|
||||
if absOffset >= r.blockStart && absOffset < r.blockStart+int64(r.j) {
|
||||
r.i = int(absOffset - r.blockStart)
|
||||
return r.blockStart + int64(r.i), nil
|
||||
}
|
||||
|
||||
rs, ok := r.r.(io.ReadSeeker)
|
||||
if r.index == nil || !ok {
|
||||
currOffset := r.blockStart + int64(r.i)
|
||||
if absOffset >= currOffset {
|
||||
err := r.Skip(absOffset - currOffset)
|
||||
return r.blockStart + int64(r.i), err
|
||||
}
|
||||
return 0, ErrUnsupported
|
||||
}
|
||||
|
||||
// We can seek and we have an index.
|
||||
c, u, err := r.index.Find(absOffset)
|
||||
if err != nil {
|
||||
return r.blockStart + int64(r.i), err
|
||||
}
|
||||
@ -995,12 +1023,57 @@ func (r *ReadSeeker) Seek(offset int64, whence int) (int64, error) {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
r.i = r.j // Remove rest of current block.
|
||||
if u < offset {
|
||||
r.i = r.j // Remove rest of current block.
|
||||
r.blockStart = u - int64(r.j) // Adjust current block start for accounting.
|
||||
if u < absOffset {
|
||||
// Forward inside block
|
||||
return offset, r.Skip(offset - u)
|
||||
return absOffset, r.Skip(absOffset - u)
|
||||
}
|
||||
return offset, nil
|
||||
if u > absOffset {
|
||||
return 0, fmt.Errorf("s2 seek: (internal error) u (%d) > absOffset (%d)", u, absOffset)
|
||||
}
|
||||
return absOffset, nil
|
||||
}
|
||||
|
||||
// ReadAt reads len(p) bytes into p starting at offset off in the
|
||||
// underlying input source. It returns the number of bytes
|
||||
// read (0 <= n <= len(p)) and any error encountered.
|
||||
//
|
||||
// When ReadAt returns n < len(p), it returns a non-nil error
|
||||
// explaining why more bytes were not returned. In this respect,
|
||||
// ReadAt is stricter than Read.
|
||||
//
|
||||
// Even if ReadAt returns n < len(p), it may use all of p as scratch
|
||||
// space during the call. If some data is available but not len(p) bytes,
|
||||
// ReadAt blocks until either all the data is available or an error occurs.
|
||||
// In this respect ReadAt is different from Read.
|
||||
//
|
||||
// If the n = len(p) bytes returned by ReadAt are at the end of the
|
||||
// input source, ReadAt may return either err == EOF or err == nil.
|
||||
//
|
||||
// If ReadAt is reading from an input source with a seek offset,
|
||||
// ReadAt should not affect nor be affected by the underlying
|
||||
// seek offset.
|
||||
//
|
||||
// Clients of ReadAt can execute parallel ReadAt calls on the
|
||||
// same input source. This is however not recommended.
|
||||
func (r *ReadSeeker) ReadAt(p []byte, offset int64) (int, error) {
|
||||
r.readAtMu.Lock()
|
||||
defer r.readAtMu.Unlock()
|
||||
_, err := r.Seek(offset, io.SeekStart)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n := 0
|
||||
for n < len(p) {
|
||||
n2, err := r.Read(p[n:])
|
||||
if err != nil {
|
||||
// This will include io.EOF
|
||||
return n + n2, err
|
||||
}
|
||||
n += n2
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// ReadByte satisfies the io.ByteReader interface.
|
||||
@ -1039,3 +1112,370 @@ func (r *Reader) SkippableCB(id uint8, fn func(r io.Reader) error) error {
|
||||
r.skippableCB[id] = fn
|
||||
return nil
|
||||
}
|
||||
|
||||
// s2DecodeDict writes the decoding of src to dst. It assumes that the varint-encoded
|
||||
// length of the decompressed bytes has already been read, and that len(dst)
|
||||
// equals that length.
|
||||
//
|
||||
// It returns 0 on success or a decodeErrCodeXxx error code on failure.
|
||||
func s2DecodeDict(dst, src []byte, dict *Dict) int {
|
||||
if dict == nil {
|
||||
return s2Decode(dst, src)
|
||||
}
|
||||
const debug = false
|
||||
const debugErrs = debug
|
||||
|
||||
if debug {
|
||||
fmt.Println("Starting decode, dst len:", len(dst))
|
||||
}
|
||||
var d, s, length int
|
||||
offset := len(dict.dict) - dict.repeat
|
||||
|
||||
// As long as we can read at least 5 bytes...
|
||||
for s < len(src)-5 {
|
||||
// Removing bounds checks is SLOWER, when if doing
|
||||
// in := src[s:s+5]
|
||||
// Checked on Go 1.18
|
||||
switch src[s] & 0x03 {
|
||||
case tagLiteral:
|
||||
x := uint32(src[s] >> 2)
|
||||
switch {
|
||||
case x < 60:
|
||||
s++
|
||||
case x == 60:
|
||||
s += 2
|
||||
x = uint32(src[s-1])
|
||||
case x == 61:
|
||||
in := src[s : s+3]
|
||||
x = uint32(in[1]) | uint32(in[2])<<8
|
||||
s += 3
|
||||
case x == 62:
|
||||
in := src[s : s+4]
|
||||
// Load as 32 bit and shift down.
|
||||
x = uint32(in[0]) | uint32(in[1])<<8 | uint32(in[2])<<16 | uint32(in[3])<<24
|
||||
x >>= 8
|
||||
s += 4
|
||||
case x == 63:
|
||||
in := src[s : s+5]
|
||||
x = uint32(in[1]) | uint32(in[2])<<8 | uint32(in[3])<<16 | uint32(in[4])<<24
|
||||
s += 5
|
||||
}
|
||||
length = int(x) + 1
|
||||
if debug {
|
||||
fmt.Println("literals, length:", length, "d-after:", d+length)
|
||||
}
|
||||
if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
|
||||
if debugErrs {
|
||||
fmt.Println("corrupt literal: length:", length, "d-left:", len(dst)-d, "src-left:", len(src)-s)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
|
||||
copy(dst[d:], src[s:s+length])
|
||||
d += length
|
||||
s += length
|
||||
continue
|
||||
|
||||
case tagCopy1:
|
||||
s += 2
|
||||
toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
|
||||
length = int(src[s-2]) >> 2 & 0x7
|
||||
if toffset == 0 {
|
||||
if debug {
|
||||
fmt.Print("(repeat) ")
|
||||
}
|
||||
// keep last offset
|
||||
switch length {
|
||||
case 5:
|
||||
length = int(src[s]) + 4
|
||||
s += 1
|
||||
case 6:
|
||||
in := src[s : s+2]
|
||||
length = int(uint32(in[0])|(uint32(in[1])<<8)) + (1 << 8)
|
||||
s += 2
|
||||
case 7:
|
||||
in := src[s : s+3]
|
||||
length = int((uint32(in[2])<<16)|(uint32(in[1])<<8)|uint32(in[0])) + (1 << 16)
|
||||
s += 3
|
||||
default: // 0-> 4
|
||||
}
|
||||
} else {
|
||||
offset = toffset
|
||||
}
|
||||
length += 4
|
||||
case tagCopy2:
|
||||
in := src[s : s+3]
|
||||
offset = int(uint32(in[1]) | uint32(in[2])<<8)
|
||||
length = 1 + int(in[0])>>2
|
||||
s += 3
|
||||
|
||||
case tagCopy4:
|
||||
in := src[s : s+5]
|
||||
offset = int(uint32(in[1]) | uint32(in[2])<<8 | uint32(in[3])<<16 | uint32(in[4])<<24)
|
||||
length = 1 + int(in[0])>>2
|
||||
s += 5
|
||||
}
|
||||
|
||||
if offset <= 0 || length > len(dst)-d {
|
||||
if debugErrs {
|
||||
fmt.Println("match error; offset:", offset, "length:", length, "dst-left:", len(dst)-d)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
|
||||
// copy from dict
|
||||
if d < offset {
|
||||
if d > MaxDictSrcOffset {
|
||||
if debugErrs {
|
||||
fmt.Println("dict after", MaxDictSrcOffset, "d:", d, "offset:", offset, "length:", length)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
startOff := len(dict.dict) - offset + d
|
||||
if startOff < 0 || startOff+length > len(dict.dict) {
|
||||
if debugErrs {
|
||||
fmt.Printf("offset (%d) + length (%d) bigger than dict (%d)\n", offset, length, len(dict.dict))
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("dict copy, length:", length, "offset:", offset, "d-after:", d+length, "dict start offset:", startOff)
|
||||
}
|
||||
copy(dst[d:d+length], dict.dict[startOff:])
|
||||
d += length
|
||||
continue
|
||||
}
|
||||
|
||||
if debug {
|
||||
fmt.Println("copy, length:", length, "offset:", offset, "d-after:", d+length)
|
||||
}
|
||||
|
||||
// Copy from an earlier sub-slice of dst to a later sub-slice.
|
||||
// If no overlap, use the built-in copy:
|
||||
if offset > length {
|
||||
copy(dst[d:d+length], dst[d-offset:])
|
||||
d += length
|
||||
continue
|
||||
}
|
||||
|
||||
// Unlike the built-in copy function, this byte-by-byte copy always runs
|
||||
// forwards, even if the slices overlap. Conceptually, this is:
|
||||
//
|
||||
// d += forwardCopy(dst[d:d+length], dst[d-offset:])
|
||||
//
|
||||
// We align the slices into a and b and show the compiler they are the same size.
|
||||
// This allows the loop to run without bounds checks.
|
||||
a := dst[d : d+length]
|
||||
b := dst[d-offset:]
|
||||
b = b[:len(a)]
|
||||
for i := range a {
|
||||
a[i] = b[i]
|
||||
}
|
||||
d += length
|
||||
}
|
||||
|
||||
// Remaining with extra checks...
|
||||
for s < len(src) {
|
||||
switch src[s] & 0x03 {
|
||||
case tagLiteral:
|
||||
x := uint32(src[s] >> 2)
|
||||
switch {
|
||||
case x < 60:
|
||||
s++
|
||||
case x == 60:
|
||||
s += 2
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-1])
|
||||
case x == 61:
|
||||
s += 3
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-2]) | uint32(src[s-1])<<8
|
||||
case x == 62:
|
||||
s += 4
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
|
||||
case x == 63:
|
||||
s += 5
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
|
||||
}
|
||||
length = int(x) + 1
|
||||
if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
|
||||
if debugErrs {
|
||||
fmt.Println("corrupt literal: length:", length, "d-left:", len(dst)-d, "src-left:", len(src)-s)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("literals, length:", length, "d-after:", d+length)
|
||||
}
|
||||
|
||||
copy(dst[d:], src[s:s+length])
|
||||
d += length
|
||||
s += length
|
||||
continue
|
||||
|
||||
case tagCopy1:
|
||||
s += 2
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(src[s-2]) >> 2 & 0x7
|
||||
toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
|
||||
if toffset == 0 {
|
||||
if debug {
|
||||
fmt.Print("(repeat) ")
|
||||
}
|
||||
// keep last offset
|
||||
switch length {
|
||||
case 5:
|
||||
s += 1
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(uint32(src[s-1])) + 4
|
||||
case 6:
|
||||
s += 2
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(uint32(src[s-2])|(uint32(src[s-1])<<8)) + (1 << 8)
|
||||
case 7:
|
||||
s += 3
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(uint32(src[s-3])|(uint32(src[s-2])<<8)|(uint32(src[s-1])<<16)) + (1 << 16)
|
||||
default: // 0-> 4
|
||||
}
|
||||
} else {
|
||||
offset = toffset
|
||||
}
|
||||
length += 4
|
||||
case tagCopy2:
|
||||
s += 3
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = 1 + int(src[s-3])>>2
|
||||
offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
|
||||
|
||||
case tagCopy4:
|
||||
s += 5
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = 1 + int(src[s-5])>>2
|
||||
offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
|
||||
}
|
||||
|
||||
if offset <= 0 || length > len(dst)-d {
|
||||
if debugErrs {
|
||||
fmt.Println("match error; offset:", offset, "length:", length, "dst-left:", len(dst)-d)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
|
||||
// copy from dict
|
||||
if d < offset {
|
||||
if d > MaxDictSrcOffset {
|
||||
if debugErrs {
|
||||
fmt.Println("dict after", MaxDictSrcOffset, "d:", d, "offset:", offset, "length:", length)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
rOff := len(dict.dict) - (offset - d)
|
||||
if debug {
|
||||
fmt.Println("starting dict entry from dict offset", len(dict.dict)-rOff)
|
||||
}
|
||||
if rOff+length > len(dict.dict) {
|
||||
if debugErrs {
|
||||
fmt.Println("err: END offset", rOff+length, "bigger than dict", len(dict.dict), "dict offset:", rOff, "length:", length)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
if rOff < 0 {
|
||||
if debugErrs {
|
||||
fmt.Println("err: START offset", rOff, "less than 0", len(dict.dict), "dict offset:", rOff, "length:", length)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
copy(dst[d:d+length], dict.dict[rOff:])
|
||||
d += length
|
||||
continue
|
||||
}
|
||||
|
||||
if debug {
|
||||
fmt.Println("copy, length:", length, "offset:", offset, "d-after:", d+length)
|
||||
}
|
||||
|
||||
// Copy from an earlier sub-slice of dst to a later sub-slice.
|
||||
// If no overlap, use the built-in copy:
|
||||
if offset > length {
|
||||
copy(dst[d:d+length], dst[d-offset:])
|
||||
d += length
|
||||
continue
|
||||
}
|
||||
|
||||
// Unlike the built-in copy function, this byte-by-byte copy always runs
|
||||
// forwards, even if the slices overlap. Conceptually, this is:
|
||||
//
|
||||
// d += forwardCopy(dst[d:d+length], dst[d-offset:])
|
||||
//
|
||||
// We align the slices into a and b and show the compiler they are the same size.
|
||||
// This allows the loop to run without bounds checks.
|
||||
a := dst[d : d+length]
|
||||
b := dst[d-offset:]
|
||||
b = b[:len(a)]
|
||||
for i := range a {
|
||||
a[i] = b[i]
|
||||
}
|
||||
d += length
|
||||
}
|
||||
|
||||
if d != len(dst) {
|
||||
if debugErrs {
|
||||
fmt.Println("wanted length", len(dst), "got", d)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
47
vendor/github.com/klauspost/compress/s2/decode_other.go
generated
vendored
47
vendor/github.com/klauspost/compress/s2/decode_other.go
generated
vendored
@ -28,6 +28,9 @@ func s2Decode(dst, src []byte) int {
|
||||
|
||||
// As long as we can read at least 5 bytes...
|
||||
for s < len(src)-5 {
|
||||
// Removing bounds checks is SLOWER, when if doing
|
||||
// in := src[s:s+5]
|
||||
// Checked on Go 1.18
|
||||
switch src[s] & 0x03 {
|
||||
case tagLiteral:
|
||||
x := uint32(src[s] >> 2)
|
||||
@ -38,17 +41,25 @@ func s2Decode(dst, src []byte) int {
|
||||
s += 2
|
||||
x = uint32(src[s-1])
|
||||
case x == 61:
|
||||
in := src[s : s+3]
|
||||
x = uint32(in[1]) | uint32(in[2])<<8
|
||||
s += 3
|
||||
x = uint32(src[s-2]) | uint32(src[s-1])<<8
|
||||
case x == 62:
|
||||
in := src[s : s+4]
|
||||
// Load as 32 bit and shift down.
|
||||
x = uint32(in[0]) | uint32(in[1])<<8 | uint32(in[2])<<16 | uint32(in[3])<<24
|
||||
x >>= 8
|
||||
s += 4
|
||||
x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
|
||||
case x == 63:
|
||||
in := src[s : s+5]
|
||||
x = uint32(in[1]) | uint32(in[2])<<8 | uint32(in[3])<<16 | uint32(in[4])<<24
|
||||
s += 5
|
||||
x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
|
||||
}
|
||||
length = int(x) + 1
|
||||
if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
|
||||
if debug {
|
||||
fmt.Println("corrupt: lit size", length)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
if debug {
|
||||
@ -62,8 +73,8 @@ func s2Decode(dst, src []byte) int {
|
||||
|
||||
case tagCopy1:
|
||||
s += 2
|
||||
length = int(src[s-2]) >> 2 & 0x7
|
||||
toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
|
||||
length = int(src[s-2]) >> 2 & 0x7
|
||||
if toffset == 0 {
|
||||
if debug {
|
||||
fmt.Print("(repeat) ")
|
||||
@ -71,14 +82,16 @@ func s2Decode(dst, src []byte) int {
|
||||
// keep last offset
|
||||
switch length {
|
||||
case 5:
|
||||
length = int(src[s]) + 4
|
||||
s += 1
|
||||
length = int(uint32(src[s-1])) + 4
|
||||
case 6:
|
||||
in := src[s : s+2]
|
||||
length = int(uint32(in[0])|(uint32(in[1])<<8)) + (1 << 8)
|
||||
s += 2
|
||||
length = int(uint32(src[s-2])|(uint32(src[s-1])<<8)) + (1 << 8)
|
||||
case 7:
|
||||
in := src[s : s+3]
|
||||
length = int((uint32(in[2])<<16)|(uint32(in[1])<<8)|uint32(in[0])) + (1 << 16)
|
||||
s += 3
|
||||
length = int(uint32(src[s-3])|(uint32(src[s-2])<<8)|(uint32(src[s-1])<<16)) + (1 << 16)
|
||||
default: // 0-> 4
|
||||
}
|
||||
} else {
|
||||
@ -86,17 +99,23 @@ func s2Decode(dst, src []byte) int {
|
||||
}
|
||||
length += 4
|
||||
case tagCopy2:
|
||||
in := src[s : s+3]
|
||||
offset = int(uint32(in[1]) | uint32(in[2])<<8)
|
||||
length = 1 + int(in[0])>>2
|
||||
s += 3
|
||||
length = 1 + int(src[s-3])>>2
|
||||
offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
|
||||
|
||||
case tagCopy4:
|
||||
in := src[s : s+5]
|
||||
offset = int(uint32(in[1]) | uint32(in[2])<<8 | uint32(in[3])<<16 | uint32(in[4])<<24)
|
||||
length = 1 + int(in[0])>>2
|
||||
s += 5
|
||||
length = 1 + int(src[s-5])>>2
|
||||
offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
|
||||
}
|
||||
|
||||
if offset <= 0 || d < offset || length > len(dst)-d {
|
||||
if debug {
|
||||
fmt.Println("corrupt: match, length", length, "offset:", offset, "dst avail:", len(dst)-d, "dst pos:", d)
|
||||
}
|
||||
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
|
||||
@ -163,6 +182,9 @@ func s2Decode(dst, src []byte) int {
|
||||
}
|
||||
length = int(x) + 1
|
||||
if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
|
||||
if debug {
|
||||
fmt.Println("corrupt: lit size", length)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
if debug {
|
||||
@ -229,6 +251,9 @@ func s2Decode(dst, src []byte) int {
|
||||
}
|
||||
|
||||
if offset <= 0 || d < offset || length > len(dst)-d {
|
||||
if debug {
|
||||
fmt.Println("corrupt: match, length", length, "offset:", offset, "dst avail:", len(dst)-d, "dst pos:", d)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
|
||||
|
||||
331
vendor/github.com/klauspost/compress/s2/dict.go
generated
vendored
Normal file
331
vendor/github.com/klauspost/compress/s2/dict.go
generated
vendored
Normal file
@ -0,0 +1,331 @@
|
||||
// Copyright (c) 2022+ Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
// MinDictSize is the minimum dictionary size when repeat has been read.
|
||||
MinDictSize = 16
|
||||
|
||||
// MaxDictSize is the maximum dictionary size when repeat has been read.
|
||||
MaxDictSize = 65536
|
||||
|
||||
// MaxDictSrcOffset is the maximum offset where a dictionary entry can start.
|
||||
MaxDictSrcOffset = 65535
|
||||
)
|
||||
|
||||
// Dict contains a dictionary that can be used for encoding and decoding s2
|
||||
type Dict struct {
|
||||
dict []byte
|
||||
repeat int // Repeat as index of dict
|
||||
|
||||
fast, better, best sync.Once
|
||||
fastTable *[1 << 14]uint16
|
||||
|
||||
betterTableShort *[1 << 14]uint16
|
||||
betterTableLong *[1 << 17]uint16
|
||||
|
||||
bestTableShort *[1 << 16]uint32
|
||||
bestTableLong *[1 << 19]uint32
|
||||
}
|
||||
|
||||
// NewDict will read a dictionary.
|
||||
// It will return nil if the dictionary is invalid.
|
||||
func NewDict(dict []byte) *Dict {
|
||||
if len(dict) == 0 {
|
||||
return nil
|
||||
}
|
||||
var d Dict
|
||||
// Repeat is the first value of the dict
|
||||
r, n := binary.Uvarint(dict)
|
||||
if n <= 0 {
|
||||
return nil
|
||||
}
|
||||
dict = dict[n:]
|
||||
d.dict = dict
|
||||
if cap(d.dict) < len(d.dict)+16 {
|
||||
d.dict = append(make([]byte, 0, len(d.dict)+16), d.dict...)
|
||||
}
|
||||
if len(dict) < MinDictSize || len(dict) > MaxDictSize {
|
||||
return nil
|
||||
}
|
||||
d.repeat = int(r)
|
||||
if d.repeat > len(dict) {
|
||||
return nil
|
||||
}
|
||||
return &d
|
||||
}
|
||||
|
||||
// Bytes will return a serialized version of the dictionary.
|
||||
// The output can be sent to NewDict.
|
||||
func (d *Dict) Bytes() []byte {
|
||||
dst := make([]byte, binary.MaxVarintLen16+len(d.dict))
|
||||
return append(dst[:binary.PutUvarint(dst, uint64(d.repeat))], d.dict...)
|
||||
}
|
||||
|
||||
// MakeDict will create a dictionary.
|
||||
// 'data' must be at least MinDictSize.
|
||||
// If data is longer than MaxDictSize only the last MaxDictSize bytes will be used.
|
||||
// If searchStart is set the start repeat value will be set to the last
|
||||
// match of this content.
|
||||
// If no matches are found, it will attempt to find shorter matches.
|
||||
// This content should match the typical start of a block.
|
||||
// If at least 4 bytes cannot be matched, repeat is set to start of block.
|
||||
func MakeDict(data []byte, searchStart []byte) *Dict {
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(data) > MaxDictSize {
|
||||
data = data[len(data)-MaxDictSize:]
|
||||
}
|
||||
var d Dict
|
||||
dict := data
|
||||
d.dict = dict
|
||||
if cap(d.dict) < len(d.dict)+16 {
|
||||
d.dict = append(make([]byte, 0, len(d.dict)+16), d.dict...)
|
||||
}
|
||||
if len(dict) < MinDictSize {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Find the longest match possible, last entry if multiple.
|
||||
for s := len(searchStart); s > 4; s-- {
|
||||
if idx := bytes.LastIndex(data, searchStart[:s]); idx >= 0 && idx <= len(data)-8 {
|
||||
d.repeat = idx
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return &d
|
||||
}
|
||||
|
||||
// Encode returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
//
|
||||
// The blocks will require the same amount of memory to decode as encoding,
|
||||
// and does not make for concurrent decoding.
|
||||
// Also note that blocks do not contain CRC information, so corruption may be undetected.
|
||||
//
|
||||
// If you need to encode larger amounts of data, consider using
|
||||
// the streaming interface which gives all of these features.
|
||||
func (d *Dict) Encode(dst, src []byte) []byte {
|
||||
if n := MaxEncodedLen(len(src)); n < 0 {
|
||||
panic(ErrTooLarge)
|
||||
} else if cap(dst) < n {
|
||||
dst = make([]byte, n)
|
||||
} else {
|
||||
dst = dst[:n]
|
||||
}
|
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
dstP := binary.PutUvarint(dst, uint64(len(src)))
|
||||
|
||||
if len(src) == 0 {
|
||||
return dst[:dstP]
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
dstP += emitLiteral(dst[dstP:], src)
|
||||
return dst[:dstP]
|
||||
}
|
||||
n := encodeBlockDictGo(dst[dstP:], src, d)
|
||||
if n > 0 {
|
||||
dstP += n
|
||||
return dst[:dstP]
|
||||
}
|
||||
// Not compressible
|
||||
dstP += emitLiteral(dst[dstP:], src)
|
||||
return dst[:dstP]
|
||||
}
|
||||
|
||||
// EncodeBetter returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// EncodeBetter compresses better than Encode but typically with a
|
||||
// 10-40% speed decrease on both compression and decompression.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
//
|
||||
// The blocks will require the same amount of memory to decode as encoding,
|
||||
// and does not make for concurrent decoding.
|
||||
// Also note that blocks do not contain CRC information, so corruption may be undetected.
|
||||
//
|
||||
// If you need to encode larger amounts of data, consider using
|
||||
// the streaming interface which gives all of these features.
|
||||
func (d *Dict) EncodeBetter(dst, src []byte) []byte {
|
||||
if n := MaxEncodedLen(len(src)); n < 0 {
|
||||
panic(ErrTooLarge)
|
||||
} else if len(dst) < n {
|
||||
dst = make([]byte, n)
|
||||
}
|
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
dstP := binary.PutUvarint(dst, uint64(len(src)))
|
||||
|
||||
if len(src) == 0 {
|
||||
return dst[:dstP]
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
dstP += emitLiteral(dst[dstP:], src)
|
||||
return dst[:dstP]
|
||||
}
|
||||
n := encodeBlockBetterDict(dst[dstP:], src, d)
|
||||
if n > 0 {
|
||||
dstP += n
|
||||
return dst[:dstP]
|
||||
}
|
||||
// Not compressible
|
||||
dstP += emitLiteral(dst[dstP:], src)
|
||||
return dst[:dstP]
|
||||
}
|
||||
|
||||
// EncodeBest returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// EncodeBest compresses as good as reasonably possible but with a
|
||||
// big speed decrease.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
//
|
||||
// The blocks will require the same amount of memory to decode as encoding,
|
||||
// and does not make for concurrent decoding.
|
||||
// Also note that blocks do not contain CRC information, so corruption may be undetected.
|
||||
//
|
||||
// If you need to encode larger amounts of data, consider using
|
||||
// the streaming interface which gives all of these features.
|
||||
func (d *Dict) EncodeBest(dst, src []byte) []byte {
|
||||
if n := MaxEncodedLen(len(src)); n < 0 {
|
||||
panic(ErrTooLarge)
|
||||
} else if len(dst) < n {
|
||||
dst = make([]byte, n)
|
||||
}
|
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
dstP := binary.PutUvarint(dst, uint64(len(src)))
|
||||
|
||||
if len(src) == 0 {
|
||||
return dst[:dstP]
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
dstP += emitLiteral(dst[dstP:], src)
|
||||
return dst[:dstP]
|
||||
}
|
||||
n := encodeBlockBest(dst[dstP:], src, d)
|
||||
if n > 0 {
|
||||
dstP += n
|
||||
return dst[:dstP]
|
||||
}
|
||||
// Not compressible
|
||||
dstP += emitLiteral(dst[dstP:], src)
|
||||
return dst[:dstP]
|
||||
}
|
||||
|
||||
// Decode returns the decoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire decoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
func (d *Dict) Decode(dst, src []byte) ([]byte, error) {
|
||||
dLen, s, err := decodedLen(src)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if dLen <= cap(dst) {
|
||||
dst = dst[:dLen]
|
||||
} else {
|
||||
dst = make([]byte, dLen)
|
||||
}
|
||||
if s2DecodeDict(dst, src[s:], d) != 0 {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
func (d *Dict) initFast() {
|
||||
d.fast.Do(func() {
|
||||
const (
|
||||
tableBits = 14
|
||||
maxTableSize = 1 << tableBits
|
||||
)
|
||||
|
||||
var table [maxTableSize]uint16
|
||||
// We stop so any entry of length 8 can always be read.
|
||||
for i := 0; i < len(d.dict)-8-2; i += 3 {
|
||||
x0 := load64(d.dict, i)
|
||||
h0 := hash6(x0, tableBits)
|
||||
h1 := hash6(x0>>8, tableBits)
|
||||
h2 := hash6(x0>>16, tableBits)
|
||||
table[h0] = uint16(i)
|
||||
table[h1] = uint16(i + 1)
|
||||
table[h2] = uint16(i + 2)
|
||||
}
|
||||
d.fastTable = &table
|
||||
})
|
||||
}
|
||||
|
||||
func (d *Dict) initBetter() {
|
||||
d.better.Do(func() {
|
||||
const (
|
||||
// Long hash matches.
|
||||
lTableBits = 17
|
||||
maxLTableSize = 1 << lTableBits
|
||||
|
||||
// Short hash matches.
|
||||
sTableBits = 14
|
||||
maxSTableSize = 1 << sTableBits
|
||||
)
|
||||
|
||||
var lTable [maxLTableSize]uint16
|
||||
var sTable [maxSTableSize]uint16
|
||||
|
||||
// We stop so any entry of length 8 can always be read.
|
||||
for i := 0; i < len(d.dict)-8; i++ {
|
||||
cv := load64(d.dict, i)
|
||||
lTable[hash7(cv, lTableBits)] = uint16(i)
|
||||
sTable[hash4(cv, sTableBits)] = uint16(i)
|
||||
}
|
||||
d.betterTableShort = &sTable
|
||||
d.betterTableLong = &lTable
|
||||
})
|
||||
}
|
||||
|
||||
func (d *Dict) initBest() {
|
||||
d.best.Do(func() {
|
||||
const (
|
||||
// Long hash matches.
|
||||
lTableBits = 19
|
||||
maxLTableSize = 1 << lTableBits
|
||||
|
||||
// Short hash matches.
|
||||
sTableBits = 16
|
||||
maxSTableSize = 1 << sTableBits
|
||||
)
|
||||
|
||||
var lTable [maxLTableSize]uint32
|
||||
var sTable [maxSTableSize]uint32
|
||||
|
||||
// We stop so any entry of length 8 can always be read.
|
||||
for i := 0; i < len(d.dict)-8; i++ {
|
||||
cv := load64(d.dict, i)
|
||||
hashL := hash8(cv, lTableBits)
|
||||
hashS := hash4(cv, sTableBits)
|
||||
candidateL := lTable[hashL]
|
||||
candidateS := sTable[hashS]
|
||||
lTable[hashL] = uint32(i) | candidateL<<16
|
||||
sTable[hashS] = uint32(i) | candidateS<<16
|
||||
}
|
||||
d.bestTableShort = &sTable
|
||||
d.bestTableLong = &lTable
|
||||
})
|
||||
}
|
||||
54
vendor/github.com/klauspost/compress/s2/encode.go
generated
vendored
54
vendor/github.com/klauspost/compress/s2/encode.go
generated
vendored
@ -58,6 +58,32 @@ func Encode(dst, src []byte) []byte {
|
||||
return dst[:d]
|
||||
}
|
||||
|
||||
// EstimateBlockSize will perform a very fast compression
|
||||
// without outputting the result and return the compressed output size.
|
||||
// The function returns -1 if no improvement could be achieved.
|
||||
// Using actual compression will most often produce better compression than the estimate.
|
||||
func EstimateBlockSize(src []byte) (d int) {
|
||||
if len(src) < 6 || int64(len(src)) > 0xffffffff {
|
||||
return -1
|
||||
}
|
||||
if len(src) <= 1024 {
|
||||
d = calcBlockSizeSmall(src)
|
||||
} else {
|
||||
d = calcBlockSize(src)
|
||||
}
|
||||
|
||||
if d == 0 {
|
||||
return -1
|
||||
}
|
||||
// Size of the varint encoded block size.
|
||||
d += (bits.Len64(uint64(len(src))) + 7) / 7
|
||||
|
||||
if d >= len(src) {
|
||||
return -1
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// EncodeBetter returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
@ -132,7 +158,7 @@ func EncodeBest(dst, src []byte) []byte {
|
||||
d += emitLiteral(dst[d:], src)
|
||||
return dst[:d]
|
||||
}
|
||||
n := encodeBlockBest(dst[d:], src)
|
||||
n := encodeBlockBest(dst[d:], src, nil)
|
||||
if n > 0 {
|
||||
d += n
|
||||
return dst[:d]
|
||||
@ -404,10 +430,11 @@ type Writer struct {
|
||||
buffers sync.Pool
|
||||
pad int
|
||||
|
||||
writer io.Writer
|
||||
randSrc io.Reader
|
||||
writerWg sync.WaitGroup
|
||||
index Index
|
||||
writer io.Writer
|
||||
randSrc io.Reader
|
||||
writerWg sync.WaitGroup
|
||||
index Index
|
||||
customEnc func(dst, src []byte) int
|
||||
|
||||
// wroteStreamHeader is whether we have written the stream header.
|
||||
wroteStreamHeader bool
|
||||
@ -773,6 +800,9 @@ func (w *Writer) EncodeBuffer(buf []byte) (err error) {
|
||||
}
|
||||
|
||||
func (w *Writer) encodeBlock(obuf, uncompressed []byte) int {
|
||||
if w.customEnc != nil {
|
||||
return w.customEnc(obuf, uncompressed)
|
||||
}
|
||||
if w.snappy {
|
||||
switch w.level {
|
||||
case levelFast:
|
||||
@ -790,7 +820,7 @@ func (w *Writer) encodeBlock(obuf, uncompressed []byte) int {
|
||||
case levelBetter:
|
||||
return encodeBlockBetter(obuf, uncompressed)
|
||||
case levelBest:
|
||||
return encodeBlockBest(obuf, uncompressed)
|
||||
return encodeBlockBest(obuf, uncompressed, nil)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@ -1339,3 +1369,15 @@ func WriterFlushOnWrite() WriterOption {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WriterCustomEncoder allows to override the encoder for blocks on the stream.
|
||||
// The function must compress 'src' into 'dst' and return the bytes used in dst as an integer.
|
||||
// Block size (initial varint) should not be added by the encoder.
|
||||
// Returning value 0 indicates the block could not be compressed.
|
||||
// The function should expect to be called concurrently.
|
||||
func WriterCustomEncoder(fn func(dst, src []byte) int) WriterOption {
|
||||
return func(w *Writer) error {
|
||||
w.customEnc = fn
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
595
vendor/github.com/klauspost/compress/s2/encode_all.go
generated
vendored
595
vendor/github.com/klauspost/compress/s2/encode_all.go
generated
vendored
@ -8,6 +8,7 @@ package s2
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
@ -58,8 +59,9 @@ func encodeGo(dst, src []byte) []byte {
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockGo(dst, src []byte) (d int) {
|
||||
// Initialize the hash table.
|
||||
const (
|
||||
@ -454,3 +456,594 @@ emitRemainder:
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// encodeBlockGo encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockDictGo(dst, src []byte, dict *Dict) (d int) {
|
||||
// Initialize the hash table.
|
||||
const (
|
||||
tableBits = 14
|
||||
maxTableSize = 1 << tableBits
|
||||
maxAhead = 8 // maximum bytes ahead without checking sLimit
|
||||
|
||||
debug = false
|
||||
)
|
||||
dict.initFast()
|
||||
|
||||
var table [maxTableSize]uint32
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
if sLimit > MaxDictSrcOffset-maxAhead {
|
||||
sLimit = MaxDictSrcOffset - maxAhead
|
||||
}
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - len(src)>>5 - 5
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form can start with a dict entry (copy or repeat).
|
||||
s := 0
|
||||
|
||||
// Convert dict repeat to offset
|
||||
repeat := len(dict.dict) - dict.repeat
|
||||
cv := load64(src, 0)
|
||||
|
||||
// While in dict
|
||||
searchDict:
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := s + (s-nextEmit)>>6 + 4
|
||||
hash0 := hash6(cv, tableBits)
|
||||
hash1 := hash6(cv>>8, tableBits)
|
||||
if nextS > sLimit {
|
||||
if debug {
|
||||
fmt.Println("slimit reached", s, nextS)
|
||||
}
|
||||
break searchDict
|
||||
}
|
||||
candidateDict := int(dict.fastTable[hash0])
|
||||
candidateDict2 := int(dict.fastTable[hash1])
|
||||
candidate2 := int(table[hash1])
|
||||
candidate := int(table[hash0])
|
||||
table[hash0] = uint32(s)
|
||||
table[hash1] = uint32(s + 1)
|
||||
hash2 := hash6(cv>>16, tableBits)
|
||||
|
||||
// Check repeat at offset checkRep.
|
||||
const checkRep = 1
|
||||
|
||||
if repeat > s {
|
||||
candidate := len(dict.dict) - repeat + s
|
||||
if repeat-s >= 4 && uint32(cv) == load32(dict.dict, candidate) {
|
||||
// Extend back
|
||||
base := s
|
||||
for i := candidate; base > nextEmit && i > 0 && dict.dict[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
if debug && nextEmit != base {
|
||||
fmt.Println("emitted ", base-nextEmit, "literals")
|
||||
}
|
||||
s += 4
|
||||
candidate += 4
|
||||
for candidate < len(dict.dict)-8 && s <= len(src)-8 {
|
||||
if diff := load64(src, s) ^ load64(dict.dict, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
d += emitRepeat(dst[d:], repeat, s-base)
|
||||
if debug {
|
||||
fmt.Println("emitted dict repeat length", s-base, "offset:", repeat, "s:", s)
|
||||
}
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
break searchDict
|
||||
}
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
} else if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) {
|
||||
base := s + checkRep
|
||||
// Extend back
|
||||
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
if debug && nextEmit != base {
|
||||
fmt.Println("emitted ", base-nextEmit, "literals")
|
||||
}
|
||||
|
||||
// Extend forward
|
||||
candidate := s - repeat + 4 + checkRep
|
||||
s += 4 + checkRep
|
||||
for s <= sLimit {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
if debug {
|
||||
// Validate match.
|
||||
if s <= candidate {
|
||||
panic("s <= candidate")
|
||||
}
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
if nextEmit > 0 {
|
||||
// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
|
||||
d += emitRepeat(dst[d:], repeat, s-base)
|
||||
} else {
|
||||
// First match, cannot be repeat.
|
||||
d += emitCopy(dst[d:], repeat, s-base)
|
||||
}
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
break searchDict
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("emitted reg repeat", s-base, "s:", s)
|
||||
}
|
||||
cv = load64(src, s)
|
||||
continue searchDict
|
||||
}
|
||||
if s == 0 {
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
continue searchDict
|
||||
}
|
||||
// Start with table. These matches will always be closer.
|
||||
if uint32(cv) == load32(src, candidate) {
|
||||
goto emitMatch
|
||||
}
|
||||
candidate = int(table[hash2])
|
||||
if uint32(cv>>8) == load32(src, candidate2) {
|
||||
table[hash2] = uint32(s + 2)
|
||||
candidate = candidate2
|
||||
s++
|
||||
goto emitMatch
|
||||
}
|
||||
|
||||
// Check dict. Dicts have longer offsets, so we want longer matches.
|
||||
if cv == load64(dict.dict, candidateDict) {
|
||||
table[hash2] = uint32(s + 2)
|
||||
goto emitDict
|
||||
}
|
||||
|
||||
candidateDict = int(dict.fastTable[hash2])
|
||||
// Check if upper 7 bytes match
|
||||
if candidateDict2 >= 1 {
|
||||
if cv^load64(dict.dict, candidateDict2-1) < (1 << 8) {
|
||||
table[hash2] = uint32(s + 2)
|
||||
candidateDict = candidateDict2
|
||||
s++
|
||||
goto emitDict
|
||||
}
|
||||
}
|
||||
|
||||
table[hash2] = uint32(s + 2)
|
||||
if uint32(cv>>16) == load32(src, candidate) {
|
||||
s += 2
|
||||
goto emitMatch
|
||||
}
|
||||
if candidateDict >= 2 {
|
||||
// Check if upper 6 bytes match
|
||||
if cv^load64(dict.dict, candidateDict-2) < (1 << 16) {
|
||||
s += 2
|
||||
goto emitDict
|
||||
}
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
continue searchDict
|
||||
|
||||
emitDict:
|
||||
{
|
||||
if debug {
|
||||
if load32(dict.dict, candidateDict) != load32(src, s) {
|
||||
panic("dict emit mismatch")
|
||||
}
|
||||
}
|
||||
// Extend backwards.
|
||||
// The top bytes will be rechecked to get the full match.
|
||||
for candidateDict > 0 && s > nextEmit && dict.dict[candidateDict-1] == src[s-1] {
|
||||
candidateDict--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
d += emitLiteral(dst[d:], src[nextEmit:s])
|
||||
if debug && nextEmit != s {
|
||||
fmt.Println("emitted ", s-nextEmit, "literals")
|
||||
}
|
||||
{
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
base := s
|
||||
repeat = s + (len(dict.dict)) - candidateDict
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidateDict += 4
|
||||
for s <= len(src)-8 && len(dict.dict)-candidateDict >= 8 {
|
||||
if diff := load64(src, s) ^ load64(dict.dict, candidateDict); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidateDict += 8
|
||||
}
|
||||
|
||||
// Matches longer than 64 are split.
|
||||
if s <= sLimit || s-base < 8 {
|
||||
d += emitCopy(dst[d:], repeat, s-base)
|
||||
} else {
|
||||
// Split to ensure we don't start a copy within next block
|
||||
d += emitCopy(dst[d:], repeat, 4)
|
||||
d += emitRepeat(dst[d:], repeat, s-base-4)
|
||||
}
|
||||
if false {
|
||||
// Validate match.
|
||||
if s <= candidate {
|
||||
panic("s <= candidate")
|
||||
}
|
||||
a := src[base:s]
|
||||
b := dict.dict[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("emitted dict copy, length", s-base, "offset:", repeat, "s:", s)
|
||||
}
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
break searchDict
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
|
||||
// Index and continue loop to try new candidate.
|
||||
x := load64(src, s-2)
|
||||
m2Hash := hash6(x, tableBits)
|
||||
currHash := hash6(x>>8, tableBits)
|
||||
candidate = int(table[currHash])
|
||||
table[m2Hash] = uint32(s - 2)
|
||||
table[currHash] = uint32(s - 1)
|
||||
cv = load64(src, s)
|
||||
}
|
||||
continue
|
||||
}
|
||||
emitMatch:
|
||||
|
||||
// Extend backwards.
|
||||
// The top bytes will be rechecked to get the full match.
|
||||
for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] {
|
||||
candidate--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
d += emitLiteral(dst[d:], src[nextEmit:s])
|
||||
if debug && nextEmit != s {
|
||||
fmt.Println("emitted ", s-nextEmit, "literals")
|
||||
}
|
||||
// Call emitCopy, and then see if another emitCopy could be our next
|
||||
// move. Repeat until we find no match for the input immediately after
|
||||
// what was consumed by the last emitCopy call.
|
||||
//
|
||||
// If we exit this loop normally then we need to call emitLiteral next,
|
||||
// though we don't yet know how big the literal will be. We handle that
|
||||
// by proceeding to the next iteration of the main loop. We also can
|
||||
// exit this loop via goto if we get close to exhausting the input.
|
||||
for {
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
base := s
|
||||
repeat = base - candidate
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidate += 4
|
||||
for s <= len(src)-8 {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopy(dst[d:], repeat, s-base)
|
||||
if debug {
|
||||
// Validate match.
|
||||
if s <= candidate {
|
||||
panic("s <= candidate")
|
||||
}
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("emitted src copy, length", s-base, "offset:", repeat, "s:", s)
|
||||
}
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
break searchDict
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Check for an immediate match, otherwise start search at s+1
|
||||
x := load64(src, s-2)
|
||||
m2Hash := hash6(x, tableBits)
|
||||
currHash := hash6(x>>16, tableBits)
|
||||
candidate = int(table[currHash])
|
||||
table[m2Hash] = uint32(s - 2)
|
||||
table[currHash] = uint32(s)
|
||||
if debug && s == candidate {
|
||||
panic("s == candidate")
|
||||
}
|
||||
if uint32(x>>16) != load32(src, candidate) {
|
||||
cv = load64(src, s+1)
|
||||
s++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Search without dict:
|
||||
if repeat > s {
|
||||
repeat = 0
|
||||
}
|
||||
|
||||
// No more dict
|
||||
sLimit = len(src) - inputMargin
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("non-dict matching at", s, "repeat:", repeat)
|
||||
}
|
||||
cv = load64(src, s)
|
||||
if debug {
|
||||
fmt.Println("now", s, "->", sLimit, "out:", d, "left:", len(src)-s, "nextemit:", nextEmit, "dstLimit:", dstLimit, "s:", s)
|
||||
}
|
||||
for {
|
||||
candidate := 0
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := s + (s-nextEmit)>>6 + 4
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hash0 := hash6(cv, tableBits)
|
||||
hash1 := hash6(cv>>8, tableBits)
|
||||
candidate = int(table[hash0])
|
||||
candidate2 := int(table[hash1])
|
||||
table[hash0] = uint32(s)
|
||||
table[hash1] = uint32(s + 1)
|
||||
hash2 := hash6(cv>>16, tableBits)
|
||||
|
||||
// Check repeat at offset checkRep.
|
||||
const checkRep = 1
|
||||
if repeat > 0 && uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) {
|
||||
base := s + checkRep
|
||||
// Extend back
|
||||
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
if debug && nextEmit != base {
|
||||
fmt.Println("emitted ", base-nextEmit, "literals")
|
||||
}
|
||||
// Extend forward
|
||||
candidate := s - repeat + 4 + checkRep
|
||||
s += 4 + checkRep
|
||||
for s <= sLimit {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
if debug {
|
||||
// Validate match.
|
||||
if s <= candidate {
|
||||
panic("s <= candidate")
|
||||
}
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
if nextEmit > 0 {
|
||||
// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
|
||||
d += emitRepeat(dst[d:], repeat, s-base)
|
||||
} else {
|
||||
// First match, cannot be repeat.
|
||||
d += emitCopy(dst[d:], repeat, s-base)
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("emitted src repeat length", s-base, "offset:", repeat, "s:", s)
|
||||
}
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
if uint32(cv) == load32(src, candidate) {
|
||||
break
|
||||
}
|
||||
candidate = int(table[hash2])
|
||||
if uint32(cv>>8) == load32(src, candidate2) {
|
||||
table[hash2] = uint32(s + 2)
|
||||
candidate = candidate2
|
||||
s++
|
||||
break
|
||||
}
|
||||
table[hash2] = uint32(s + 2)
|
||||
if uint32(cv>>16) == load32(src, candidate) {
|
||||
s += 2
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards.
|
||||
// The top bytes will be rechecked to get the full match.
|
||||
for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] {
|
||||
candidate--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
d += emitLiteral(dst[d:], src[nextEmit:s])
|
||||
if debug && nextEmit != s {
|
||||
fmt.Println("emitted ", s-nextEmit, "literals")
|
||||
}
|
||||
// Call emitCopy, and then see if another emitCopy could be our next
|
||||
// move. Repeat until we find no match for the input immediately after
|
||||
// what was consumed by the last emitCopy call.
|
||||
//
|
||||
// If we exit this loop normally then we need to call emitLiteral next,
|
||||
// though we don't yet know how big the literal will be. We handle that
|
||||
// by proceeding to the next iteration of the main loop. We also can
|
||||
// exit this loop via goto if we get close to exhausting the input.
|
||||
for {
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
base := s
|
||||
repeat = base - candidate
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidate += 4
|
||||
for s <= len(src)-8 {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopy(dst[d:], repeat, s-base)
|
||||
if debug {
|
||||
// Validate match.
|
||||
if s <= candidate {
|
||||
panic("s <= candidate")
|
||||
}
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("emitted src copy, length", s-base, "offset:", repeat, "s:", s)
|
||||
}
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Check for an immediate match, otherwise start search at s+1
|
||||
x := load64(src, s-2)
|
||||
m2Hash := hash6(x, tableBits)
|
||||
currHash := hash6(x>>16, tableBits)
|
||||
candidate = int(table[currHash])
|
||||
table[m2Hash] = uint32(s - 2)
|
||||
table[currHash] = uint32(s)
|
||||
if debug && s == candidate {
|
||||
panic("s == candidate")
|
||||
}
|
||||
if uint32(x>>16) != load32(src, candidate) {
|
||||
cv = load64(src, s+1)
|
||||
s++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:])
|
||||
if debug && nextEmit != s {
|
||||
fmt.Println("emitted ", len(src)-nextEmit, "literals")
|
||||
}
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
14
vendor/github.com/klauspost/compress/s2/encode_amd64.go
generated
vendored
14
vendor/github.com/klauspost/compress/s2/encode_amd64.go
generated
vendored
@ -3,13 +3,16 @@
|
||||
|
||||
package s2
|
||||
|
||||
const hasAmd64Asm = true
|
||||
|
||||
// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlock(dst, src []byte) (d int) {
|
||||
const (
|
||||
// Use 12 bit table when less than...
|
||||
@ -43,8 +46,9 @@ func encodeBlock(dst, src []byte) (d int) {
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBetter(dst, src []byte) (d int) {
|
||||
const (
|
||||
// Use 12 bit table when less than...
|
||||
@ -78,8 +82,9 @@ func encodeBlockBetter(dst, src []byte) (d int) {
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockSnappy(dst, src []byte) (d int) {
|
||||
const (
|
||||
// Use 12 bit table when less than...
|
||||
@ -112,8 +117,9 @@ func encodeBlockSnappy(dst, src []byte) (d int) {
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBetterSnappy(dst, src []byte) (d int) {
|
||||
const (
|
||||
// Use 12 bit table when less than...
|
||||
|
||||
225
vendor/github.com/klauspost/compress/s2/encode_best.go
generated
vendored
225
vendor/github.com/klauspost/compress/s2/encode_best.go
generated
vendored
@ -7,6 +7,7 @@ package s2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
@ -15,9 +16,10 @@ import (
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBest(dst, src []byte) (d int) {
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBest(dst, src []byte, dict *Dict) (d int) {
|
||||
// Initialize the hash tables.
|
||||
const (
|
||||
// Long hash matches.
|
||||
@ -29,6 +31,8 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
||||
maxSTableSize = 1 << sTableBits
|
||||
|
||||
inputMargin = 8 + 2
|
||||
|
||||
debug = false
|
||||
)
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
@ -38,6 +42,10 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
sLimitDict := len(src) - inputMargin
|
||||
if sLimitDict > MaxDictSrcOffset-inputMargin {
|
||||
sLimitDict = MaxDictSrcOffset - inputMargin
|
||||
}
|
||||
|
||||
var lTable [maxLTableSize]uint64
|
||||
var sTable [maxSTableSize]uint64
|
||||
@ -51,10 +59,15 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
repeat := 1
|
||||
if dict != nil {
|
||||
dict.initBest()
|
||||
s = 0
|
||||
repeat = len(dict.dict) - dict.repeat
|
||||
}
|
||||
cv := load64(src, s)
|
||||
|
||||
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
|
||||
repeat := 1
|
||||
const lowbitMask = 0xffffffff
|
||||
getCur := func(x uint64) int {
|
||||
return int(x & lowbitMask)
|
||||
@ -66,11 +79,11 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
||||
|
||||
for {
|
||||
type match struct {
|
||||
offset int
|
||||
s int
|
||||
length int
|
||||
score int
|
||||
rep bool
|
||||
offset int
|
||||
s int
|
||||
length int
|
||||
score int
|
||||
rep, dict bool
|
||||
}
|
||||
var best match
|
||||
for {
|
||||
@ -84,6 +97,12 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
if dict != nil && s >= MaxDictSrcOffset {
|
||||
dict = nil
|
||||
if repeat > s {
|
||||
repeat = math.MinInt32
|
||||
}
|
||||
}
|
||||
hashL := hash8(cv, lTableBits)
|
||||
hashS := hash4(cv, sTableBits)
|
||||
candidateL := lTable[hashL]
|
||||
@ -113,7 +132,15 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
||||
}
|
||||
m := match{offset: offset, s: s, length: 4 + offset, rep: rep}
|
||||
s += 4
|
||||
for s <= sLimit {
|
||||
for s < len(src) {
|
||||
if len(src)-s < 8 {
|
||||
if src[s] == src[m.length] {
|
||||
m.length++
|
||||
s++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if diff := load64(src, s) ^ load64(src, m.length); diff != 0 {
|
||||
m.length += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
@ -129,6 +156,62 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
||||
}
|
||||
return m
|
||||
}
|
||||
matchDict := func(candidate, s int, first uint32, rep bool) match {
|
||||
// Calculate offset as if in continuous array with s
|
||||
offset := -len(dict.dict) + candidate
|
||||
if best.length != 0 && best.s-best.offset == s-offset && !rep {
|
||||
// Don't retest if we have the same offset.
|
||||
return match{offset: offset, s: s}
|
||||
}
|
||||
|
||||
if load32(dict.dict, candidate) != first {
|
||||
return match{offset: offset, s: s}
|
||||
}
|
||||
m := match{offset: offset, s: s, length: 4 + candidate, rep: rep, dict: true}
|
||||
s += 4
|
||||
if !rep {
|
||||
for s < sLimitDict && m.length < len(dict.dict) {
|
||||
if len(src)-s < 8 || len(dict.dict)-m.length < 8 {
|
||||
if src[s] == dict.dict[m.length] {
|
||||
m.length++
|
||||
s++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if diff := load64(src, s) ^ load64(dict.dict, m.length); diff != 0 {
|
||||
m.length += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
m.length += 8
|
||||
}
|
||||
} else {
|
||||
for s < len(src) && m.length < len(dict.dict) {
|
||||
if len(src)-s < 8 || len(dict.dict)-m.length < 8 {
|
||||
if src[s] == dict.dict[m.length] {
|
||||
m.length++
|
||||
s++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if diff := load64(src, s) ^ load64(dict.dict, m.length); diff != 0 {
|
||||
m.length += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
m.length += 8
|
||||
}
|
||||
}
|
||||
m.length -= candidate
|
||||
m.score = score(m)
|
||||
if m.score <= -m.s {
|
||||
// Eliminate if no savings, we might find a better one.
|
||||
m.length = 0
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
bestOf := func(a, b match) match {
|
||||
if b.length == 0 {
|
||||
@ -145,45 +228,99 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
||||
return b
|
||||
}
|
||||
|
||||
best = bestOf(matchAt(getCur(candidateL), s, uint32(cv), false), matchAt(getPrev(candidateL), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getCur(candidateS), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(candidateS), s, uint32(cv), false))
|
||||
|
||||
if s > 0 {
|
||||
best = bestOf(matchAt(getCur(candidateL), s, uint32(cv), false), matchAt(getPrev(candidateL), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getCur(candidateS), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(candidateS), s, uint32(cv), false))
|
||||
}
|
||||
if dict != nil {
|
||||
candidateL := dict.bestTableLong[hashL]
|
||||
candidateS := dict.bestTableShort[hashS]
|
||||
best = bestOf(best, matchDict(int(candidateL&0xffff), s, uint32(cv), false))
|
||||
best = bestOf(best, matchDict(int(candidateL>>16), s, uint32(cv), false))
|
||||
best = bestOf(best, matchDict(int(candidateS&0xffff), s, uint32(cv), false))
|
||||
best = bestOf(best, matchDict(int(candidateS>>16), s, uint32(cv), false))
|
||||
}
|
||||
{
|
||||
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8), true))
|
||||
if (dict == nil || repeat <= s) && repeat > 0 {
|
||||
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8), true))
|
||||
} else if s-repeat < -4 && dict != nil {
|
||||
candidate := len(dict.dict) - (repeat - s)
|
||||
best = bestOf(best, matchDict(candidate, s, uint32(cv), true))
|
||||
candidate++
|
||||
best = bestOf(best, matchDict(candidate, s+1, uint32(cv>>8), true))
|
||||
}
|
||||
|
||||
if best.length > 0 {
|
||||
hashS := hash4(cv>>8, sTableBits)
|
||||
// s+1
|
||||
nextShort := sTable[hash4(cv>>8, sTableBits)]
|
||||
nextShort := sTable[hashS]
|
||||
s := s + 1
|
||||
cv := load64(src, s)
|
||||
nextLong := lTable[hash8(cv, lTableBits)]
|
||||
hashL := hash8(cv, lTableBits)
|
||||
nextLong := lTable[hashL]
|
||||
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv), false))
|
||||
// Repeat at + 2
|
||||
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8), true))
|
||||
|
||||
// Dict at + 1
|
||||
if dict != nil {
|
||||
candidateL := dict.bestTableLong[hashL]
|
||||
candidateS := dict.bestTableShort[hashS]
|
||||
|
||||
best = bestOf(best, matchDict(int(candidateL&0xffff), s, uint32(cv), false))
|
||||
best = bestOf(best, matchDict(int(candidateS&0xffff), s, uint32(cv), false))
|
||||
}
|
||||
|
||||
// s+2
|
||||
if true {
|
||||
nextShort = sTable[hash4(cv>>8, sTableBits)]
|
||||
hashS := hash4(cv>>8, sTableBits)
|
||||
|
||||
nextShort = sTable[hashS]
|
||||
s++
|
||||
cv = load64(src, s)
|
||||
nextLong = lTable[hash8(cv, lTableBits)]
|
||||
hashL := hash8(cv, lTableBits)
|
||||
nextLong = lTable[hashL]
|
||||
|
||||
if (dict == nil || repeat <= s) && repeat > 0 {
|
||||
// Repeat at + 2
|
||||
best = bestOf(best, matchAt(s-repeat, s, uint32(cv), true))
|
||||
} else if repeat-s > 4 && dict != nil {
|
||||
candidate := len(dict.dict) - (repeat - s)
|
||||
best = bestOf(best, matchDict(candidate, s, uint32(cv), true))
|
||||
}
|
||||
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv), false))
|
||||
|
||||
// Dict at +2
|
||||
// Very small gain
|
||||
if dict != nil {
|
||||
candidateL := dict.bestTableLong[hashL]
|
||||
candidateS := dict.bestTableShort[hashS]
|
||||
|
||||
best = bestOf(best, matchDict(int(candidateL&0xffff), s, uint32(cv), false))
|
||||
best = bestOf(best, matchDict(int(candidateS&0xffff), s, uint32(cv), false))
|
||||
}
|
||||
}
|
||||
// Search for a match at best match end, see if that is better.
|
||||
if sAt := best.s + best.length; sAt < sLimit {
|
||||
sBack := best.s
|
||||
backL := best.length
|
||||
// Allow some bytes at the beginning to mismatch.
|
||||
// Sweet spot is around 1-2 bytes, but depends on input.
|
||||
// The skipped bytes are tested in Extend backwards,
|
||||
// and still picked up as part of the match if they do.
|
||||
const skipBeginning = 2
|
||||
const skipEnd = 1
|
||||
if sAt := best.s + best.length - skipEnd; sAt < sLimit {
|
||||
|
||||
sBack := best.s + skipBeginning - skipEnd
|
||||
backL := best.length - skipBeginning
|
||||
// Load initial values
|
||||
cv = load64(src, sBack)
|
||||
// Search for mismatch
|
||||
|
||||
// Grab candidates...
|
||||
next := lTable[hash8(load64(src, sAt), lTableBits)]
|
||||
//next := sTable[hash4(load64(src, sAt), sTableBits)]
|
||||
|
||||
if checkAt := getCur(next) - backL; checkAt > 0 {
|
||||
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
|
||||
@ -191,6 +328,16 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
||||
if checkAt := getPrev(next) - backL; checkAt > 0 {
|
||||
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
|
||||
}
|
||||
// Disabled: Extremely small gain
|
||||
if false {
|
||||
next = sTable[hash4(load64(src, sAt), sTableBits)]
|
||||
if checkAt := getCur(next) - backL; checkAt > 0 {
|
||||
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
|
||||
}
|
||||
if checkAt := getPrev(next) - backL; checkAt > 0 {
|
||||
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -209,7 +356,7 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
||||
|
||||
// Extend backwards, not needed for repeats...
|
||||
s = best.s
|
||||
if !best.rep {
|
||||
if !best.rep && !best.dict {
|
||||
for best.offset > 0 && s > nextEmit && src[best.offset-1] == src[s-1] {
|
||||
best.offset--
|
||||
best.length++
|
||||
@ -226,7 +373,6 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
||||
|
||||
base := s
|
||||
offset := s - best.offset
|
||||
|
||||
s += best.length
|
||||
|
||||
if offset > 65535 && s-base <= 5 && !best.rep {
|
||||
@ -238,16 +384,28 @@ func encodeBlockBest(dst, src []byte) (d int) {
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
if debug && nextEmit != base {
|
||||
fmt.Println("EMIT", base-nextEmit, "literals. base-after:", base)
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
if best.rep {
|
||||
if nextEmit > 0 {
|
||||
if nextEmit > 0 || best.dict {
|
||||
if debug {
|
||||
fmt.Println("REPEAT, length", best.length, "offset:", offset, "s-after:", s, "dict:", best.dict, "best:", best)
|
||||
}
|
||||
// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
|
||||
d += emitRepeat(dst[d:], offset, best.length)
|
||||
} else {
|
||||
// First match, cannot be repeat.
|
||||
// First match without dict cannot be a repeat.
|
||||
if debug {
|
||||
fmt.Println("COPY, length", best.length, "offset:", offset, "s-after:", s, "dict:", best.dict, "best:", best)
|
||||
}
|
||||
d += emitCopy(dst[d:], offset, best.length)
|
||||
}
|
||||
} else {
|
||||
if debug {
|
||||
fmt.Println("COPY, length", best.length, "offset:", offset, "s-after:", s, "dict:", best.dict, "best:", best)
|
||||
}
|
||||
d += emitCopy(dst[d:], offset, best.length)
|
||||
}
|
||||
repeat = offset
|
||||
@ -278,6 +436,9 @@ emitRemainder:
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
if debug && nextEmit != s {
|
||||
fmt.Println("emitted ", len(src)-nextEmit, "literals")
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
@ -288,8 +449,9 @@ emitRemainder:
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBestSnappy(dst, src []byte) (d int) {
|
||||
// Initialize the hash tables.
|
||||
const (
|
||||
@ -546,6 +708,7 @@ emitRemainder:
|
||||
// emitCopySize returns the size to encode the offset+length
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
func emitCopySize(offset, length int) int {
|
||||
@ -584,6 +747,7 @@ func emitCopySize(offset, length int) int {
|
||||
// emitCopyNoRepeatSize returns the size to encode the offset+length
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
func emitCopyNoRepeatSize(offset, length int) int {
|
||||
@ -621,7 +785,6 @@ func emitRepeatSize(offset, length int) int {
|
||||
left := 0
|
||||
if length > maxRepeat {
|
||||
left = length - maxRepeat + 4
|
||||
length = maxRepeat - 4
|
||||
}
|
||||
if left > 0 {
|
||||
return 5 + emitRepeatSize(offset, left)
|
||||
|
||||
727
vendor/github.com/klauspost/compress/s2/encode_better.go
generated
vendored
727
vendor/github.com/klauspost/compress/s2/encode_better.go
generated
vendored
@ -6,6 +6,8 @@
|
||||
package s2
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
@ -42,8 +44,9 @@ func hash8(u uint64, h uint8) uint32 {
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBetterGo(dst, src []byte) (d int) {
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
@ -56,7 +59,7 @@ func encodeBlockBetterGo(dst, src []byte) (d int) {
|
||||
// Initialize the hash tables.
|
||||
const (
|
||||
// Long hash matches.
|
||||
lTableBits = 16
|
||||
lTableBits = 17
|
||||
maxLTableSize = 1 << lTableBits
|
||||
|
||||
// Short hash matches.
|
||||
@ -97,9 +100,26 @@ func encodeBlockBetterGo(dst, src []byte) (d int) {
|
||||
lTable[hashL] = uint32(s)
|
||||
sTable[hashS] = uint32(s)
|
||||
|
||||
valLong := load64(src, candidateL)
|
||||
valShort := load64(src, candidateS)
|
||||
|
||||
// If long matches at least 8 bytes, use that.
|
||||
if cv == valLong {
|
||||
break
|
||||
}
|
||||
if cv == valShort {
|
||||
candidateL = candidateS
|
||||
break
|
||||
}
|
||||
|
||||
// Check repeat at offset checkRep.
|
||||
const checkRep = 1
|
||||
if false && uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) {
|
||||
// Minimum length of a repeat. Tested with various values.
|
||||
// While 4-5 offers improvements in some, 6 reduces
|
||||
// regressions significantly.
|
||||
const wantRepeatBytes = 6
|
||||
const repeatMask = ((1 << (wantRepeatBytes * 8)) - 1) << (8 * checkRep)
|
||||
if false && repeat > 0 && cv&repeatMask == load64(src, s-repeat)&repeatMask {
|
||||
base := s + checkRep
|
||||
// Extend back
|
||||
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
|
||||
@ -109,8 +129,8 @@ func encodeBlockBetterGo(dst, src []byte) (d int) {
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
|
||||
// Extend forward
|
||||
candidate := s - repeat + 4 + checkRep
|
||||
s += 4 + checkRep
|
||||
candidate := s - repeat + wantRepeatBytes + checkRep
|
||||
s += wantRepeatBytes + checkRep
|
||||
for s < len(src) {
|
||||
if len(src)-s < 8 {
|
||||
if src[s] == src[candidate] {
|
||||
@ -127,28 +147,40 @@ func encodeBlockBetterGo(dst, src []byte) (d int) {
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
if nextEmit > 0 {
|
||||
// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
|
||||
d += emitRepeat(dst[d:], repeat, s-base)
|
||||
} else {
|
||||
// First match, cannot be repeat.
|
||||
d += emitCopy(dst[d:], repeat, s-base)
|
||||
}
|
||||
// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
|
||||
d += emitRepeat(dst[d:], repeat, s-base)
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
// Index in-between
|
||||
index0 := base + 1
|
||||
index1 := s - 2
|
||||
|
||||
cv = load64(src, s)
|
||||
for index0 < index1 {
|
||||
cv0 := load64(src, index0)
|
||||
cv1 := load64(src, index1)
|
||||
lTable[hash7(cv0, lTableBits)] = uint32(index0)
|
||||
sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
|
||||
|
||||
lTable[hash7(cv1, lTableBits)] = uint32(index1)
|
||||
sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
|
||||
index0 += 2
|
||||
index1 -= 2
|
||||
}
|
||||
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
if uint32(cv) == load32(src, candidateL) {
|
||||
// Long likely matches 7, so take that.
|
||||
if uint32(cv) == uint32(valLong) {
|
||||
break
|
||||
}
|
||||
|
||||
// Check our short candidate
|
||||
if uint32(cv) == load32(src, candidateS) {
|
||||
if uint32(cv) == uint32(valShort) {
|
||||
// Try a long candidate at s+1
|
||||
hashL = hash7(cv>>8, lTableBits)
|
||||
candidateL = int(lTable[hashL])
|
||||
@ -227,21 +259,29 @@ func encodeBlockBetterGo(dst, src []byte) (d int) {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Index match start+1 (long) and start+2 (short)
|
||||
|
||||
// Index short & long
|
||||
index0 := base + 1
|
||||
// Index match end-2 (long) and end-1 (short)
|
||||
index1 := s - 2
|
||||
|
||||
cv0 := load64(src, index0)
|
||||
cv1 := load64(src, index1)
|
||||
cv = load64(src, s)
|
||||
lTable[hash7(cv0, lTableBits)] = uint32(index0)
|
||||
lTable[hash7(cv0>>8, lTableBits)] = uint32(index0 + 1)
|
||||
lTable[hash7(cv1, lTableBits)] = uint32(index1)
|
||||
lTable[hash7(cv1>>8, lTableBits)] = uint32(index1 + 1)
|
||||
sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
|
||||
sTable[hash4(cv0>>16, sTableBits)] = uint32(index0 + 2)
|
||||
|
||||
lTable[hash7(cv1, lTableBits)] = uint32(index1)
|
||||
sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
|
||||
index0 += 1
|
||||
index1 -= 1
|
||||
cv = load64(src, s)
|
||||
|
||||
// index every second long in between.
|
||||
for index0 < index1 {
|
||||
lTable[hash7(load64(src, index0), lTableBits)] = uint32(index0)
|
||||
lTable[hash7(load64(src, index1), lTableBits)] = uint32(index1)
|
||||
index0 += 2
|
||||
index1 -= 2
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
@ -260,8 +300,9 @@ emitRemainder:
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBetterSnappyGo(dst, src []byte) (d int) {
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
@ -402,21 +443,649 @@ func encodeBlockBetterSnappyGo(dst, src []byte) (d int) {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Index match start+1 (long) and start+2 (short)
|
||||
|
||||
// Index short & long
|
||||
index0 := base + 1
|
||||
// Index match end-2 (long) and end-1 (short)
|
||||
index1 := s - 2
|
||||
|
||||
cv0 := load64(src, index0)
|
||||
cv1 := load64(src, index1)
|
||||
cv = load64(src, s)
|
||||
lTable[hash7(cv0, lTableBits)] = uint32(index0)
|
||||
lTable[hash7(cv0>>8, lTableBits)] = uint32(index0 + 1)
|
||||
lTable[hash7(cv1, lTableBits)] = uint32(index1)
|
||||
lTable[hash7(cv1>>8, lTableBits)] = uint32(index1 + 1)
|
||||
sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
|
||||
sTable[hash4(cv0>>16, sTableBits)] = uint32(index0 + 2)
|
||||
|
||||
lTable[hash7(cv1, lTableBits)] = uint32(index1)
|
||||
sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
|
||||
index0 += 1
|
||||
index1 -= 1
|
||||
cv = load64(src, s)
|
||||
|
||||
// index every second long in between.
|
||||
for index0 < index1 {
|
||||
lTable[hash7(load64(src, index0), lTableBits)] = uint32(index0)
|
||||
lTable[hash7(load64(src, index1), lTableBits)] = uint32(index1)
|
||||
index0 += 2
|
||||
index1 -= 2
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// encodeBlockBetterDict encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBetterDict(dst, src []byte, dict *Dict) (d int) {
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
// Initialize the hash tables.
|
||||
const (
|
||||
// Long hash matches.
|
||||
lTableBits = 17
|
||||
maxLTableSize = 1 << lTableBits
|
||||
|
||||
// Short hash matches.
|
||||
sTableBits = 14
|
||||
maxSTableSize = 1 << sTableBits
|
||||
|
||||
maxAhead = 8 // maximum bytes ahead without checking sLimit
|
||||
|
||||
debug = false
|
||||
)
|
||||
|
||||
sLimit := len(src) - inputMargin
|
||||
if sLimit > MaxDictSrcOffset-maxAhead {
|
||||
sLimit = MaxDictSrcOffset - maxAhead
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
|
||||
dict.initBetter()
|
||||
|
||||
var lTable [maxLTableSize]uint32
|
||||
var sTable [maxSTableSize]uint32
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - len(src)>>5 - 6
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 0
|
||||
cv := load64(src, s)
|
||||
|
||||
// We initialize repeat to 0, so we never match on first attempt
|
||||
repeat := len(dict.dict) - dict.repeat
|
||||
|
||||
// While in dict
|
||||
searchDict:
|
||||
for {
|
||||
candidateL := 0
|
||||
nextS := 0
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS = s + (s-nextEmit)>>7 + 1
|
||||
if nextS > sLimit {
|
||||
break searchDict
|
||||
}
|
||||
hashL := hash7(cv, lTableBits)
|
||||
hashS := hash4(cv, sTableBits)
|
||||
candidateL = int(lTable[hashL])
|
||||
candidateS := int(sTable[hashS])
|
||||
dictL := int(dict.betterTableLong[hashL])
|
||||
dictS := int(dict.betterTableShort[hashS])
|
||||
lTable[hashL] = uint32(s)
|
||||
sTable[hashS] = uint32(s)
|
||||
|
||||
valLong := load64(src, candidateL)
|
||||
valShort := load64(src, candidateS)
|
||||
|
||||
// If long matches at least 8 bytes, use that.
|
||||
if s != 0 {
|
||||
if cv == valLong {
|
||||
goto emitMatch
|
||||
}
|
||||
if cv == valShort {
|
||||
candidateL = candidateS
|
||||
goto emitMatch
|
||||
}
|
||||
}
|
||||
|
||||
// Check dict repeat.
|
||||
if repeat >= s+4 {
|
||||
candidate := len(dict.dict) - repeat + s
|
||||
if candidate > 0 && uint32(cv) == load32(dict.dict, candidate) {
|
||||
// Extend back
|
||||
base := s
|
||||
for i := candidate; base > nextEmit && i > 0 && dict.dict[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
if debug && nextEmit != base {
|
||||
fmt.Println("emitted ", base-nextEmit, "literals")
|
||||
}
|
||||
s += 4
|
||||
candidate += 4
|
||||
for candidate < len(dict.dict)-8 && s <= len(src)-8 {
|
||||
if diff := load64(src, s) ^ load64(dict.dict, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
d += emitRepeat(dst[d:], repeat, s-base)
|
||||
if debug {
|
||||
fmt.Println("emitted dict repeat length", s-base, "offset:", repeat, "s:", s)
|
||||
}
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
break searchDict
|
||||
}
|
||||
cv = load64(src, s)
|
||||
// Index in-between
|
||||
index0 := base + 1
|
||||
index1 := s - 2
|
||||
|
||||
cv = load64(src, s)
|
||||
for index0 < index1 {
|
||||
cv0 := load64(src, index0)
|
||||
cv1 := load64(src, index1)
|
||||
lTable[hash7(cv0, lTableBits)] = uint32(index0)
|
||||
sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
|
||||
|
||||
lTable[hash7(cv1, lTableBits)] = uint32(index1)
|
||||
sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
|
||||
index0 += 2
|
||||
index1 -= 2
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
// Don't try to find match at s==0
|
||||
if s == 0 {
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
continue
|
||||
}
|
||||
|
||||
// Long likely matches 7, so take that.
|
||||
if uint32(cv) == uint32(valLong) {
|
||||
goto emitMatch
|
||||
}
|
||||
|
||||
// Long dict...
|
||||
if uint32(cv) == load32(dict.dict, dictL) {
|
||||
candidateL = dictL
|
||||
goto emitDict
|
||||
}
|
||||
|
||||
// Check our short candidate
|
||||
if uint32(cv) == uint32(valShort) {
|
||||
// Try a long candidate at s+1
|
||||
hashL = hash7(cv>>8, lTableBits)
|
||||
candidateL = int(lTable[hashL])
|
||||
lTable[hashL] = uint32(s + 1)
|
||||
if uint32(cv>>8) == load32(src, candidateL) {
|
||||
s++
|
||||
goto emitMatch
|
||||
}
|
||||
// Use our short candidate.
|
||||
candidateL = candidateS
|
||||
goto emitMatch
|
||||
}
|
||||
if uint32(cv) == load32(dict.dict, dictS) {
|
||||
// Try a long candidate at s+1
|
||||
hashL = hash7(cv>>8, lTableBits)
|
||||
candidateL = int(lTable[hashL])
|
||||
lTable[hashL] = uint32(s + 1)
|
||||
if uint32(cv>>8) == load32(src, candidateL) {
|
||||
s++
|
||||
goto emitMatch
|
||||
}
|
||||
candidateL = dictS
|
||||
goto emitDict
|
||||
}
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
emitDict:
|
||||
{
|
||||
if debug {
|
||||
if load32(dict.dict, candidateL) != load32(src, s) {
|
||||
panic("dict emit mismatch")
|
||||
}
|
||||
}
|
||||
// Extend backwards.
|
||||
// The top bytes will be rechecked to get the full match.
|
||||
for candidateL > 0 && s > nextEmit && dict.dict[candidateL-1] == src[s-1] {
|
||||
candidateL--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
d += emitLiteral(dst[d:], src[nextEmit:s])
|
||||
if debug && nextEmit != s {
|
||||
fmt.Println("emitted ", s-nextEmit, "literals")
|
||||
}
|
||||
{
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
base := s
|
||||
offset := s + (len(dict.dict)) - candidateL
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidateL += 4
|
||||
for s <= len(src)-8 && len(dict.dict)-candidateL >= 8 {
|
||||
if diff := load64(src, s) ^ load64(dict.dict, candidateL); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidateL += 8
|
||||
}
|
||||
|
||||
if repeat == offset {
|
||||
if debug {
|
||||
fmt.Println("emitted dict repeat, length", s-base, "offset:", offset, "s:", s, "dict offset:", candidateL)
|
||||
}
|
||||
d += emitRepeat(dst[d:], offset, s-base)
|
||||
} else {
|
||||
if debug {
|
||||
fmt.Println("emitted dict copy, length", s-base, "offset:", offset, "s:", s, "dict offset:", candidateL)
|
||||
}
|
||||
// Matches longer than 64 are split.
|
||||
if s <= sLimit || s-base < 8 {
|
||||
d += emitCopy(dst[d:], offset, s-base)
|
||||
} else {
|
||||
// Split to ensure we don't start a copy within next block.
|
||||
d += emitCopy(dst[d:], offset, 4)
|
||||
d += emitRepeat(dst[d:], offset, s-base-4)
|
||||
}
|
||||
repeat = offset
|
||||
}
|
||||
if false {
|
||||
// Validate match.
|
||||
if s <= candidateL {
|
||||
panic("s <= candidate")
|
||||
}
|
||||
a := src[base:s]
|
||||
b := dict.dict[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
break searchDict
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
|
||||
// Index short & long
|
||||
index0 := base + 1
|
||||
index1 := s - 2
|
||||
|
||||
cv0 := load64(src, index0)
|
||||
cv1 := load64(src, index1)
|
||||
lTable[hash7(cv0, lTableBits)] = uint32(index0)
|
||||
sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
|
||||
|
||||
lTable[hash7(cv1, lTableBits)] = uint32(index1)
|
||||
sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
|
||||
index0 += 1
|
||||
index1 -= 1
|
||||
cv = load64(src, s)
|
||||
|
||||
// index every second long in between.
|
||||
for index0 < index1 {
|
||||
lTable[hash7(load64(src, index0), lTableBits)] = uint32(index0)
|
||||
lTable[hash7(load64(src, index1), lTableBits)] = uint32(index1)
|
||||
index0 += 2
|
||||
index1 -= 2
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
emitMatch:
|
||||
|
||||
// Extend backwards
|
||||
for candidateL > 0 && s > nextEmit && src[candidateL-1] == src[s-1] {
|
||||
candidateL--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
base := s
|
||||
offset := base - candidateL
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidateL += 4
|
||||
for s < len(src) {
|
||||
if len(src)-s < 8 {
|
||||
if src[s] == src[candidateL] {
|
||||
s++
|
||||
candidateL++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if diff := load64(src, s) ^ load64(src, candidateL); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidateL += 8
|
||||
}
|
||||
|
||||
if offset > 65535 && s-base <= 5 && repeat != offset {
|
||||
// Bail if the match is equal or worse to the encoding.
|
||||
s = nextS + 1
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
if debug && nextEmit != s {
|
||||
fmt.Println("emitted ", s-nextEmit, "literals")
|
||||
}
|
||||
if repeat == offset {
|
||||
if debug {
|
||||
fmt.Println("emitted match repeat, length", s-base, "offset:", offset, "s:", s)
|
||||
}
|
||||
d += emitRepeat(dst[d:], offset, s-base)
|
||||
} else {
|
||||
if debug {
|
||||
fmt.Println("emitted match copy, length", s-base, "offset:", offset, "s:", s)
|
||||
}
|
||||
d += emitCopy(dst[d:], offset, s-base)
|
||||
repeat = offset
|
||||
}
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
|
||||
// Index short & long
|
||||
index0 := base + 1
|
||||
index1 := s - 2
|
||||
|
||||
cv0 := load64(src, index0)
|
||||
cv1 := load64(src, index1)
|
||||
lTable[hash7(cv0, lTableBits)] = uint32(index0)
|
||||
sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
|
||||
|
||||
lTable[hash7(cv1, lTableBits)] = uint32(index1)
|
||||
sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
|
||||
index0 += 1
|
||||
index1 -= 1
|
||||
cv = load64(src, s)
|
||||
|
||||
// index every second long in between.
|
||||
for index0 < index1 {
|
||||
lTable[hash7(load64(src, index0), lTableBits)] = uint32(index0)
|
||||
lTable[hash7(load64(src, index1), lTableBits)] = uint32(index1)
|
||||
index0 += 2
|
||||
index1 -= 2
|
||||
}
|
||||
}
|
||||
|
||||
// Search without dict:
|
||||
if repeat > s {
|
||||
repeat = 0
|
||||
}
|
||||
|
||||
// No more dict
|
||||
sLimit = len(src) - inputMargin
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
cv = load64(src, s)
|
||||
if debug {
|
||||
fmt.Println("now", s, "->", sLimit, "out:", d, "left:", len(src)-s, "nextemit:", nextEmit, "dstLimit:", dstLimit, "s:", s)
|
||||
}
|
||||
for {
|
||||
candidateL := 0
|
||||
nextS := 0
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS = s + (s-nextEmit)>>7 + 1
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hashL := hash7(cv, lTableBits)
|
||||
hashS := hash4(cv, sTableBits)
|
||||
candidateL = int(lTable[hashL])
|
||||
candidateS := int(sTable[hashS])
|
||||
lTable[hashL] = uint32(s)
|
||||
sTable[hashS] = uint32(s)
|
||||
|
||||
valLong := load64(src, candidateL)
|
||||
valShort := load64(src, candidateS)
|
||||
|
||||
// If long matches at least 8 bytes, use that.
|
||||
if cv == valLong {
|
||||
break
|
||||
}
|
||||
if cv == valShort {
|
||||
candidateL = candidateS
|
||||
break
|
||||
}
|
||||
|
||||
// Check repeat at offset checkRep.
|
||||
const checkRep = 1
|
||||
// Minimum length of a repeat. Tested with various values.
|
||||
// While 4-5 offers improvements in some, 6 reduces
|
||||
// regressions significantly.
|
||||
const wantRepeatBytes = 6
|
||||
const repeatMask = ((1 << (wantRepeatBytes * 8)) - 1) << (8 * checkRep)
|
||||
if false && repeat > 0 && cv&repeatMask == load64(src, s-repeat)&repeatMask {
|
||||
base := s + checkRep
|
||||
// Extend back
|
||||
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
|
||||
// Extend forward
|
||||
candidate := s - repeat + wantRepeatBytes + checkRep
|
||||
s += wantRepeatBytes + checkRep
|
||||
for s < len(src) {
|
||||
if len(src)-s < 8 {
|
||||
if src[s] == src[candidate] {
|
||||
s++
|
||||
candidate++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
|
||||
d += emitRepeat(dst[d:], repeat, s-base)
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
// Index in-between
|
||||
index0 := base + 1
|
||||
index1 := s - 2
|
||||
|
||||
cv = load64(src, s)
|
||||
for index0 < index1 {
|
||||
cv0 := load64(src, index0)
|
||||
cv1 := load64(src, index1)
|
||||
lTable[hash7(cv0, lTableBits)] = uint32(index0)
|
||||
sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
|
||||
|
||||
lTable[hash7(cv1, lTableBits)] = uint32(index1)
|
||||
sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
|
||||
index0 += 2
|
||||
index1 -= 2
|
||||
}
|
||||
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
// Long likely matches 7, so take that.
|
||||
if uint32(cv) == uint32(valLong) {
|
||||
break
|
||||
}
|
||||
|
||||
// Check our short candidate
|
||||
if uint32(cv) == uint32(valShort) {
|
||||
// Try a long candidate at s+1
|
||||
hashL = hash7(cv>>8, lTableBits)
|
||||
candidateL = int(lTable[hashL])
|
||||
lTable[hashL] = uint32(s + 1)
|
||||
if uint32(cv>>8) == load32(src, candidateL) {
|
||||
s++
|
||||
break
|
||||
}
|
||||
// Use our short candidate.
|
||||
candidateL = candidateS
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards
|
||||
for candidateL > 0 && s > nextEmit && src[candidateL-1] == src[s-1] {
|
||||
candidateL--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
base := s
|
||||
offset := base - candidateL
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidateL += 4
|
||||
for s < len(src) {
|
||||
if len(src)-s < 8 {
|
||||
if src[s] == src[candidateL] {
|
||||
s++
|
||||
candidateL++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if diff := load64(src, s) ^ load64(src, candidateL); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidateL += 8
|
||||
}
|
||||
|
||||
if offset > 65535 && s-base <= 5 && repeat != offset {
|
||||
// Bail if the match is equal or worse to the encoding.
|
||||
s = nextS + 1
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
if repeat == offset {
|
||||
d += emitRepeat(dst[d:], offset, s-base)
|
||||
} else {
|
||||
d += emitCopy(dst[d:], offset, s-base)
|
||||
repeat = offset
|
||||
}
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
|
||||
// Index short & long
|
||||
index0 := base + 1
|
||||
index1 := s - 2
|
||||
|
||||
cv0 := load64(src, index0)
|
||||
cv1 := load64(src, index1)
|
||||
lTable[hash7(cv0, lTableBits)] = uint32(index0)
|
||||
sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
|
||||
|
||||
lTable[hash7(cv1, lTableBits)] = uint32(index1)
|
||||
sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
|
||||
index0 += 1
|
||||
index1 -= 1
|
||||
cv = load64(src, s)
|
||||
|
||||
// index every second long in between.
|
||||
for index0 < index1 {
|
||||
lTable[hash7(load64(src, index0), lTableBits)] = uint32(index0)
|
||||
lTable[hash7(load64(src, index1), lTableBits)] = uint32(index1)
|
||||
index0 += 2
|
||||
index1 -= 2
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
|
||||
414
vendor/github.com/klauspost/compress/s2/encode_go.go
generated
vendored
414
vendor/github.com/klauspost/compress/s2/encode_go.go
generated
vendored
@ -4,14 +4,18 @@
|
||||
package s2
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
const hasAmd64Asm = false
|
||||
|
||||
// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src))
|
||||
func encodeBlock(dst, src []byte) (d int) {
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
@ -25,6 +29,7 @@ func encodeBlock(dst, src []byte) (d int) {
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src))
|
||||
func encodeBlockBetter(dst, src []byte) (d int) {
|
||||
return encodeBlockBetterGo(dst, src)
|
||||
@ -35,6 +40,7 @@ func encodeBlockBetter(dst, src []byte) (d int) {
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src))
|
||||
func encodeBlockBetterSnappy(dst, src []byte) (d int) {
|
||||
return encodeBlockBetterSnappyGo(dst, src)
|
||||
@ -45,6 +51,7 @@ func encodeBlockBetterSnappy(dst, src []byte) (d int) {
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src))
|
||||
func encodeBlockSnappy(dst, src []byte) (d int) {
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
@ -56,6 +63,7 @@ func encodeBlockSnappy(dst, src []byte) (d int) {
|
||||
// emitLiteral writes a literal chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 0 <= len(lit) && len(lit) <= math.MaxUint32
|
||||
func emitLiteral(dst, lit []byte) int {
|
||||
@ -146,6 +154,7 @@ func emitRepeat(dst []byte, offset, length int) int {
|
||||
// emitCopy writes a copy chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
@ -214,6 +223,7 @@ func emitCopy(dst []byte, offset, length int) int {
|
||||
// emitCopyNoRepeat writes a copy chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
@ -273,8 +283,8 @@ func emitCopyNoRepeat(dst []byte, offset, length int) int {
|
||||
// matchLen returns how many bytes match in a and b
|
||||
//
|
||||
// It assumes that:
|
||||
// len(a) <= len(b)
|
||||
//
|
||||
// len(a) <= len(b)
|
||||
func matchLen(a []byte, b []byte) int {
|
||||
b = b[:len(a)]
|
||||
var checked int
|
||||
@ -305,3 +315,405 @@ func matchLen(a []byte, b []byte) int {
|
||||
}
|
||||
return len(a) + checked
|
||||
}
|
||||
|
||||
func calcBlockSize(src []byte) (d int) {
|
||||
// Initialize the hash table.
|
||||
const (
|
||||
tableBits = 13
|
||||
maxTableSize = 1 << tableBits
|
||||
)
|
||||
|
||||
var table [maxTableSize]uint32
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - len(src)>>5 - 5
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
cv := load64(src, s)
|
||||
|
||||
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
|
||||
repeat := 1
|
||||
|
||||
for {
|
||||
candidate := 0
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := s + (s-nextEmit)>>6 + 4
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hash0 := hash6(cv, tableBits)
|
||||
hash1 := hash6(cv>>8, tableBits)
|
||||
candidate = int(table[hash0])
|
||||
candidate2 := int(table[hash1])
|
||||
table[hash0] = uint32(s)
|
||||
table[hash1] = uint32(s + 1)
|
||||
hash2 := hash6(cv>>16, tableBits)
|
||||
|
||||
// Check repeat at offset checkRep.
|
||||
const checkRep = 1
|
||||
if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) {
|
||||
base := s + checkRep
|
||||
// Extend back
|
||||
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteralSize(src[nextEmit:base])
|
||||
|
||||
// Extend forward
|
||||
candidate := s - repeat + 4 + checkRep
|
||||
s += 4 + checkRep
|
||||
for s <= sLimit {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopyNoRepeatSize(repeat, s-base)
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
if uint32(cv) == load32(src, candidate) {
|
||||
break
|
||||
}
|
||||
candidate = int(table[hash2])
|
||||
if uint32(cv>>8) == load32(src, candidate2) {
|
||||
table[hash2] = uint32(s + 2)
|
||||
candidate = candidate2
|
||||
s++
|
||||
break
|
||||
}
|
||||
table[hash2] = uint32(s + 2)
|
||||
if uint32(cv>>16) == load32(src, candidate) {
|
||||
s += 2
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards
|
||||
for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] {
|
||||
candidate--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
d += emitLiteralSize(src[nextEmit:s])
|
||||
|
||||
// Call emitCopy, and then see if another emitCopy could be our next
|
||||
// move. Repeat until we find no match for the input immediately after
|
||||
// what was consumed by the last emitCopy call.
|
||||
//
|
||||
// If we exit this loop normally then we need to call emitLiteral next,
|
||||
// though we don't yet know how big the literal will be. We handle that
|
||||
// by proceeding to the next iteration of the main loop. We also can
|
||||
// exit this loop via goto if we get close to exhausting the input.
|
||||
for {
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
base := s
|
||||
repeat = base - candidate
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidate += 4
|
||||
for s <= len(src)-8 {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopyNoRepeatSize(repeat, s-base)
|
||||
if false {
|
||||
// Validate match.
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Check for an immediate match, otherwise start search at s+1
|
||||
x := load64(src, s-2)
|
||||
m2Hash := hash6(x, tableBits)
|
||||
currHash := hash6(x>>16, tableBits)
|
||||
candidate = int(table[currHash])
|
||||
table[m2Hash] = uint32(s - 2)
|
||||
table[currHash] = uint32(s)
|
||||
if uint32(x>>16) != load32(src, candidate) {
|
||||
cv = load64(src, s+1)
|
||||
s++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteralSize(src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func calcBlockSizeSmall(src []byte) (d int) {
|
||||
// Initialize the hash table.
|
||||
const (
|
||||
tableBits = 9
|
||||
maxTableSize = 1 << tableBits
|
||||
)
|
||||
|
||||
var table [maxTableSize]uint32
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - len(src)>>5 - 5
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
cv := load64(src, s)
|
||||
|
||||
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
|
||||
repeat := 1
|
||||
|
||||
for {
|
||||
candidate := 0
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := s + (s-nextEmit)>>6 + 4
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hash0 := hash6(cv, tableBits)
|
||||
hash1 := hash6(cv>>8, tableBits)
|
||||
candidate = int(table[hash0])
|
||||
candidate2 := int(table[hash1])
|
||||
table[hash0] = uint32(s)
|
||||
table[hash1] = uint32(s + 1)
|
||||
hash2 := hash6(cv>>16, tableBits)
|
||||
|
||||
// Check repeat at offset checkRep.
|
||||
const checkRep = 1
|
||||
if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) {
|
||||
base := s + checkRep
|
||||
// Extend back
|
||||
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteralSize(src[nextEmit:base])
|
||||
|
||||
// Extend forward
|
||||
candidate := s - repeat + 4 + checkRep
|
||||
s += 4 + checkRep
|
||||
for s <= sLimit {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopyNoRepeatSize(repeat, s-base)
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
if uint32(cv) == load32(src, candidate) {
|
||||
break
|
||||
}
|
||||
candidate = int(table[hash2])
|
||||
if uint32(cv>>8) == load32(src, candidate2) {
|
||||
table[hash2] = uint32(s + 2)
|
||||
candidate = candidate2
|
||||
s++
|
||||
break
|
||||
}
|
||||
table[hash2] = uint32(s + 2)
|
||||
if uint32(cv>>16) == load32(src, candidate) {
|
||||
s += 2
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards
|
||||
for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] {
|
||||
candidate--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
d += emitLiteralSize(src[nextEmit:s])
|
||||
|
||||
// Call emitCopy, and then see if another emitCopy could be our next
|
||||
// move. Repeat until we find no match for the input immediately after
|
||||
// what was consumed by the last emitCopy call.
|
||||
//
|
||||
// If we exit this loop normally then we need to call emitLiteral next,
|
||||
// though we don't yet know how big the literal will be. We handle that
|
||||
// by proceeding to the next iteration of the main loop. We also can
|
||||
// exit this loop via goto if we get close to exhausting the input.
|
||||
for {
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
base := s
|
||||
repeat = base - candidate
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidate += 4
|
||||
for s <= len(src)-8 {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopyNoRepeatSize(repeat, s-base)
|
||||
if false {
|
||||
// Validate match.
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Check for an immediate match, otherwise start search at s+1
|
||||
x := load64(src, s-2)
|
||||
m2Hash := hash6(x, tableBits)
|
||||
currHash := hash6(x>>16, tableBits)
|
||||
candidate = int(table[currHash])
|
||||
table[m2Hash] = uint32(s - 2)
|
||||
table[currHash] = uint32(s)
|
||||
if uint32(x>>16) != load32(src, candidate) {
|
||||
cv = load64(src, s+1)
|
||||
s++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteralSize(src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// emitLiteral writes a literal chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 0 <= len(lit) && len(lit) <= math.MaxUint32
|
||||
func emitLiteralSize(lit []byte) int {
|
||||
if len(lit) == 0 {
|
||||
return 0
|
||||
}
|
||||
switch {
|
||||
case len(lit) <= 60:
|
||||
return len(lit) + 1
|
||||
case len(lit) <= 1<<8:
|
||||
return len(lit) + 2
|
||||
case len(lit) <= 1<<16:
|
||||
return len(lit) + 3
|
||||
case len(lit) <= 1<<24:
|
||||
return len(lit) + 4
|
||||
default:
|
||||
return len(lit) + 5
|
||||
}
|
||||
}
|
||||
|
||||
func cvtLZ4BlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) {
|
||||
panic("cvtLZ4BlockAsm should be unreachable")
|
||||
}
|
||||
|
||||
func cvtLZ4BlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) {
|
||||
panic("cvtLZ4BlockSnappyAsm should be unreachable")
|
||||
}
|
||||
|
||||
47
vendor/github.com/klauspost/compress/s2/encodeblock_amd64.go
generated
vendored
47
vendor/github.com/klauspost/compress/s2/encodeblock_amd64.go
generated
vendored
@ -1,7 +1,6 @@
|
||||
// Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_amd64.go -pkg=s2. DO NOT EDIT.
|
||||
|
||||
//go:build !appengine && !noasm && gc && !noasm
|
||||
// +build !appengine,!noasm,gc,!noasm
|
||||
|
||||
package s2
|
||||
|
||||
@ -147,11 +146,26 @@ func encodeSnappyBetterBlockAsm10B(dst []byte, src []byte) int
|
||||
//go:noescape
|
||||
func encodeSnappyBetterBlockAsm8B(dst []byte, src []byte) int
|
||||
|
||||
// calcBlockSize encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4294967295 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func calcBlockSize(src []byte) int
|
||||
|
||||
// calcBlockSizeSmall encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 1024 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func calcBlockSizeSmall(src []byte) int
|
||||
|
||||
// emitLiteral writes a literal chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
// dst is long enough to hold the encoded bytes with margin of 0 bytes
|
||||
// 0 <= len(lit) && len(lit) <= math.MaxUint32
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes with margin of 0 bytes
|
||||
// 0 <= len(lit) && len(lit) <= math.MaxUint32
|
||||
//
|
||||
//go:noescape
|
||||
func emitLiteral(dst []byte, lit []byte) int
|
||||
@ -165,9 +179,10 @@ func emitRepeat(dst []byte, offset int, length int) int
|
||||
// emitCopy writes a copy chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
//
|
||||
//go:noescape
|
||||
func emitCopy(dst []byte, offset int, length int) int
|
||||
@ -175,9 +190,10 @@ func emitCopy(dst []byte, offset int, length int) int
|
||||
// emitCopyNoRepeat writes a copy chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
//
|
||||
//go:noescape
|
||||
func emitCopyNoRepeat(dst []byte, offset int, length int) int
|
||||
@ -185,7 +201,18 @@ func emitCopyNoRepeat(dst []byte, offset int, length int) int
|
||||
// matchLen returns how many bytes match in a and b
|
||||
//
|
||||
// It assumes that:
|
||||
// len(a) <= len(b)
|
||||
//
|
||||
// len(a) <= len(b)
|
||||
//
|
||||
//go:noescape
|
||||
func matchLen(a []byte, b []byte) int
|
||||
|
||||
// cvtLZ4Block converts an LZ4 block to S2
|
||||
//
|
||||
//go:noescape
|
||||
func cvtLZ4BlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
|
||||
|
||||
// cvtLZ4Block converts an LZ4 block to S2
|
||||
//
|
||||
//go:noescape
|
||||
func cvtLZ4BlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
|
||||
|
||||
18233
vendor/github.com/klauspost/compress/s2/encodeblock_amd64.s
generated
vendored
18233
vendor/github.com/klauspost/compress/s2/encodeblock_amd64.s
generated
vendored
File diff suppressed because it is too large
Load Diff
585
vendor/github.com/klauspost/compress/s2/lz4convert.go
generated
vendored
Normal file
585
vendor/github.com/klauspost/compress/s2/lz4convert.go
generated
vendored
Normal file
@ -0,0 +1,585 @@
|
||||
// Copyright (c) 2022 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// LZ4Converter provides conversion from LZ4 blocks as defined here:
|
||||
// https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md
|
||||
type LZ4Converter struct {
|
||||
}
|
||||
|
||||
// ErrDstTooSmall is returned when provided destination is too small.
|
||||
var ErrDstTooSmall = errors.New("s2: destination too small")
|
||||
|
||||
// ConvertBlock will convert an LZ4 block and append it as an S2
|
||||
// block without block length to dst.
|
||||
// The uncompressed size is returned as well.
|
||||
// dst must have capacity to contain the entire compressed block.
|
||||
func (l *LZ4Converter) ConvertBlock(dst, src []byte) ([]byte, int, error) {
|
||||
if len(src) == 0 {
|
||||
return dst, 0, nil
|
||||
}
|
||||
const debug = false
|
||||
const inline = true
|
||||
const lz4MinMatch = 4
|
||||
|
||||
s, d := 0, len(dst)
|
||||
dst = dst[:cap(dst)]
|
||||
if !debug && hasAmd64Asm {
|
||||
res, sz := cvtLZ4BlockAsm(dst[d:], src)
|
||||
if res < 0 {
|
||||
const (
|
||||
errCorrupt = -1
|
||||
errDstTooSmall = -2
|
||||
)
|
||||
switch res {
|
||||
case errCorrupt:
|
||||
return nil, 0, ErrCorrupt
|
||||
case errDstTooSmall:
|
||||
return nil, 0, ErrDstTooSmall
|
||||
default:
|
||||
return nil, 0, fmt.Errorf("unexpected result: %d", res)
|
||||
}
|
||||
}
|
||||
if d+sz > len(dst) {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
return dst[:d+sz], res, nil
|
||||
}
|
||||
|
||||
dLimit := len(dst) - 10
|
||||
var lastOffset uint16
|
||||
var uncompressed int
|
||||
if debug {
|
||||
fmt.Printf("convert block start: len(src): %d, len(dst):%d \n", len(src), len(dst))
|
||||
}
|
||||
|
||||
for {
|
||||
if s >= len(src) {
|
||||
return dst[:d], 0, ErrCorrupt
|
||||
}
|
||||
// Read literal info
|
||||
token := src[s]
|
||||
ll := int(token >> 4)
|
||||
ml := int(lz4MinMatch + (token & 0xf))
|
||||
|
||||
// If upper nibble is 15, literal length is extended
|
||||
if token >= 0xf0 {
|
||||
for {
|
||||
s++
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ll: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return dst[:d], 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
ll += int(val)
|
||||
if val != 255 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// Skip past token
|
||||
if s+ll >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error literals: s+ll (%d+%d) >= len(src) (%d)\n", s, ll, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
s++
|
||||
if ll > 0 {
|
||||
if d+ll > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
if debug {
|
||||
fmt.Printf("emit %d literals\n", ll)
|
||||
}
|
||||
d += emitLiteralGo(dst[d:], src[s:s+ll])
|
||||
s += ll
|
||||
uncompressed += ll
|
||||
}
|
||||
|
||||
// Check if we are done...
|
||||
if s == len(src) && ml == lz4MinMatch {
|
||||
break
|
||||
}
|
||||
// 2 byte offset
|
||||
if s >= len(src)-2 {
|
||||
if debug {
|
||||
fmt.Printf("s (%d) >= len(src)-2 (%d)", s, len(src)-2)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
offset := binary.LittleEndian.Uint16(src[s:])
|
||||
s += 2
|
||||
if offset == 0 {
|
||||
if debug {
|
||||
fmt.Printf("error: offset 0, ml: %d, len(src)-s: %d\n", ml, len(src)-s)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
if int(offset) > uncompressed {
|
||||
if debug {
|
||||
fmt.Printf("error: offset (%d)> uncompressed (%d)\n", offset, uncompressed)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
|
||||
if ml == lz4MinMatch+15 {
|
||||
for {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
s++
|
||||
ml += int(val)
|
||||
if val != 255 {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if offset == lastOffset {
|
||||
if debug {
|
||||
fmt.Printf("emit repeat, length: %d, offset: %d\n", ml, offset)
|
||||
}
|
||||
if !inline {
|
||||
d += emitRepeat16(dst[d:], offset, ml)
|
||||
} else {
|
||||
length := ml
|
||||
dst := dst[d:]
|
||||
for len(dst) > 5 {
|
||||
// Repeat offset, make length cheaper
|
||||
length -= 4
|
||||
if length <= 4 {
|
||||
dst[0] = uint8(length)<<2 | tagCopy1
|
||||
dst[1] = 0
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
if length < 8 && offset < 2048 {
|
||||
// Encode WITH offset
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length)<<2 | tagCopy1
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
if length < (1<<8)+4 {
|
||||
length -= 4
|
||||
dst[2] = uint8(length)
|
||||
dst[1] = 0
|
||||
dst[0] = 5<<2 | tagCopy1
|
||||
d += 3
|
||||
break
|
||||
}
|
||||
if length < (1<<16)+(1<<8) {
|
||||
length -= 1 << 8
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 6<<2 | tagCopy1
|
||||
d += 4
|
||||
break
|
||||
}
|
||||
const maxRepeat = (1 << 24) - 1
|
||||
length -= 1 << 16
|
||||
left := 0
|
||||
if length > maxRepeat {
|
||||
left = length - maxRepeat + 4
|
||||
length = maxRepeat - 4
|
||||
}
|
||||
dst[4] = uint8(length >> 16)
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 7<<2 | tagCopy1
|
||||
if left > 0 {
|
||||
d += 5 + emitRepeat16(dst[5:], offset, left)
|
||||
break
|
||||
}
|
||||
d += 5
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if debug {
|
||||
fmt.Printf("emit copy, length: %d, offset: %d\n", ml, offset)
|
||||
}
|
||||
if !inline {
|
||||
d += emitCopy16(dst[d:], offset, ml)
|
||||
} else {
|
||||
length := ml
|
||||
dst := dst[d:]
|
||||
for len(dst) > 5 {
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
off := 3
|
||||
if offset < 2048 {
|
||||
// emit 8 bytes as tagCopy1, rest as repeats.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(8-4)<<2 | tagCopy1
|
||||
length -= 8
|
||||
off = 2
|
||||
} else {
|
||||
// Emit a length 60 copy, encoded as 3 bytes.
|
||||
// Emit remaining as repeat value (minimum 4 bytes).
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = 59<<2 | tagCopy2
|
||||
length -= 60
|
||||
}
|
||||
// Emit remaining as repeats, at least 4 bytes remain.
|
||||
d += off + emitRepeat16(dst[off:], offset, length)
|
||||
break
|
||||
}
|
||||
if length >= 12 || offset >= 2048 {
|
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(length-1)<<2 | tagCopy2
|
||||
d += 3
|
||||
break
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
}
|
||||
lastOffset = offset
|
||||
}
|
||||
uncompressed += ml
|
||||
if d > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
}
|
||||
|
||||
return dst[:d], uncompressed, nil
|
||||
}
|
||||
|
||||
// ConvertBlockSnappy will convert an LZ4 block and append it
|
||||
// as a Snappy block without block length to dst.
|
||||
// The uncompressed size is returned as well.
|
||||
// dst must have capacity to contain the entire compressed block.
|
||||
func (l *LZ4Converter) ConvertBlockSnappy(dst, src []byte) ([]byte, int, error) {
|
||||
if len(src) == 0 {
|
||||
return dst, 0, nil
|
||||
}
|
||||
const debug = false
|
||||
const lz4MinMatch = 4
|
||||
|
||||
s, d := 0, len(dst)
|
||||
dst = dst[:cap(dst)]
|
||||
// Use assembly when possible
|
||||
if !debug && hasAmd64Asm {
|
||||
res, sz := cvtLZ4BlockSnappyAsm(dst[d:], src)
|
||||
if res < 0 {
|
||||
const (
|
||||
errCorrupt = -1
|
||||
errDstTooSmall = -2
|
||||
)
|
||||
switch res {
|
||||
case errCorrupt:
|
||||
return nil, 0, ErrCorrupt
|
||||
case errDstTooSmall:
|
||||
return nil, 0, ErrDstTooSmall
|
||||
default:
|
||||
return nil, 0, fmt.Errorf("unexpected result: %d", res)
|
||||
}
|
||||
}
|
||||
if d+sz > len(dst) {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
return dst[:d+sz], res, nil
|
||||
}
|
||||
|
||||
dLimit := len(dst) - 10
|
||||
var uncompressed int
|
||||
if debug {
|
||||
fmt.Printf("convert block start: len(src): %d, len(dst):%d \n", len(src), len(dst))
|
||||
}
|
||||
|
||||
for {
|
||||
if s >= len(src) {
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
// Read literal info
|
||||
token := src[s]
|
||||
ll := int(token >> 4)
|
||||
ml := int(lz4MinMatch + (token & 0xf))
|
||||
|
||||
// If upper nibble is 15, literal length is extended
|
||||
if token >= 0xf0 {
|
||||
for {
|
||||
s++
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ll: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
ll += int(val)
|
||||
if val != 255 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// Skip past token
|
||||
if s+ll >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error literals: s+ll (%d+%d) >= len(src) (%d)\n", s, ll, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
s++
|
||||
if ll > 0 {
|
||||
if d+ll > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
if debug {
|
||||
fmt.Printf("emit %d literals\n", ll)
|
||||
}
|
||||
d += emitLiteralGo(dst[d:], src[s:s+ll])
|
||||
s += ll
|
||||
uncompressed += ll
|
||||
}
|
||||
|
||||
// Check if we are done...
|
||||
if s == len(src) && ml == lz4MinMatch {
|
||||
break
|
||||
}
|
||||
// 2 byte offset
|
||||
if s >= len(src)-2 {
|
||||
if debug {
|
||||
fmt.Printf("s (%d) >= len(src)-2 (%d)", s, len(src)-2)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
offset := binary.LittleEndian.Uint16(src[s:])
|
||||
s += 2
|
||||
if offset == 0 {
|
||||
if debug {
|
||||
fmt.Printf("error: offset 0, ml: %d, len(src)-s: %d\n", ml, len(src)-s)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
if int(offset) > uncompressed {
|
||||
if debug {
|
||||
fmt.Printf("error: offset (%d)> uncompressed (%d)\n", offset, uncompressed)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
|
||||
if ml == lz4MinMatch+15 {
|
||||
for {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
s++
|
||||
ml += int(val)
|
||||
if val != 255 {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if debug {
|
||||
fmt.Printf("emit copy, length: %d, offset: %d\n", ml, offset)
|
||||
}
|
||||
length := ml
|
||||
// d += emitCopyNoRepeat(dst[d:], int(offset), ml)
|
||||
for length > 0 {
|
||||
if d >= dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
// Emit a length 64 copy, encoded as 3 bytes.
|
||||
dst[d+2] = uint8(offset >> 8)
|
||||
dst[d+1] = uint8(offset)
|
||||
dst[d+0] = 63<<2 | tagCopy2
|
||||
length -= 64
|
||||
d += 3
|
||||
continue
|
||||
}
|
||||
if length >= 12 || offset >= 2048 || length < 4 {
|
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[d+2] = uint8(offset >> 8)
|
||||
dst[d+1] = uint8(offset)
|
||||
dst[d+0] = uint8(length-1)<<2 | tagCopy2
|
||||
d += 3
|
||||
break
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[d+1] = uint8(offset)
|
||||
dst[d+0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
uncompressed += ml
|
||||
if d > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
}
|
||||
|
||||
return dst[:d], uncompressed, nil
|
||||
}
|
||||
|
||||
// emitRepeat writes a repeat chunk and returns the number of bytes written.
|
||||
// Length must be at least 4 and < 1<<24
|
||||
func emitRepeat16(dst []byte, offset uint16, length int) int {
|
||||
// Repeat offset, make length cheaper
|
||||
length -= 4
|
||||
if length <= 4 {
|
||||
dst[0] = uint8(length)<<2 | tagCopy1
|
||||
dst[1] = 0
|
||||
return 2
|
||||
}
|
||||
if length < 8 && offset < 2048 {
|
||||
// Encode WITH offset
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length)<<2 | tagCopy1
|
||||
return 2
|
||||
}
|
||||
if length < (1<<8)+4 {
|
||||
length -= 4
|
||||
dst[2] = uint8(length)
|
||||
dst[1] = 0
|
||||
dst[0] = 5<<2 | tagCopy1
|
||||
return 3
|
||||
}
|
||||
if length < (1<<16)+(1<<8) {
|
||||
length -= 1 << 8
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 6<<2 | tagCopy1
|
||||
return 4
|
||||
}
|
||||
const maxRepeat = (1 << 24) - 1
|
||||
length -= 1 << 16
|
||||
left := 0
|
||||
if length > maxRepeat {
|
||||
left = length - maxRepeat + 4
|
||||
length = maxRepeat - 4
|
||||
}
|
||||
dst[4] = uint8(length >> 16)
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 7<<2 | tagCopy1
|
||||
if left > 0 {
|
||||
return 5 + emitRepeat16(dst[5:], offset, left)
|
||||
}
|
||||
return 5
|
||||
}
|
||||
|
||||
// emitCopy writes a copy chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= offset && offset <= math.MaxUint16
|
||||
// 4 <= length && length <= math.MaxUint32
|
||||
func emitCopy16(dst []byte, offset uint16, length int) int {
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
off := 3
|
||||
if offset < 2048 {
|
||||
// emit 8 bytes as tagCopy1, rest as repeats.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(8-4)<<2 | tagCopy1
|
||||
length -= 8
|
||||
off = 2
|
||||
} else {
|
||||
// Emit a length 60 copy, encoded as 3 bytes.
|
||||
// Emit remaining as repeat value (minimum 4 bytes).
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = 59<<2 | tagCopy2
|
||||
length -= 60
|
||||
}
|
||||
// Emit remaining as repeats, at least 4 bytes remain.
|
||||
return off + emitRepeat16(dst[off:], offset, length)
|
||||
}
|
||||
if length >= 12 || offset >= 2048 {
|
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(length-1)<<2 | tagCopy2
|
||||
return 3
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1
|
||||
return 2
|
||||
}
|
||||
|
||||
// emitLiteral writes a literal chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 0 <= len(lit) && len(lit) <= math.MaxUint32
|
||||
func emitLiteralGo(dst, lit []byte) int {
|
||||
if len(lit) == 0 {
|
||||
return 0
|
||||
}
|
||||
i, n := 0, uint(len(lit)-1)
|
||||
switch {
|
||||
case n < 60:
|
||||
dst[0] = uint8(n)<<2 | tagLiteral
|
||||
i = 1
|
||||
case n < 1<<8:
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 60<<2 | tagLiteral
|
||||
i = 2
|
||||
case n < 1<<16:
|
||||
dst[2] = uint8(n >> 8)
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 61<<2 | tagLiteral
|
||||
i = 3
|
||||
case n < 1<<24:
|
||||
dst[3] = uint8(n >> 16)
|
||||
dst[2] = uint8(n >> 8)
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 62<<2 | tagLiteral
|
||||
i = 4
|
||||
default:
|
||||
dst[4] = uint8(n >> 24)
|
||||
dst[3] = uint8(n >> 16)
|
||||
dst[2] = uint8(n >> 8)
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 63<<2 | tagLiteral
|
||||
i = 5
|
||||
}
|
||||
return i + copy(dst[i:], lit)
|
||||
}
|
||||
2
vendor/github.com/klauspost/compress/zstd/README.md
generated
vendored
2
vendor/github.com/klauspost/compress/zstd/README.md
generated
vendored
@ -12,6 +12,8 @@ The `zstd` package is provided as open source software using a Go standard licen
|
||||
|
||||
Currently the package is heavily optimized for 64 bit processors and will be significantly slower on 32 bit processors.
|
||||
|
||||
For seekable zstd streams, see [this excellent package](https://github.com/SaveTheRbtz/zstd-seekable-format-go).
|
||||
|
||||
## Installation
|
||||
|
||||
Install using `go get -u github.com/klauspost/compress`. The package is located in `github.com/klauspost/compress/zstd`.
|
||||
|
||||
19
vendor/github.com/klauspost/compress/zstd/blockdec.go
generated
vendored
19
vendor/github.com/klauspost/compress/zstd/blockdec.go
generated
vendored
@ -10,7 +10,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
@ -83,8 +82,9 @@ type blockDec struct {
|
||||
|
||||
err error
|
||||
|
||||
// Check against this crc
|
||||
checkCRC []byte
|
||||
// Check against this crc, if hasCRC is true.
|
||||
checkCRC uint32
|
||||
hasCRC bool
|
||||
|
||||
// Frame to use for singlethreaded decoding.
|
||||
// Should not be used by the decoder itself since parent may be another frame.
|
||||
@ -192,16 +192,14 @@ func (b *blockDec) reset(br byteBuffer, windowSize uint64) error {
|
||||
}
|
||||
|
||||
// Read block data.
|
||||
if cap(b.dataStorage) < cSize {
|
||||
if _, ok := br.(*byteBuf); !ok && cap(b.dataStorage) < cSize {
|
||||
// byteBuf doesn't need a destination buffer.
|
||||
if b.lowMem || cSize > maxCompressedBlockSize {
|
||||
b.dataStorage = make([]byte, 0, cSize+compressedBlockOverAlloc)
|
||||
} else {
|
||||
b.dataStorage = make([]byte, 0, maxCompressedBlockSizeAlloc)
|
||||
}
|
||||
}
|
||||
if cap(b.dst) <= maxSize {
|
||||
b.dst = make([]byte, 0, maxSize+1)
|
||||
}
|
||||
b.data, err = br.readBig(cSize, b.dataStorage)
|
||||
if err != nil {
|
||||
if debugDecoder {
|
||||
@ -210,6 +208,9 @@ func (b *blockDec) reset(br byteBuffer, windowSize uint64) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
if cap(b.dst) <= maxSize {
|
||||
b.dst = make([]byte, 0, maxSize+1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -233,7 +234,7 @@ func (b *blockDec) decodeBuf(hist *history) error {
|
||||
if b.lowMem {
|
||||
b.dst = make([]byte, b.RLESize)
|
||||
} else {
|
||||
b.dst = make([]byte, maxBlockSize)
|
||||
b.dst = make([]byte, maxCompressedBlockSize)
|
||||
}
|
||||
}
|
||||
b.dst = b.dst[:b.RLESize]
|
||||
@ -651,7 +652,7 @@ func (b *blockDec) prepareSequences(in []byte, hist *history) (err error) {
|
||||
fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.matchLengths.fse))
|
||||
fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.offsets.fse))
|
||||
buf.Write(in)
|
||||
ioutil.WriteFile(filepath.Join("testdata", "seqs", fn), buf.Bytes(), os.ModePerm)
|
||||
os.WriteFile(filepath.Join("testdata", "seqs", fn), buf.Bytes(), os.ModePerm)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
16
vendor/github.com/klauspost/compress/zstd/bytebuf.go
generated
vendored
16
vendor/github.com/klauspost/compress/zstd/bytebuf.go
generated
vendored
@ -7,7 +7,6 @@ package zstd
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
)
|
||||
|
||||
type byteBuffer interface {
|
||||
@ -23,7 +22,7 @@ type byteBuffer interface {
|
||||
readByte() (byte, error)
|
||||
|
||||
// Skip n bytes.
|
||||
skipN(n int) error
|
||||
skipN(n int64) error
|
||||
}
|
||||
|
||||
// in-memory buffer
|
||||
@ -62,9 +61,12 @@ func (b *byteBuf) readByte() (byte, error) {
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (b *byteBuf) skipN(n int) error {
|
||||
func (b *byteBuf) skipN(n int64) error {
|
||||
bb := *b
|
||||
if len(bb) < n {
|
||||
if n < 0 {
|
||||
return fmt.Errorf("negative skip (%d) requested", n)
|
||||
}
|
||||
if int64(len(bb)) < n {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
*b = bb[n:]
|
||||
@ -120,9 +122,9 @@ func (r *readerWrapper) readByte() (byte, error) {
|
||||
return r.tmp[0], nil
|
||||
}
|
||||
|
||||
func (r *readerWrapper) skipN(n int) error {
|
||||
n2, err := io.CopyN(ioutil.Discard, r.r, int64(n))
|
||||
if n2 != int64(n) {
|
||||
func (r *readerWrapper) skipN(n int64) error {
|
||||
n2, err := io.CopyN(io.Discard, r.r, n)
|
||||
if n2 != n {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
return err
|
||||
|
||||
9
vendor/github.com/klauspost/compress/zstd/decodeheader.go
generated
vendored
9
vendor/github.com/klauspost/compress/zstd/decodeheader.go
generated
vendored
@ -4,7 +4,6 @@
|
||||
package zstd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
@ -102,8 +101,8 @@ func (h *Header) Decode(in []byte) error {
|
||||
}
|
||||
h.HeaderSize += 4
|
||||
b, in := in[:4], in[4:]
|
||||
if !bytes.Equal(b, frameMagic) {
|
||||
if !bytes.Equal(b[1:4], skippableFrameMagic) || b[0]&0xf0 != 0x50 {
|
||||
if string(b) != frameMagic {
|
||||
if string(b[1:4]) != skippableFrameMagic || b[0]&0xf0 != 0x50 {
|
||||
return ErrMagicMismatch
|
||||
}
|
||||
if len(in) < 4 {
|
||||
@ -153,7 +152,7 @@ func (h *Header) Decode(in []byte) error {
|
||||
}
|
||||
b, in = in[:size], in[size:]
|
||||
h.HeaderSize += int(size)
|
||||
switch size {
|
||||
switch len(b) {
|
||||
case 1:
|
||||
h.DictionaryID = uint32(b[0])
|
||||
case 2:
|
||||
@ -183,7 +182,7 @@ func (h *Header) Decode(in []byte) error {
|
||||
}
|
||||
b, in = in[:fcsSize], in[fcsSize:]
|
||||
h.HeaderSize += int(fcsSize)
|
||||
switch fcsSize {
|
||||
switch len(b) {
|
||||
case 1:
|
||||
h.FrameContentSize = uint64(b[0])
|
||||
case 2:
|
||||
|
||||
140
vendor/github.com/klauspost/compress/zstd/decoder.go
generated
vendored
140
vendor/github.com/klauspost/compress/zstd/decoder.go
generated
vendored
@ -5,7 +5,6 @@
|
||||
package zstd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
@ -35,13 +34,13 @@ type Decoder struct {
|
||||
br readerWrapper
|
||||
enabled bool
|
||||
inFrame bool
|
||||
dstBuf []byte
|
||||
}
|
||||
|
||||
frame *frameDec
|
||||
|
||||
// Custom dictionaries.
|
||||
// Always uses copies.
|
||||
dicts map[uint32]dict
|
||||
dicts map[uint32]*dict
|
||||
|
||||
// streamWg is the waitgroup for all streams
|
||||
streamWg sync.WaitGroup
|
||||
@ -103,7 +102,7 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) {
|
||||
}
|
||||
|
||||
// Transfer option dicts.
|
||||
d.dicts = make(map[uint32]dict, len(d.o.dicts))
|
||||
d.dicts = make(map[uint32]*dict, len(d.o.dicts))
|
||||
for _, dc := range d.o.dicts {
|
||||
d.dicts[dc.id] = dc
|
||||
}
|
||||
@ -187,21 +186,23 @@ func (d *Decoder) Reset(r io.Reader) error {
|
||||
}
|
||||
|
||||
// If bytes buffer and < 5MB, do sync decoding anyway.
|
||||
if bb, ok := r.(byter); ok && bb.Len() < 5<<20 {
|
||||
if bb, ok := r.(byter); ok && bb.Len() < d.o.decodeBufsBelow && !d.o.limitToCap {
|
||||
bb2 := bb
|
||||
if debugDecoder {
|
||||
println("*bytes.Buffer detected, doing sync decode, len:", bb.Len())
|
||||
}
|
||||
b := bb2.Bytes()
|
||||
var dst []byte
|
||||
if cap(d.current.b) > 0 {
|
||||
dst = d.current.b
|
||||
if cap(d.syncStream.dstBuf) > 0 {
|
||||
dst = d.syncStream.dstBuf[:0]
|
||||
}
|
||||
|
||||
dst, err := d.DecodeAll(b, dst[:0])
|
||||
dst, err := d.DecodeAll(b, dst)
|
||||
if err == nil {
|
||||
err = io.EOF
|
||||
}
|
||||
// Save output buffer
|
||||
d.syncStream.dstBuf = dst
|
||||
d.current.b = dst
|
||||
d.current.err = err
|
||||
d.current.flushed = true
|
||||
@ -216,6 +217,7 @@ func (d *Decoder) Reset(r io.Reader) error {
|
||||
d.current.err = nil
|
||||
d.current.flushed = false
|
||||
d.current.d = nil
|
||||
d.syncStream.dstBuf = nil
|
||||
|
||||
// Ensure no-one else is still running...
|
||||
d.streamWg.Wait()
|
||||
@ -312,6 +314,7 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
|
||||
// Grab a block decoder and frame decoder.
|
||||
block := <-d.decoders
|
||||
frame := block.localFrame
|
||||
initialSize := len(dst)
|
||||
defer func() {
|
||||
if debugDecoder {
|
||||
printf("re-adding decoder: %p", block)
|
||||
@ -337,21 +340,26 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
|
||||
}
|
||||
return dst, err
|
||||
}
|
||||
if frame.DictionaryID != nil {
|
||||
dict, ok := d.dicts[*frame.DictionaryID]
|
||||
if !ok {
|
||||
return nil, ErrUnknownDictionary
|
||||
}
|
||||
if debugDecoder {
|
||||
println("setting dict", frame.DictionaryID)
|
||||
}
|
||||
frame.history.setDict(&dict)
|
||||
if err = d.setDict(frame); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if frame.WindowSize > d.o.maxWindowSize {
|
||||
if debugDecoder {
|
||||
println("window size exceeded:", frame.WindowSize, ">", d.o.maxWindowSize)
|
||||
}
|
||||
return dst, ErrWindowSizeExceeded
|
||||
}
|
||||
if frame.FrameContentSize != fcsUnknown {
|
||||
if frame.FrameContentSize > d.o.maxDecodedSize-uint64(len(dst)) {
|
||||
if frame.FrameContentSize > d.o.maxDecodedSize-uint64(len(dst)-initialSize) {
|
||||
if debugDecoder {
|
||||
println("decoder size exceeded; fcs:", frame.FrameContentSize, "> mcs:", d.o.maxDecodedSize-uint64(len(dst)-initialSize), "len:", len(dst))
|
||||
}
|
||||
return dst, ErrDecoderSizeExceeded
|
||||
}
|
||||
if d.o.limitToCap && frame.FrameContentSize > uint64(cap(dst)-len(dst)) {
|
||||
if debugDecoder {
|
||||
println("decoder size exceeded; fcs:", frame.FrameContentSize, "> (cap-len)", cap(dst)-len(dst))
|
||||
}
|
||||
return dst, ErrDecoderSizeExceeded
|
||||
}
|
||||
if cap(dst)-len(dst) < int(frame.FrameContentSize) {
|
||||
@ -361,7 +369,7 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
|
||||
}
|
||||
}
|
||||
|
||||
if cap(dst) == 0 {
|
||||
if cap(dst) == 0 && !d.o.limitToCap {
|
||||
// Allocate len(input) * 2 by default if nothing is provided
|
||||
// and we didn't get frame content size.
|
||||
size := len(input) * 2
|
||||
@ -379,6 +387,9 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
|
||||
if err != nil {
|
||||
return dst, err
|
||||
}
|
||||
if uint64(len(dst)-initialSize) > d.o.maxDecodedSize {
|
||||
return dst, ErrDecoderSizeExceeded
|
||||
}
|
||||
if len(frame.bBuf) == 0 {
|
||||
if debugDecoder {
|
||||
println("frame dbuf empty")
|
||||
@ -439,7 +450,11 @@ func (d *Decoder) nextBlock(blocking bool) (ok bool) {
|
||||
println("got", len(d.current.b), "bytes, error:", d.current.err, "data crc:", tmp)
|
||||
}
|
||||
|
||||
if !d.o.ignoreChecksum && len(next.b) > 0 {
|
||||
if d.o.ignoreChecksum {
|
||||
return true
|
||||
}
|
||||
|
||||
if len(next.b) > 0 {
|
||||
n, err := d.current.crc.Write(next.b)
|
||||
if err == nil {
|
||||
if n != len(next.b) {
|
||||
@ -447,18 +462,16 @@ func (d *Decoder) nextBlock(blocking bool) (ok bool) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if next.err == nil && next.d != nil && len(next.d.checkCRC) != 0 {
|
||||
got := d.current.crc.Sum64()
|
||||
var tmp [4]byte
|
||||
binary.LittleEndian.PutUint32(tmp[:], uint32(got))
|
||||
if !d.o.ignoreChecksum && !bytes.Equal(tmp[:], next.d.checkCRC) {
|
||||
if next.err == nil && next.d != nil && next.d.hasCRC {
|
||||
got := uint32(d.current.crc.Sum64())
|
||||
if got != next.d.checkCRC {
|
||||
if debugDecoder {
|
||||
println("CRC Check Failed:", tmp[:], " (got) !=", next.d.checkCRC, "(on stream)")
|
||||
printf("CRC Check Failed: %08x (got) != %08x (on stream)\n", got, next.d.checkCRC)
|
||||
}
|
||||
d.current.err = ErrCRCMismatch
|
||||
} else {
|
||||
if debugDecoder {
|
||||
println("CRC ok", tmp[:])
|
||||
printf("CRC ok %08x\n", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -474,18 +487,12 @@ func (d *Decoder) nextBlockSync() (ok bool) {
|
||||
if !d.syncStream.inFrame {
|
||||
d.frame.history.reset()
|
||||
d.current.err = d.frame.reset(&d.syncStream.br)
|
||||
if d.current.err == nil {
|
||||
d.current.err = d.setDict(d.frame)
|
||||
}
|
||||
if d.current.err != nil {
|
||||
return false
|
||||
}
|
||||
if d.frame.DictionaryID != nil {
|
||||
dict, ok := d.dicts[*d.frame.DictionaryID]
|
||||
if !ok {
|
||||
d.current.err = ErrUnknownDictionary
|
||||
return false
|
||||
} else {
|
||||
d.frame.history.setDict(&dict)
|
||||
}
|
||||
}
|
||||
if d.frame.WindowSize > d.o.maxDecodedSize || d.frame.WindowSize > d.o.maxWindowSize {
|
||||
d.current.err = ErrDecoderSizeExceeded
|
||||
return false
|
||||
@ -664,6 +671,7 @@ func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output ch
|
||||
if debugDecoder {
|
||||
println("Async 1: new history, recent:", block.async.newHist.recentOffsets)
|
||||
}
|
||||
hist.reset()
|
||||
hist.decoders = block.async.newHist.decoders
|
||||
hist.recentOffsets = block.async.newHist.recentOffsets
|
||||
hist.windowSize = block.async.newHist.windowSize
|
||||
@ -695,6 +703,7 @@ func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output ch
|
||||
seqExecute <- block
|
||||
}
|
||||
close(seqExecute)
|
||||
hist.reset()
|
||||
}()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
@ -718,6 +727,7 @@ func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output ch
|
||||
if debugDecoder {
|
||||
println("Async 2: new history")
|
||||
}
|
||||
hist.reset()
|
||||
hist.windowSize = block.async.newHist.windowSize
|
||||
hist.allocFrameBuffer = block.async.newHist.allocFrameBuffer
|
||||
if block.async.newHist.dict != nil {
|
||||
@ -747,7 +757,7 @@ func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output ch
|
||||
if block.lowMem {
|
||||
block.dst = make([]byte, block.RLESize)
|
||||
} else {
|
||||
block.dst = make([]byte, maxBlockSize)
|
||||
block.dst = make([]byte, maxCompressedBlockSize)
|
||||
}
|
||||
}
|
||||
block.dst = block.dst[:block.RLESize]
|
||||
@ -799,13 +809,14 @@ func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output ch
|
||||
if debugDecoder {
|
||||
println("decoder goroutines finished")
|
||||
}
|
||||
hist.reset()
|
||||
}()
|
||||
|
||||
var hist history
|
||||
decodeStream:
|
||||
for {
|
||||
var hist history
|
||||
var hasErr bool
|
||||
|
||||
hist.reset()
|
||||
decodeBlock := func(block *blockDec) {
|
||||
if hasErr {
|
||||
if block != nil {
|
||||
@ -840,15 +851,14 @@ decodeStream:
|
||||
if debugDecoder && err != nil {
|
||||
println("Frame decoder returned", err)
|
||||
}
|
||||
if err == nil && frame.DictionaryID != nil {
|
||||
dict, ok := d.dicts[*frame.DictionaryID]
|
||||
if !ok {
|
||||
err = ErrUnknownDictionary
|
||||
} else {
|
||||
frame.history.setDict(&dict)
|
||||
}
|
||||
if err == nil {
|
||||
err = d.setDict(frame)
|
||||
}
|
||||
if err == nil && d.frame.WindowSize > d.o.maxWindowSize {
|
||||
if debugDecoder {
|
||||
println("decoder size exceeded, fws:", d.frame.WindowSize, "> mws:", d.o.maxWindowSize)
|
||||
}
|
||||
|
||||
err = ErrDecoderSizeExceeded
|
||||
}
|
||||
if err != nil {
|
||||
@ -890,18 +900,22 @@ decodeStream:
|
||||
println("next block returned error:", err)
|
||||
}
|
||||
dec.err = err
|
||||
dec.checkCRC = nil
|
||||
dec.hasCRC = false
|
||||
if dec.Last && frame.HasCheckSum && err == nil {
|
||||
crc, err := frame.rawInput.readSmall(4)
|
||||
if err != nil {
|
||||
if len(crc) < 4 {
|
||||
if err == nil {
|
||||
err = io.ErrUnexpectedEOF
|
||||
|
||||
}
|
||||
println("CRC missing?", err)
|
||||
dec.err = err
|
||||
}
|
||||
var tmp [4]byte
|
||||
copy(tmp[:], crc)
|
||||
dec.checkCRC = tmp[:]
|
||||
if debugDecoder {
|
||||
println("found crc to check:", dec.checkCRC)
|
||||
} else {
|
||||
dec.checkCRC = binary.LittleEndian.Uint32(crc)
|
||||
dec.hasCRC = true
|
||||
if debugDecoder {
|
||||
printf("found crc to check: %08x\n", dec.checkCRC)
|
||||
}
|
||||
}
|
||||
}
|
||||
err = dec.err
|
||||
@ -917,5 +931,23 @@ decodeStream:
|
||||
}
|
||||
close(seqDecode)
|
||||
wg.Wait()
|
||||
hist.reset()
|
||||
d.frame.history.b = frameHistCache
|
||||
}
|
||||
|
||||
func (d *Decoder) setDict(frame *frameDec) (err error) {
|
||||
dict, ok := d.dicts[frame.DictionaryID]
|
||||
if ok {
|
||||
if debugDecoder {
|
||||
println("setting dict", frame.DictionaryID)
|
||||
}
|
||||
frame.history.setDict(dict)
|
||||
} else if frame.DictionaryID != 0 {
|
||||
// A zero or missing dictionary id is ambiguous:
|
||||
// either dictionary zero, or no dictionary. In particular,
|
||||
// zstd --patch-from uses this id for the source file,
|
||||
// so only return an error if the dictionary id is not zero.
|
||||
err = ErrUnknownDictionary
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
68
vendor/github.com/klauspost/compress/zstd/decoder_options.go
generated
vendored
68
vendor/github.com/klauspost/compress/zstd/decoder_options.go
generated
vendored
@ -6,6 +6,8 @@ package zstd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/bits"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
@ -14,20 +16,23 @@ type DOption func(*decoderOptions) error
|
||||
|
||||
// options retains accumulated state of multiple options.
|
||||
type decoderOptions struct {
|
||||
lowMem bool
|
||||
concurrent int
|
||||
maxDecodedSize uint64
|
||||
maxWindowSize uint64
|
||||
dicts []dict
|
||||
ignoreChecksum bool
|
||||
lowMem bool
|
||||
concurrent int
|
||||
maxDecodedSize uint64
|
||||
maxWindowSize uint64
|
||||
dicts []*dict
|
||||
ignoreChecksum bool
|
||||
limitToCap bool
|
||||
decodeBufsBelow int
|
||||
}
|
||||
|
||||
func (o *decoderOptions) setDefault() {
|
||||
*o = decoderOptions{
|
||||
// use less ram: true for now, but may change.
|
||||
lowMem: true,
|
||||
concurrent: runtime.GOMAXPROCS(0),
|
||||
maxWindowSize: MaxWindowSize,
|
||||
lowMem: true,
|
||||
concurrent: runtime.GOMAXPROCS(0),
|
||||
maxWindowSize: MaxWindowSize,
|
||||
decodeBufsBelow: 128 << 10,
|
||||
}
|
||||
if o.concurrent > 4 {
|
||||
o.concurrent = 4
|
||||
@ -82,7 +87,13 @@ func WithDecoderMaxMemory(n uint64) DOption {
|
||||
}
|
||||
|
||||
// WithDecoderDicts allows to register one or more dictionaries for the decoder.
|
||||
// If several dictionaries with the same ID is provided the last one will be used.
|
||||
//
|
||||
// Each slice in dict must be in the [dictionary format] produced by
|
||||
// "zstd --train" from the Zstandard reference implementation.
|
||||
//
|
||||
// If several dictionaries with the same ID are provided, the last one will be used.
|
||||
//
|
||||
// [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format
|
||||
func WithDecoderDicts(dicts ...[]byte) DOption {
|
||||
return func(o *decoderOptions) error {
|
||||
for _, b := range dicts {
|
||||
@ -90,12 +101,24 @@ func WithDecoderDicts(dicts ...[]byte) DOption {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
o.dicts = append(o.dicts, *d)
|
||||
o.dicts = append(o.dicts, d)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithEncoderDictRaw registers a dictionary that may be used by the decoder.
|
||||
// The slice content can be arbitrary data.
|
||||
func WithDecoderDictRaw(id uint32, content []byte) DOption {
|
||||
return func(o *decoderOptions) error {
|
||||
if bits.UintSize > 32 && uint(len(content)) > dictMaxLength {
|
||||
return fmt.Errorf("dictionary of size %d > 2GiB too large", len(content))
|
||||
}
|
||||
o.dicts = append(o.dicts, &dict{id: id, content: content, offsets: [3]int{1, 4, 8}})
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithDecoderMaxWindow allows to set a maximum window size for decodes.
|
||||
// This allows rejecting packets that will cause big memory usage.
|
||||
// The Decoder will likely allocate more memory based on the WithDecoderLowmem setting.
|
||||
@ -114,6 +137,29 @@ func WithDecoderMaxWindow(size uint64) DOption {
|
||||
}
|
||||
}
|
||||
|
||||
// WithDecodeAllCapLimit will limit DecodeAll to decoding cap(dst)-len(dst) bytes,
|
||||
// or any size set in WithDecoderMaxMemory.
|
||||
// This can be used to limit decoding to a specific maximum output size.
|
||||
// Disabled by default.
|
||||
func WithDecodeAllCapLimit(b bool) DOption {
|
||||
return func(o *decoderOptions) error {
|
||||
o.limitToCap = b
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithDecodeBuffersBelow will fully decode readers that have a
|
||||
// `Bytes() []byte` and `Len() int` interface similar to bytes.Buffer.
|
||||
// This typically uses less allocations but will have the full decompressed object in memory.
|
||||
// Note that DecodeAllCapLimit will disable this, as well as giving a size of 0 or less.
|
||||
// Default is 128KiB.
|
||||
func WithDecodeBuffersBelow(size int) DOption {
|
||||
return func(o *decoderOptions) error {
|
||||
o.decodeBufsBelow = size
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// IgnoreChecksum allows to forcibly ignore checksum checking.
|
||||
func IgnoreChecksum(b bool) DOption {
|
||||
return func(o *decoderOptions) error {
|
||||
|
||||
51
vendor/github.com/klauspost/compress/zstd/dict.go
generated
vendored
51
vendor/github.com/klauspost/compress/zstd/dict.go
generated
vendored
@ -1,7 +1,6 @@
|
||||
package zstd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
@ -20,7 +19,10 @@ type dict struct {
|
||||
content []byte
|
||||
}
|
||||
|
||||
var dictMagic = [4]byte{0x37, 0xa4, 0x30, 0xec}
|
||||
const dictMagic = "\x37\xa4\x30\xec"
|
||||
|
||||
// Maximum dictionary size for the reference implementation (1.5.3) is 2 GiB.
|
||||
const dictMaxLength = 1 << 31
|
||||
|
||||
// ID returns the dictionary id or 0 if d is nil.
|
||||
func (d *dict) ID() uint32 {
|
||||
@ -30,14 +32,38 @@ func (d *dict) ID() uint32 {
|
||||
return d.id
|
||||
}
|
||||
|
||||
// DictContentSize returns the dictionary content size or 0 if d is nil.
|
||||
func (d *dict) DictContentSize() int {
|
||||
// ContentSize returns the dictionary content size or 0 if d is nil.
|
||||
func (d *dict) ContentSize() int {
|
||||
if d == nil {
|
||||
return 0
|
||||
}
|
||||
return len(d.content)
|
||||
}
|
||||
|
||||
// Content returns the dictionary content.
|
||||
func (d *dict) Content() []byte {
|
||||
if d == nil {
|
||||
return nil
|
||||
}
|
||||
return d.content
|
||||
}
|
||||
|
||||
// Offsets returns the initial offsets.
|
||||
func (d *dict) Offsets() [3]int {
|
||||
if d == nil {
|
||||
return [3]int{}
|
||||
}
|
||||
return d.offsets
|
||||
}
|
||||
|
||||
// LitEncoder returns the literal encoder.
|
||||
func (d *dict) LitEncoder() *huff0.Scratch {
|
||||
if d == nil {
|
||||
return nil
|
||||
}
|
||||
return d.litEnc
|
||||
}
|
||||
|
||||
// Load a dictionary as described in
|
||||
// https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format
|
||||
func loadDict(b []byte) (*dict, error) {
|
||||
@ -50,7 +76,7 @@ func loadDict(b []byte) (*dict, error) {
|
||||
ofDec: sequenceDec{fse: &fseDecoder{}},
|
||||
mlDec: sequenceDec{fse: &fseDecoder{}},
|
||||
}
|
||||
if !bytes.Equal(b[:4], dictMagic[:]) {
|
||||
if string(b[:4]) != dictMagic {
|
||||
return nil, ErrMagicMismatch
|
||||
}
|
||||
d.id = binary.LittleEndian.Uint32(b[4:8])
|
||||
@ -62,7 +88,7 @@ func loadDict(b []byte) (*dict, error) {
|
||||
var err error
|
||||
d.litEnc, b, err = huff0.ReadTable(b[8:], nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("loading literal table: %w", err)
|
||||
}
|
||||
d.litEnc.Reuse = huff0.ReusePolicyMust
|
||||
|
||||
@ -120,3 +146,16 @@ func loadDict(b []byte) (*dict, error) {
|
||||
|
||||
return &d, nil
|
||||
}
|
||||
|
||||
// InspectDictionary loads a zstd dictionary and provides functions to inspect the content.
|
||||
func InspectDictionary(b []byte) (interface {
|
||||
ID() uint32
|
||||
ContentSize() int
|
||||
Content() []byte
|
||||
Offsets() [3]int
|
||||
LitEncoder() *huff0.Scratch
|
||||
}, error) {
|
||||
initPredefined()
|
||||
d, err := loadDict(b)
|
||||
return d, err
|
||||
}
|
||||
|
||||
28
vendor/github.com/klauspost/compress/zstd/enc_base.go
generated
vendored
28
vendor/github.com/klauspost/compress/zstd/enc_base.go
generated
vendored
@ -16,6 +16,7 @@ type fastBase struct {
|
||||
cur int32
|
||||
// maximum offset. Should be at least 2x block size.
|
||||
maxMatchOff int32
|
||||
bufferReset int32
|
||||
hist []byte
|
||||
crc *xxhash.Digest
|
||||
tmp [8]byte
|
||||
@ -56,8 +57,8 @@ func (e *fastBase) Block() *blockEnc {
|
||||
}
|
||||
|
||||
func (e *fastBase) addBlock(src []byte) int32 {
|
||||
if debugAsserts && e.cur > bufferReset {
|
||||
panic(fmt.Sprintf("ecur (%d) > buffer reset (%d)", e.cur, bufferReset))
|
||||
if debugAsserts && e.cur > e.bufferReset {
|
||||
panic(fmt.Sprintf("ecur (%d) > buffer reset (%d)", e.cur, e.bufferReset))
|
||||
}
|
||||
// check if we have space already
|
||||
if len(e.hist)+len(src) > cap(e.hist) {
|
||||
@ -126,24 +127,7 @@ func (e *fastBase) matchlen(s, t int32, src []byte) int32 {
|
||||
panic(fmt.Sprintf("len(src)-s (%d) > maxCompressedBlockSize (%d)", len(src)-int(s), maxCompressedBlockSize))
|
||||
}
|
||||
}
|
||||
a := src[s:]
|
||||
b := src[t:]
|
||||
b = b[:len(a)]
|
||||
end := int32((len(a) >> 3) << 3)
|
||||
for i := int32(0); i < end; i += 8 {
|
||||
if diff := load6432(a, i) ^ load6432(b, i); diff != 0 {
|
||||
return i + int32(bits.TrailingZeros64(diff)>>3)
|
||||
}
|
||||
}
|
||||
|
||||
a = a[end:]
|
||||
b = b[end:]
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return int32(i) + end
|
||||
}
|
||||
}
|
||||
return int32(len(a)) + end
|
||||
return int32(matchLen(src[s:], src[t:]))
|
||||
}
|
||||
|
||||
// Reset the encoding table.
|
||||
@ -165,13 +149,13 @@ func (e *fastBase) resetBase(d *dict, singleBlock bool) {
|
||||
if singleBlock {
|
||||
e.lowMem = true
|
||||
}
|
||||
e.ensureHist(d.DictContentSize() + maxCompressedBlockSize)
|
||||
e.ensureHist(d.ContentSize() + maxCompressedBlockSize)
|
||||
e.lowMem = low
|
||||
}
|
||||
|
||||
// We offset current position so everything will be out of reach.
|
||||
// If above reset line, history will be purged.
|
||||
if e.cur < bufferReset {
|
||||
if e.cur < e.bufferReset {
|
||||
e.cur += e.maxMatchOff + int32(len(e.hist))
|
||||
}
|
||||
e.hist = e.hist[:0]
|
||||
|
||||
64
vendor/github.com/klauspost/compress/zstd/enc_best.go
generated
vendored
64
vendor/github.com/klauspost/compress/zstd/enc_best.go
generated
vendored
@ -32,6 +32,7 @@ type match struct {
|
||||
length int32
|
||||
rep int32
|
||||
est int32
|
||||
_ [12]byte // Aligned size to cache line: 4+4+4+4+4 bytes + 12 bytes padding = 32 bytes
|
||||
}
|
||||
|
||||
const highScore = 25000
|
||||
@ -84,14 +85,10 @@ func (e *bestFastEncoder) Encode(blk *blockEnc, src []byte) {
|
||||
)
|
||||
|
||||
// Protect against e.cur wraparound.
|
||||
for e.cur >= bufferReset {
|
||||
for e.cur >= e.bufferReset-int32(len(e.hist)) {
|
||||
if len(e.hist) == 0 {
|
||||
for i := range e.table[:] {
|
||||
e.table[i] = prevEntry{}
|
||||
}
|
||||
for i := range e.longTable[:] {
|
||||
e.longTable[i] = prevEntry{}
|
||||
}
|
||||
e.table = [bestShortTableSize]prevEntry{}
|
||||
e.longTable = [bestLongTableSize]prevEntry{}
|
||||
e.cur = e.maxMatchOff
|
||||
break
|
||||
}
|
||||
@ -192,8 +189,8 @@ encodeLoop:
|
||||
panic("offset0 was 0")
|
||||
}
|
||||
|
||||
bestOf := func(a, b match) match {
|
||||
if a.est+(a.s-b.s)*bitsPerByte>>10 < b.est+(b.s-a.s)*bitsPerByte>>10 {
|
||||
bestOf := func(a, b *match) *match {
|
||||
if a.est-b.est+(a.s-b.s)*bitsPerByte>>10 < 0 {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
@ -219,22 +216,26 @@ encodeLoop:
|
||||
return m
|
||||
}
|
||||
|
||||
best := bestOf(matchAt(candidateL.offset-e.cur, s, uint32(cv), -1), matchAt(candidateL.prev-e.cur, s, uint32(cv), -1))
|
||||
best = bestOf(best, matchAt(candidateS.offset-e.cur, s, uint32(cv), -1))
|
||||
best = bestOf(best, matchAt(candidateS.prev-e.cur, s, uint32(cv), -1))
|
||||
m1 := matchAt(candidateL.offset-e.cur, s, uint32(cv), -1)
|
||||
m2 := matchAt(candidateL.prev-e.cur, s, uint32(cv), -1)
|
||||
m3 := matchAt(candidateS.offset-e.cur, s, uint32(cv), -1)
|
||||
m4 := matchAt(candidateS.prev-e.cur, s, uint32(cv), -1)
|
||||
best := bestOf(bestOf(&m1, &m2), bestOf(&m3, &m4))
|
||||
|
||||
if canRepeat && best.length < goodEnough {
|
||||
cv32 := uint32(cv >> 8)
|
||||
spp := s + 1
|
||||
best = bestOf(best, matchAt(spp-offset1, spp, cv32, 1))
|
||||
best = bestOf(best, matchAt(spp-offset2, spp, cv32, 2))
|
||||
best = bestOf(best, matchAt(spp-offset3, spp, cv32, 3))
|
||||
m1 := matchAt(spp-offset1, spp, cv32, 1)
|
||||
m2 := matchAt(spp-offset2, spp, cv32, 2)
|
||||
m3 := matchAt(spp-offset3, spp, cv32, 3)
|
||||
best = bestOf(bestOf(best, &m1), bestOf(&m2, &m3))
|
||||
if best.length > 0 {
|
||||
cv32 = uint32(cv >> 24)
|
||||
spp += 2
|
||||
best = bestOf(best, matchAt(spp-offset1, spp, cv32, 1))
|
||||
best = bestOf(best, matchAt(spp-offset2, spp, cv32, 2))
|
||||
best = bestOf(best, matchAt(spp-offset3, spp, cv32, 3))
|
||||
m1 := matchAt(spp-offset1, spp, cv32, 1)
|
||||
m2 := matchAt(spp-offset2, spp, cv32, 2)
|
||||
m3 := matchAt(spp-offset3, spp, cv32, 3)
|
||||
best = bestOf(bestOf(best, &m1), bestOf(&m2, &m3))
|
||||
}
|
||||
}
|
||||
// Load next and check...
|
||||
@ -261,26 +262,33 @@ encodeLoop:
|
||||
candidateL2 := e.longTable[hashLen(cv2, bestLongTableBits, bestLongLen)]
|
||||
|
||||
// Short at s+1
|
||||
best = bestOf(best, matchAt(candidateS.offset-e.cur, s, uint32(cv), -1))
|
||||
m1 := matchAt(candidateS.offset-e.cur, s, uint32(cv), -1)
|
||||
// Long at s+1, s+2
|
||||
best = bestOf(best, matchAt(candidateL.offset-e.cur, s, uint32(cv), -1))
|
||||
best = bestOf(best, matchAt(candidateL.prev-e.cur, s, uint32(cv), -1))
|
||||
best = bestOf(best, matchAt(candidateL2.offset-e.cur, s+1, uint32(cv2), -1))
|
||||
best = bestOf(best, matchAt(candidateL2.prev-e.cur, s+1, uint32(cv2), -1))
|
||||
m2 := matchAt(candidateL.offset-e.cur, s, uint32(cv), -1)
|
||||
m3 := matchAt(candidateL.prev-e.cur, s, uint32(cv), -1)
|
||||
m4 := matchAt(candidateL2.offset-e.cur, s+1, uint32(cv2), -1)
|
||||
m5 := matchAt(candidateL2.prev-e.cur, s+1, uint32(cv2), -1)
|
||||
best = bestOf(bestOf(bestOf(best, &m1), &m2), bestOf(bestOf(&m3, &m4), &m5))
|
||||
if false {
|
||||
// Short at s+3.
|
||||
// Too often worse...
|
||||
best = bestOf(best, matchAt(e.table[hashLen(cv2>>8, bestShortTableBits, bestShortLen)].offset-e.cur, s+2, uint32(cv2>>8), -1))
|
||||
m := matchAt(e.table[hashLen(cv2>>8, bestShortTableBits, bestShortLen)].offset-e.cur, s+2, uint32(cv2>>8), -1)
|
||||
best = bestOf(best, &m)
|
||||
}
|
||||
// See if we can find a better match by checking where the current best ends.
|
||||
// Use that offset to see if we can find a better full match.
|
||||
if sAt := best.s + best.length; sAt < sLimit {
|
||||
nextHashL := hashLen(load6432(src, sAt), bestLongTableBits, bestLongLen)
|
||||
candidateEnd := e.longTable[nextHashL]
|
||||
if pos := candidateEnd.offset - e.cur - best.length; pos >= 0 {
|
||||
bestEnd := bestOf(best, matchAt(pos, best.s, load3232(src, best.s), -1))
|
||||
if pos := candidateEnd.prev - e.cur - best.length; pos >= 0 {
|
||||
bestEnd = bestOf(bestEnd, matchAt(pos, best.s, load3232(src, best.s), -1))
|
||||
// Start check at a fixed offset to allow for a few mismatches.
|
||||
// For this compression level 2 yields the best results.
|
||||
const skipBeginning = 2
|
||||
if pos := candidateEnd.offset - e.cur - best.length + skipBeginning; pos >= 0 {
|
||||
m := matchAt(pos, best.s+skipBeginning, load3232(src, best.s+skipBeginning), -1)
|
||||
bestEnd := bestOf(best, &m)
|
||||
if pos := candidateEnd.prev - e.cur - best.length + skipBeginning; pos >= 0 {
|
||||
m := matchAt(pos, best.s+skipBeginning, load3232(src, best.s+skipBeginning), -1)
|
||||
bestEnd = bestOf(bestEnd, &m)
|
||||
}
|
||||
best = bestEnd
|
||||
}
|
||||
|
||||
35
vendor/github.com/klauspost/compress/zstd/enc_better.go
generated
vendored
35
vendor/github.com/klauspost/compress/zstd/enc_better.go
generated
vendored
@ -62,14 +62,10 @@ func (e *betterFastEncoder) Encode(blk *blockEnc, src []byte) {
|
||||
)
|
||||
|
||||
// Protect against e.cur wraparound.
|
||||
for e.cur >= bufferReset {
|
||||
for e.cur >= e.bufferReset-int32(len(e.hist)) {
|
||||
if len(e.hist) == 0 {
|
||||
for i := range e.table[:] {
|
||||
e.table[i] = tableEntry{}
|
||||
}
|
||||
for i := range e.longTable[:] {
|
||||
e.longTable[i] = prevEntry{}
|
||||
}
|
||||
e.table = [betterShortTableSize]tableEntry{}
|
||||
e.longTable = [betterLongTableSize]prevEntry{}
|
||||
e.cur = e.maxMatchOff
|
||||
break
|
||||
}
|
||||
@ -416,15 +412,23 @@ encodeLoop:
|
||||
|
||||
// Try to find a better match by searching for a long match at the end of the current best match
|
||||
if s+matched < sLimit {
|
||||
// Allow some bytes at the beginning to mismatch.
|
||||
// Sweet spot is around 3 bytes, but depends on input.
|
||||
// The skipped bytes are tested in Extend backwards,
|
||||
// and still picked up as part of the match if they do.
|
||||
const skipBeginning = 3
|
||||
|
||||
nextHashL := hashLen(load6432(src, s+matched), betterLongTableBits, betterLongLen)
|
||||
cv := load3232(src, s)
|
||||
s2 := s + skipBeginning
|
||||
cv := load3232(src, s2)
|
||||
candidateL := e.longTable[nextHashL]
|
||||
coffsetL := candidateL.offset - e.cur - matched
|
||||
if coffsetL >= 0 && coffsetL < s && s-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) {
|
||||
coffsetL := candidateL.offset - e.cur - matched + skipBeginning
|
||||
if coffsetL >= 0 && coffsetL < s2 && s2-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) {
|
||||
// Found a long match, at least 4 bytes.
|
||||
matchedNext := e.matchlen(s+4, coffsetL+4, src) + 4
|
||||
matchedNext := e.matchlen(s2+4, coffsetL+4, src) + 4
|
||||
if matchedNext > matched {
|
||||
t = coffsetL
|
||||
s = s2
|
||||
matched = matchedNext
|
||||
if debugMatches {
|
||||
println("long match at end-of-match")
|
||||
@ -434,12 +438,13 @@ encodeLoop:
|
||||
|
||||
// Check prev long...
|
||||
if true {
|
||||
coffsetL = candidateL.prev - e.cur - matched
|
||||
if coffsetL >= 0 && coffsetL < s && s-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) {
|
||||
coffsetL = candidateL.prev - e.cur - matched + skipBeginning
|
||||
if coffsetL >= 0 && coffsetL < s2 && s2-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) {
|
||||
// Found a long match, at least 4 bytes.
|
||||
matchedNext := e.matchlen(s+4, coffsetL+4, src) + 4
|
||||
matchedNext := e.matchlen(s2+4, coffsetL+4, src) + 4
|
||||
if matchedNext > matched {
|
||||
t = coffsetL
|
||||
s = s2
|
||||
matched = matchedNext
|
||||
if debugMatches {
|
||||
println("prev long match at end-of-match")
|
||||
@ -578,7 +583,7 @@ func (e *betterFastEncoderDict) Encode(blk *blockEnc, src []byte) {
|
||||
)
|
||||
|
||||
// Protect against e.cur wraparound.
|
||||
for e.cur >= bufferReset {
|
||||
for e.cur >= e.bufferReset-int32(len(e.hist)) {
|
||||
if len(e.hist) == 0 {
|
||||
for i := range e.table[:] {
|
||||
e.table[i] = tableEntry{}
|
||||
|
||||
23
vendor/github.com/klauspost/compress/zstd/enc_dfast.go
generated
vendored
23
vendor/github.com/klauspost/compress/zstd/enc_dfast.go
generated
vendored
@ -44,14 +44,10 @@ func (e *doubleFastEncoder) Encode(blk *blockEnc, src []byte) {
|
||||
)
|
||||
|
||||
// Protect against e.cur wraparound.
|
||||
for e.cur >= bufferReset {
|
||||
for e.cur >= e.bufferReset-int32(len(e.hist)) {
|
||||
if len(e.hist) == 0 {
|
||||
for i := range e.table[:] {
|
||||
e.table[i] = tableEntry{}
|
||||
}
|
||||
for i := range e.longTable[:] {
|
||||
e.longTable[i] = tableEntry{}
|
||||
}
|
||||
e.table = [dFastShortTableSize]tableEntry{}
|
||||
e.longTable = [dFastLongTableSize]tableEntry{}
|
||||
e.cur = e.maxMatchOff
|
||||
break
|
||||
}
|
||||
@ -388,7 +384,7 @@ func (e *doubleFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) {
|
||||
)
|
||||
|
||||
// Protect against e.cur wraparound.
|
||||
if e.cur >= bufferReset {
|
||||
if e.cur >= e.bufferReset {
|
||||
for i := range e.table[:] {
|
||||
e.table[i] = tableEntry{}
|
||||
}
|
||||
@ -685,7 +681,7 @@ encodeLoop:
|
||||
}
|
||||
|
||||
// We do not store history, so we must offset e.cur to avoid false matches for next user.
|
||||
if e.cur < bufferReset {
|
||||
if e.cur < e.bufferReset {
|
||||
e.cur += int32(len(src))
|
||||
}
|
||||
}
|
||||
@ -700,7 +696,7 @@ func (e *doubleFastEncoderDict) Encode(blk *blockEnc, src []byte) {
|
||||
)
|
||||
|
||||
// Protect against e.cur wraparound.
|
||||
for e.cur >= bufferReset {
|
||||
for e.cur >= e.bufferReset-int32(len(e.hist)) {
|
||||
if len(e.hist) == 0 {
|
||||
for i := range e.table[:] {
|
||||
e.table[i] = tableEntry{}
|
||||
@ -1103,7 +1099,8 @@ func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) {
|
||||
}
|
||||
|
||||
if allDirty || dirtyShardCnt > dLongTableShardCnt/2 {
|
||||
copy(e.longTable[:], e.dictLongTable)
|
||||
//copy(e.longTable[:], e.dictLongTable)
|
||||
e.longTable = *(*[dFastLongTableSize]tableEntry)(e.dictLongTable)
|
||||
for i := range e.longTableShardDirty {
|
||||
e.longTableShardDirty[i] = false
|
||||
}
|
||||
@ -1114,7 +1111,9 @@ func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) {
|
||||
continue
|
||||
}
|
||||
|
||||
copy(e.longTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize], e.dictLongTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize])
|
||||
// copy(e.longTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize], e.dictLongTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize])
|
||||
*(*[dLongTableShardSize]tableEntry)(e.longTable[i*dLongTableShardSize:]) = *(*[dLongTableShardSize]tableEntry)(e.dictLongTable[i*dLongTableShardSize:])
|
||||
|
||||
e.longTableShardDirty[i] = false
|
||||
}
|
||||
}
|
||||
|
||||
20
vendor/github.com/klauspost/compress/zstd/enc_fast.go
generated
vendored
20
vendor/github.com/klauspost/compress/zstd/enc_fast.go
generated
vendored
@ -43,7 +43,7 @@ func (e *fastEncoder) Encode(blk *blockEnc, src []byte) {
|
||||
)
|
||||
|
||||
// Protect against e.cur wraparound.
|
||||
for e.cur >= bufferReset {
|
||||
for e.cur >= e.bufferReset-int32(len(e.hist)) {
|
||||
if len(e.hist) == 0 {
|
||||
for i := range e.table[:] {
|
||||
e.table[i] = tableEntry{}
|
||||
@ -304,13 +304,13 @@ func (e *fastEncoder) EncodeNoHist(blk *blockEnc, src []byte) {
|
||||
minNonLiteralBlockSize = 1 + 1 + inputMargin
|
||||
)
|
||||
if debugEncoder {
|
||||
if len(src) > maxBlockSize {
|
||||
if len(src) > maxCompressedBlockSize {
|
||||
panic("src too big")
|
||||
}
|
||||
}
|
||||
|
||||
// Protect against e.cur wraparound.
|
||||
if e.cur >= bufferReset {
|
||||
if e.cur >= e.bufferReset {
|
||||
for i := range e.table[:] {
|
||||
e.table[i] = tableEntry{}
|
||||
}
|
||||
@ -538,7 +538,7 @@ encodeLoop:
|
||||
println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits)
|
||||
}
|
||||
// We do not store history, so we must offset e.cur to avoid false matches for next user.
|
||||
if e.cur < bufferReset {
|
||||
if e.cur < e.bufferReset {
|
||||
e.cur += int32(len(src))
|
||||
}
|
||||
}
|
||||
@ -555,11 +555,9 @@ func (e *fastEncoderDict) Encode(blk *blockEnc, src []byte) {
|
||||
return
|
||||
}
|
||||
// Protect against e.cur wraparound.
|
||||
for e.cur >= bufferReset {
|
||||
for e.cur >= e.bufferReset-int32(len(e.hist)) {
|
||||
if len(e.hist) == 0 {
|
||||
for i := range e.table[:] {
|
||||
e.table[i] = tableEntry{}
|
||||
}
|
||||
e.table = [tableSize]tableEntry{}
|
||||
e.cur = e.maxMatchOff
|
||||
break
|
||||
}
|
||||
@ -871,7 +869,8 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) {
|
||||
const shardCnt = tableShardCnt
|
||||
const shardSize = tableShardSize
|
||||
if e.allDirty || dirtyShardCnt > shardCnt*4/6 {
|
||||
copy(e.table[:], e.dictTable)
|
||||
//copy(e.table[:], e.dictTable)
|
||||
e.table = *(*[tableSize]tableEntry)(e.dictTable)
|
||||
for i := range e.tableShardDirty {
|
||||
e.tableShardDirty[i] = false
|
||||
}
|
||||
@ -883,7 +882,8 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) {
|
||||
continue
|
||||
}
|
||||
|
||||
copy(e.table[i*shardSize:(i+1)*shardSize], e.dictTable[i*shardSize:(i+1)*shardSize])
|
||||
//copy(e.table[i*shardSize:(i+1)*shardSize], e.dictTable[i*shardSize:(i+1)*shardSize])
|
||||
*(*[shardSize]tableEntry)(e.table[i*shardSize:]) = *(*[shardSize]tableEntry)(e.dictTable[i*shardSize:])
|
||||
e.tableShardDirty[i] = false
|
||||
}
|
||||
e.allDirty = false
|
||||
|
||||
39
vendor/github.com/klauspost/compress/zstd/encoder.go
generated
vendored
39
vendor/github.com/klauspost/compress/zstd/encoder.go
generated
vendored
@ -8,6 +8,7 @@ import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
rdebug "runtime/debug"
|
||||
"sync"
|
||||
|
||||
@ -528,8 +529,8 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte {
|
||||
// If a non-single block is needed the encoder will reset again.
|
||||
e.encoders <- enc
|
||||
}()
|
||||
// Use single segments when above minimum window and below 1MB.
|
||||
single := len(src) < 1<<20 && len(src) > MinWindowSize
|
||||
// Use single segments when above minimum window and below window size.
|
||||
single := len(src) <= e.o.windowSize && len(src) > MinWindowSize
|
||||
if e.o.single != nil {
|
||||
single = *e.o.single
|
||||
}
|
||||
@ -639,3 +640,37 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte {
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// MaxEncodedSize returns the expected maximum
|
||||
// size of an encoded block or stream.
|
||||
func (e *Encoder) MaxEncodedSize(size int) int {
|
||||
frameHeader := 4 + 2 // magic + frame header & window descriptor
|
||||
if e.o.dict != nil {
|
||||
frameHeader += 4
|
||||
}
|
||||
// Frame content size:
|
||||
if size < 256 {
|
||||
frameHeader++
|
||||
} else if size < 65536+256 {
|
||||
frameHeader += 2
|
||||
} else if size < math.MaxInt32 {
|
||||
frameHeader += 4
|
||||
} else {
|
||||
frameHeader += 8
|
||||
}
|
||||
// Final crc
|
||||
if e.o.crc {
|
||||
frameHeader += 4
|
||||
}
|
||||
|
||||
// Max overhead is 3 bytes/block.
|
||||
// There cannot be 0 blocks.
|
||||
blocks := (size + e.o.blockSize) / e.o.blockSize
|
||||
|
||||
// Combine, add padding.
|
||||
maxSz := frameHeader + 3*blocks + size
|
||||
if e.o.pad > 1 {
|
||||
maxSz += calcSkippableFrame(int64(maxSz), int64(e.o.pad))
|
||||
}
|
||||
return maxSz
|
||||
}
|
||||
|
||||
38
vendor/github.com/klauspost/compress/zstd/encoder_options.go
generated
vendored
38
vendor/github.com/klauspost/compress/zstd/encoder_options.go
generated
vendored
@ -3,6 +3,8 @@ package zstd
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"math/bits"
|
||||
"runtime"
|
||||
"strings"
|
||||
)
|
||||
@ -47,22 +49,22 @@ func (o encoderOptions) encoder() encoder {
|
||||
switch o.level {
|
||||
case SpeedFastest:
|
||||
if o.dict != nil {
|
||||
return &fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}}
|
||||
return &fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}}
|
||||
}
|
||||
return &fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}
|
||||
return &fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}
|
||||
|
||||
case SpeedDefault:
|
||||
if o.dict != nil {
|
||||
return &doubleFastEncoderDict{fastEncoderDict: fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}}}
|
||||
return &doubleFastEncoderDict{fastEncoderDict: fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}}}
|
||||
}
|
||||
return &doubleFastEncoder{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}}
|
||||
return &doubleFastEncoder{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}}
|
||||
case SpeedBetterCompression:
|
||||
if o.dict != nil {
|
||||
return &betterFastEncoderDict{betterFastEncoder: betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}}
|
||||
return &betterFastEncoderDict{betterFastEncoder: betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}}
|
||||
}
|
||||
return &betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}
|
||||
return &betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}
|
||||
case SpeedBestCompression:
|
||||
return &bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}
|
||||
return &bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}
|
||||
}
|
||||
panic("unknown compression level")
|
||||
}
|
||||
@ -283,7 +285,7 @@ func WithNoEntropyCompression(b bool) EOption {
|
||||
// a decoder is allowed to reject a compressed frame which requests a memory size beyond decoder's authorized range.
|
||||
// For broader compatibility, decoders are recommended to support memory sizes of at least 8 MB.
|
||||
// This is only a recommendation, each decoder is free to support higher or lower limits, depending on local limitations.
|
||||
// If this is not specified, block encodes will automatically choose this based on the input size.
|
||||
// If this is not specified, block encodes will automatically choose this based on the input size and the window size.
|
||||
// This setting has no effect on streamed encodes.
|
||||
func WithSingleSegment(b bool) EOption {
|
||||
return func(o *encoderOptions) error {
|
||||
@ -304,7 +306,13 @@ func WithLowerEncoderMem(b bool) EOption {
|
||||
}
|
||||
|
||||
// WithEncoderDict allows to register a dictionary that will be used for the encode.
|
||||
//
|
||||
// The slice dict must be in the [dictionary format] produced by
|
||||
// "zstd --train" from the Zstandard reference implementation.
|
||||
//
|
||||
// The encoder *may* choose to use no dictionary instead for certain payloads.
|
||||
//
|
||||
// [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format
|
||||
func WithEncoderDict(dict []byte) EOption {
|
||||
return func(o *encoderOptions) error {
|
||||
d, err := loadDict(dict)
|
||||
@ -315,3 +323,17 @@ func WithEncoderDict(dict []byte) EOption {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithEncoderDictRaw registers a dictionary that may be used by the encoder.
|
||||
//
|
||||
// The slice content may contain arbitrary data. It will be used as an initial
|
||||
// history.
|
||||
func WithEncoderDictRaw(id uint32, content []byte) EOption {
|
||||
return func(o *encoderOptions) error {
|
||||
if bits.UintSize > 32 && uint(len(content)) > dictMaxLength {
|
||||
return fmt.Errorf("dictionary of size %d > 2GiB too large", len(content))
|
||||
}
|
||||
o.dict = &dict{id: id, content: content, offsets: [3]int{1, 4, 8}}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
104
vendor/github.com/klauspost/compress/zstd/framedec.go
generated
vendored
104
vendor/github.com/klauspost/compress/zstd/framedec.go
generated
vendored
@ -5,7 +5,7 @@
|
||||
package zstd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
@ -29,7 +29,7 @@ type frameDec struct {
|
||||
|
||||
FrameContentSize uint64
|
||||
|
||||
DictionaryID *uint32
|
||||
DictionaryID uint32
|
||||
HasCheckSum bool
|
||||
SingleSegment bool
|
||||
}
|
||||
@ -43,9 +43,9 @@ const (
|
||||
MaxWindowSize = 1 << 29
|
||||
)
|
||||
|
||||
var (
|
||||
frameMagic = []byte{0x28, 0xb5, 0x2f, 0xfd}
|
||||
skippableFrameMagic = []byte{0x2a, 0x4d, 0x18}
|
||||
const (
|
||||
frameMagic = "\x28\xb5\x2f\xfd"
|
||||
skippableFrameMagic = "\x2a\x4d\x18"
|
||||
)
|
||||
|
||||
func newFrameDec(o decoderOptions) *frameDec {
|
||||
@ -89,9 +89,9 @@ func (d *frameDec) reset(br byteBuffer) error {
|
||||
copy(signature[1:], b)
|
||||
}
|
||||
|
||||
if !bytes.Equal(signature[1:4], skippableFrameMagic) || signature[0]&0xf0 != 0x50 {
|
||||
if string(signature[1:4]) != skippableFrameMagic || signature[0]&0xf0 != 0x50 {
|
||||
if debugDecoder {
|
||||
println("Not skippable", hex.EncodeToString(signature[:]), hex.EncodeToString(skippableFrameMagic))
|
||||
println("Not skippable", hex.EncodeToString(signature[:]), hex.EncodeToString([]byte(skippableFrameMagic)))
|
||||
}
|
||||
// Break if not skippable frame.
|
||||
break
|
||||
@ -106,7 +106,7 @@ func (d *frameDec) reset(br byteBuffer) error {
|
||||
}
|
||||
n := uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24)
|
||||
println("Skipping frame with", n, "bytes.")
|
||||
err = br.skipN(int(n))
|
||||
err = br.skipN(int64(n))
|
||||
if err != nil {
|
||||
if debugDecoder {
|
||||
println("Reading discarded frame", err)
|
||||
@ -114,9 +114,9 @@ func (d *frameDec) reset(br byteBuffer) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !bytes.Equal(signature[:], frameMagic) {
|
||||
if string(signature[:]) != frameMagic {
|
||||
if debugDecoder {
|
||||
println("Got magic numbers: ", signature, "want:", frameMagic)
|
||||
println("Got magic numbers: ", signature, "want:", []byte(frameMagic))
|
||||
}
|
||||
return ErrMagicMismatch
|
||||
}
|
||||
@ -155,7 +155,7 @@ func (d *frameDec) reset(br byteBuffer) error {
|
||||
|
||||
// Read Dictionary_ID
|
||||
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary_id
|
||||
d.DictionaryID = nil
|
||||
d.DictionaryID = 0
|
||||
if size := fhd & 3; size != 0 {
|
||||
if size == 3 {
|
||||
size = 4
|
||||
@ -167,7 +167,7 @@ func (d *frameDec) reset(br byteBuffer) error {
|
||||
return err
|
||||
}
|
||||
var id uint32
|
||||
switch size {
|
||||
switch len(b) {
|
||||
case 1:
|
||||
id = uint32(b[0])
|
||||
case 2:
|
||||
@ -178,11 +178,7 @@ func (d *frameDec) reset(br byteBuffer) error {
|
||||
if debugDecoder {
|
||||
println("Dict size", size, "ID:", id)
|
||||
}
|
||||
if id > 0 {
|
||||
// ID 0 means "sorry, no dictionary anyway".
|
||||
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format
|
||||
d.DictionaryID = &id
|
||||
}
|
||||
d.DictionaryID = id
|
||||
}
|
||||
|
||||
// Read Frame_Content_Size
|
||||
@ -204,7 +200,7 @@ func (d *frameDec) reset(br byteBuffer) error {
|
||||
println("Reading Frame content", err)
|
||||
return err
|
||||
}
|
||||
switch fcsSize {
|
||||
switch len(b) {
|
||||
case 1:
|
||||
d.FrameContentSize = uint64(b[0])
|
||||
case 2:
|
||||
@ -231,20 +227,27 @@ func (d *frameDec) reset(br byteBuffer) error {
|
||||
d.crc.Reset()
|
||||
}
|
||||
|
||||
if d.WindowSize > d.o.maxWindowSize {
|
||||
if debugDecoder {
|
||||
printf("window size %d > max %d\n", d.WindowSize, d.o.maxWindowSize)
|
||||
}
|
||||
return ErrWindowSizeExceeded
|
||||
}
|
||||
|
||||
if d.WindowSize == 0 && d.SingleSegment {
|
||||
// We may not need window in this case.
|
||||
d.WindowSize = d.FrameContentSize
|
||||
if d.WindowSize < MinWindowSize {
|
||||
d.WindowSize = MinWindowSize
|
||||
}
|
||||
if d.WindowSize > d.o.maxDecodedSize {
|
||||
if debugDecoder {
|
||||
printf("window size %d > max %d\n", d.WindowSize, d.o.maxWindowSize)
|
||||
}
|
||||
return ErrDecoderSizeExceeded
|
||||
}
|
||||
}
|
||||
|
||||
if d.WindowSize > uint64(d.o.maxWindowSize) {
|
||||
if debugDecoder {
|
||||
printf("window size %d > max %d\n", d.WindowSize, d.o.maxWindowSize)
|
||||
}
|
||||
return ErrWindowSizeExceeded
|
||||
}
|
||||
// The minimum Window_Size is 1 KB.
|
||||
if d.WindowSize < MinWindowSize {
|
||||
if debugDecoder {
|
||||
@ -254,11 +257,16 @@ func (d *frameDec) reset(br byteBuffer) error {
|
||||
}
|
||||
d.history.windowSize = int(d.WindowSize)
|
||||
if !d.o.lowMem || d.history.windowSize < maxBlockSize {
|
||||
// Alloc 2x window size if not low-mem, or very small window size.
|
||||
// Alloc 2x window size if not low-mem, or window size below 2MB.
|
||||
d.history.allocFrameBuffer = d.history.windowSize * 2
|
||||
} else {
|
||||
// Alloc with one additional block
|
||||
d.history.allocFrameBuffer = d.history.windowSize + maxBlockSize
|
||||
if d.o.lowMem {
|
||||
// Alloc with 1MB extra.
|
||||
d.history.allocFrameBuffer = d.history.windowSize + maxBlockSize/2
|
||||
} else {
|
||||
// Alloc with 2MB extra.
|
||||
d.history.allocFrameBuffer = d.history.windowSize + maxBlockSize
|
||||
}
|
||||
}
|
||||
|
||||
if debugDecoder {
|
||||
@ -293,7 +301,7 @@ func (d *frameDec) checkCRC() error {
|
||||
}
|
||||
|
||||
// We can overwrite upper tmp now
|
||||
want, err := d.rawInput.readSmall(4)
|
||||
buf, err := d.rawInput.readSmall(4)
|
||||
if err != nil {
|
||||
println("CRC missing?", err)
|
||||
return err
|
||||
@ -303,22 +311,17 @@ func (d *frameDec) checkCRC() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
var tmp [4]byte
|
||||
got := d.crc.Sum64()
|
||||
// Flip to match file order.
|
||||
tmp[0] = byte(got >> 0)
|
||||
tmp[1] = byte(got >> 8)
|
||||
tmp[2] = byte(got >> 16)
|
||||
tmp[3] = byte(got >> 24)
|
||||
want := binary.LittleEndian.Uint32(buf[:4])
|
||||
got := uint32(d.crc.Sum64())
|
||||
|
||||
if !bytes.Equal(tmp[:], want) {
|
||||
if got != want {
|
||||
if debugDecoder {
|
||||
println("CRC Check Failed:", tmp[:], "!=", want)
|
||||
printf("CRC check failed: got %08x, want %08x\n", got, want)
|
||||
}
|
||||
return ErrCRCMismatch
|
||||
}
|
||||
if debugDecoder {
|
||||
println("CRC ok", tmp[:])
|
||||
printf("CRC ok %08x\n", got)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -336,7 +339,7 @@ func (d *frameDec) consumeCRC() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// runDecoder will create a sync decoder that will decode a block of data.
|
||||
// runDecoder will run the decoder for the remainder of the frame.
|
||||
func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) {
|
||||
saved := d.history.b
|
||||
|
||||
@ -346,12 +349,23 @@ func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) {
|
||||
// Store input length, so we only check new data.
|
||||
crcStart := len(dst)
|
||||
d.history.decoders.maxSyncLen = 0
|
||||
if d.o.limitToCap {
|
||||
d.history.decoders.maxSyncLen = uint64(cap(dst) - len(dst))
|
||||
}
|
||||
if d.FrameContentSize != fcsUnknown {
|
||||
d.history.decoders.maxSyncLen = d.FrameContentSize + uint64(len(dst))
|
||||
if !d.o.limitToCap || d.FrameContentSize+uint64(len(dst)) < d.history.decoders.maxSyncLen {
|
||||
d.history.decoders.maxSyncLen = d.FrameContentSize + uint64(len(dst))
|
||||
}
|
||||
if d.history.decoders.maxSyncLen > d.o.maxDecodedSize {
|
||||
if debugDecoder {
|
||||
println("maxSyncLen:", d.history.decoders.maxSyncLen, "> maxDecodedSize:", d.o.maxDecodedSize)
|
||||
}
|
||||
return dst, ErrDecoderSizeExceeded
|
||||
}
|
||||
if uint64(cap(dst)) < d.history.decoders.maxSyncLen {
|
||||
if debugDecoder {
|
||||
println("maxSyncLen:", d.history.decoders.maxSyncLen)
|
||||
}
|
||||
if !d.o.limitToCap && uint64(cap(dst)) < d.history.decoders.maxSyncLen {
|
||||
// Alloc for output
|
||||
dst2 := make([]byte, len(dst), d.history.decoders.maxSyncLen+compressedBlockOverAlloc)
|
||||
copy(dst2, dst)
|
||||
@ -371,7 +385,13 @@ func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) {
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if uint64(len(d.history.b)) > d.o.maxDecodedSize {
|
||||
if uint64(len(d.history.b)-crcStart) > d.o.maxDecodedSize {
|
||||
println("runDecoder: maxDecodedSize exceeded", uint64(len(d.history.b)-crcStart), ">", d.o.maxDecodedSize)
|
||||
err = ErrDecoderSizeExceeded
|
||||
break
|
||||
}
|
||||
if d.o.limitToCap && len(d.history.b) > cap(dst) {
|
||||
println("runDecoder: cap exceeded", uint64(len(d.history.b)), ">", cap(dst))
|
||||
err = ErrDecoderSizeExceeded
|
||||
break
|
||||
}
|
||||
|
||||
7
vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go
generated
vendored
7
vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go
generated
vendored
@ -21,7 +21,8 @@ type buildDtableAsmContext struct {
|
||||
|
||||
// buildDtable_asm is an x86 assembly implementation of fseDecoder.buildDtable.
|
||||
// Function returns non-zero exit code on error.
|
||||
// go:noescape
|
||||
//
|
||||
//go:noescape
|
||||
func buildDtable_asm(s *fseDecoder, ctx *buildDtableAsmContext) int
|
||||
|
||||
// please keep in sync with _generate/gen_fse.go
|
||||
@ -34,8 +35,8 @@ const (
|
||||
// buildDtable will build the decoding table.
|
||||
func (s *fseDecoder) buildDtable() error {
|
||||
ctx := buildDtableAsmContext{
|
||||
stateTable: (*uint16)(&s.stateTable[0]),
|
||||
norm: (*int16)(&s.norm[0]),
|
||||
stateTable: &s.stateTable[0],
|
||||
norm: &s.norm[0],
|
||||
dt: (*uint64)(&s.dt[0]),
|
||||
}
|
||||
code := buildDtable_asm(s, &ctx)
|
||||
|
||||
1
vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s
generated
vendored
1
vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s
generated
vendored
@ -1,7 +1,6 @@
|
||||
// Code generated by command: go run gen_fse.go -out ../fse_decoder_amd64.s -pkg=zstd. DO NOT EDIT.
|
||||
|
||||
//go:build !appengine && !noasm && gc && !noasm
|
||||
// +build !appengine,!noasm,gc,!noasm
|
||||
|
||||
// func buildDtable_asm(s *fseDecoder, ctx *buildDtableAsmContext) int
|
||||
TEXT ·buildDtable_asm(SB), $0-24
|
||||
|
||||
25
vendor/github.com/klauspost/compress/zstd/history.go
generated
vendored
25
vendor/github.com/klauspost/compress/zstd/history.go
generated
vendored
@ -37,26 +37,23 @@ func (h *history) reset() {
|
||||
h.ignoreBuffer = 0
|
||||
h.error = false
|
||||
h.recentOffsets = [3]int{1, 4, 8}
|
||||
if f := h.decoders.litLengths.fse; f != nil && !f.preDefined {
|
||||
fseDecoderPool.Put(f)
|
||||
}
|
||||
if f := h.decoders.offsets.fse; f != nil && !f.preDefined {
|
||||
fseDecoderPool.Put(f)
|
||||
}
|
||||
if f := h.decoders.matchLengths.fse; f != nil && !f.preDefined {
|
||||
fseDecoderPool.Put(f)
|
||||
}
|
||||
h.decoders.freeDecoders()
|
||||
h.decoders = sequenceDecs{br: h.decoders.br}
|
||||
if h.huffTree != nil {
|
||||
if h.dict == nil || h.dict.litEnc != h.huffTree {
|
||||
huffDecoderPool.Put(h.huffTree)
|
||||
}
|
||||
}
|
||||
h.freeHuffDecoder()
|
||||
h.huffTree = nil
|
||||
h.dict = nil
|
||||
//printf("history created: %+v (l: %d, c: %d)", *h, len(h.b), cap(h.b))
|
||||
}
|
||||
|
||||
func (h *history) freeHuffDecoder() {
|
||||
if h.huffTree != nil {
|
||||
if h.dict == nil || h.dict.litEnc != h.huffTree {
|
||||
huffDecoderPool.Put(h.huffTree)
|
||||
h.huffTree = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *history) setDict(dict *dict) {
|
||||
if dict == nil {
|
||||
return
|
||||
|
||||
49
vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md
generated
vendored
49
vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md
generated
vendored
@ -2,12 +2,7 @@
|
||||
|
||||
VENDORED: Go to [github.com/cespare/xxhash](https://github.com/cespare/xxhash) for original package.
|
||||
|
||||
|
||||
[](https://godoc.org/github.com/cespare/xxhash)
|
||||
[](https://travis-ci.org/cespare/xxhash)
|
||||
|
||||
xxhash is a Go implementation of the 64-bit
|
||||
[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a
|
||||
xxhash is a Go implementation of the 64-bit [xxHash] algorithm, XXH64. This is a
|
||||
high-quality hashing algorithm that is much faster than anything in the Go
|
||||
standard library.
|
||||
|
||||
@ -28,31 +23,49 @@ func (*Digest) WriteString(string) (int, error)
|
||||
func (*Digest) Sum64() uint64
|
||||
```
|
||||
|
||||
This implementation provides a fast pure-Go implementation and an even faster
|
||||
assembly implementation for amd64.
|
||||
The package is written with optimized pure Go and also contains even faster
|
||||
assembly implementations for amd64 and arm64. If desired, the `purego` build tag
|
||||
opts into using the Go code even on those architectures.
|
||||
|
||||
[xxHash]: http://cyan4973.github.io/xxHash/
|
||||
|
||||
## Compatibility
|
||||
|
||||
This package is in a module and the latest code is in version 2 of the module.
|
||||
You need a version of Go with at least "minimal module compatibility" to use
|
||||
github.com/cespare/xxhash/v2:
|
||||
|
||||
* 1.9.7+ for Go 1.9
|
||||
* 1.10.3+ for Go 1.10
|
||||
* Go 1.11 or later
|
||||
|
||||
I recommend using the latest release of Go.
|
||||
|
||||
## Benchmarks
|
||||
|
||||
Here are some quick benchmarks comparing the pure-Go and assembly
|
||||
implementations of Sum64.
|
||||
|
||||
| input size | purego | asm |
|
||||
| --- | --- | --- |
|
||||
| 5 B | 979.66 MB/s | 1291.17 MB/s |
|
||||
| 100 B | 7475.26 MB/s | 7973.40 MB/s |
|
||||
| 4 KB | 17573.46 MB/s | 17602.65 MB/s |
|
||||
| 10 MB | 17131.46 MB/s | 17142.16 MB/s |
|
||||
| input size | purego | asm |
|
||||
| ---------- | --------- | --------- |
|
||||
| 4 B | 1.3 GB/s | 1.2 GB/s |
|
||||
| 16 B | 2.9 GB/s | 3.5 GB/s |
|
||||
| 100 B | 6.9 GB/s | 8.1 GB/s |
|
||||
| 4 KB | 11.7 GB/s | 16.7 GB/s |
|
||||
| 10 MB | 12.0 GB/s | 17.3 GB/s |
|
||||
|
||||
These numbers were generated on Ubuntu 18.04 with an Intel i7-8700K CPU using
|
||||
the following commands under Go 1.11.2:
|
||||
These numbers were generated on Ubuntu 20.04 with an Intel Xeon Platinum 8252C
|
||||
CPU using the following commands under Go 1.19.2:
|
||||
|
||||
```
|
||||
$ go test -tags purego -benchtime 10s -bench '/xxhash,direct,bytes'
|
||||
$ go test -benchtime 10s -bench '/xxhash,direct,bytes'
|
||||
benchstat <(go test -tags purego -benchtime 500ms -count 15 -bench 'Sum64$')
|
||||
benchstat <(go test -benchtime 500ms -count 15 -bench 'Sum64$')
|
||||
```
|
||||
|
||||
## Projects using this package
|
||||
|
||||
- [InfluxDB](https://github.com/influxdata/influxdb)
|
||||
- [Prometheus](https://github.com/prometheus/prometheus)
|
||||
- [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics)
|
||||
- [FreeCache](https://github.com/coocood/freecache)
|
||||
- [FastCache](https://github.com/VictoriaMetrics/fastcache)
|
||||
|
||||
47
vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go
generated
vendored
47
vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go
generated
vendored
@ -18,19 +18,11 @@ const (
|
||||
prime5 uint64 = 2870177450012600261
|
||||
)
|
||||
|
||||
// NOTE(caleb): I'm using both consts and vars of the primes. Using consts where
|
||||
// possible in the Go code is worth a small (but measurable) performance boost
|
||||
// by avoiding some MOVQs. Vars are needed for the asm and also are useful for
|
||||
// convenience in the Go code in a few places where we need to intentionally
|
||||
// avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the
|
||||
// result overflows a uint64).
|
||||
var (
|
||||
prime1v = prime1
|
||||
prime2v = prime2
|
||||
prime3v = prime3
|
||||
prime4v = prime4
|
||||
prime5v = prime5
|
||||
)
|
||||
// Store the primes in an array as well.
|
||||
//
|
||||
// The consts are used when possible in Go code to avoid MOVs but we need a
|
||||
// contiguous array of the assembly code.
|
||||
var primes = [...]uint64{prime1, prime2, prime3, prime4, prime5}
|
||||
|
||||
// Digest implements hash.Hash64.
|
||||
type Digest struct {
|
||||
@ -52,10 +44,10 @@ func New() *Digest {
|
||||
|
||||
// Reset clears the Digest's state so that it can be reused.
|
||||
func (d *Digest) Reset() {
|
||||
d.v1 = prime1v + prime2
|
||||
d.v1 = primes[0] + prime2
|
||||
d.v2 = prime2
|
||||
d.v3 = 0
|
||||
d.v4 = -prime1v
|
||||
d.v4 = -primes[0]
|
||||
d.total = 0
|
||||
d.n = 0
|
||||
}
|
||||
@ -71,21 +63,23 @@ func (d *Digest) Write(b []byte) (n int, err error) {
|
||||
n = len(b)
|
||||
d.total += uint64(n)
|
||||
|
||||
memleft := d.mem[d.n&(len(d.mem)-1):]
|
||||
|
||||
if d.n+n < 32 {
|
||||
// This new data doesn't even fill the current block.
|
||||
copy(d.mem[d.n:], b)
|
||||
copy(memleft, b)
|
||||
d.n += n
|
||||
return
|
||||
}
|
||||
|
||||
if d.n > 0 {
|
||||
// Finish off the partial block.
|
||||
copy(d.mem[d.n:], b)
|
||||
c := copy(memleft, b)
|
||||
d.v1 = round(d.v1, u64(d.mem[0:8]))
|
||||
d.v2 = round(d.v2, u64(d.mem[8:16]))
|
||||
d.v3 = round(d.v3, u64(d.mem[16:24]))
|
||||
d.v4 = round(d.v4, u64(d.mem[24:32]))
|
||||
b = b[32-d.n:]
|
||||
b = b[c:]
|
||||
d.n = 0
|
||||
}
|
||||
|
||||
@ -135,21 +129,20 @@ func (d *Digest) Sum64() uint64 {
|
||||
|
||||
h += d.total
|
||||
|
||||
i, end := 0, d.n
|
||||
for ; i+8 <= end; i += 8 {
|
||||
k1 := round(0, u64(d.mem[i:i+8]))
|
||||
b := d.mem[:d.n&(len(d.mem)-1)]
|
||||
for ; len(b) >= 8; b = b[8:] {
|
||||
k1 := round(0, u64(b[:8]))
|
||||
h ^= k1
|
||||
h = rol27(h)*prime1 + prime4
|
||||
}
|
||||
if i+4 <= end {
|
||||
h ^= uint64(u32(d.mem[i:i+4])) * prime1
|
||||
if len(b) >= 4 {
|
||||
h ^= uint64(u32(b[:4])) * prime1
|
||||
h = rol23(h)*prime2 + prime3
|
||||
i += 4
|
||||
b = b[4:]
|
||||
}
|
||||
for i < end {
|
||||
h ^= uint64(d.mem[i]) * prime5
|
||||
for ; len(b) > 0; b = b[1:] {
|
||||
h ^= uint64(b[0]) * prime5
|
||||
h = rol11(h) * prime1
|
||||
i++
|
||||
}
|
||||
|
||||
h ^= h >> 33
|
||||
|
||||
308
vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s
generated
vendored
308
vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s
generated
vendored
@ -1,3 +1,4 @@
|
||||
//go:build !appengine && gc && !purego && !noasm
|
||||
// +build !appengine
|
||||
// +build gc
|
||||
// +build !purego
|
||||
@ -5,212 +6,205 @@
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
// Register allocation:
|
||||
// AX h
|
||||
// SI pointer to advance through b
|
||||
// DX n
|
||||
// BX loop end
|
||||
// R8 v1, k1
|
||||
// R9 v2
|
||||
// R10 v3
|
||||
// R11 v4
|
||||
// R12 tmp
|
||||
// R13 prime1v
|
||||
// R14 prime2v
|
||||
// DI prime4v
|
||||
// Registers:
|
||||
#define h AX
|
||||
#define d AX
|
||||
#define p SI // pointer to advance through b
|
||||
#define n DX
|
||||
#define end BX // loop end
|
||||
#define v1 R8
|
||||
#define v2 R9
|
||||
#define v3 R10
|
||||
#define v4 R11
|
||||
#define x R12
|
||||
#define prime1 R13
|
||||
#define prime2 R14
|
||||
#define prime4 DI
|
||||
|
||||
// round reads from and advances the buffer pointer in SI.
|
||||
// It assumes that R13 has prime1v and R14 has prime2v.
|
||||
#define round(r) \
|
||||
MOVQ (SI), R12 \
|
||||
ADDQ $8, SI \
|
||||
IMULQ R14, R12 \
|
||||
ADDQ R12, r \
|
||||
ROLQ $31, r \
|
||||
IMULQ R13, r
|
||||
#define round(acc, x) \
|
||||
IMULQ prime2, x \
|
||||
ADDQ x, acc \
|
||||
ROLQ $31, acc \
|
||||
IMULQ prime1, acc
|
||||
|
||||
// mergeRound applies a merge round on the two registers acc and val.
|
||||
// It assumes that R13 has prime1v, R14 has prime2v, and DI has prime4v.
|
||||
#define mergeRound(acc, val) \
|
||||
IMULQ R14, val \
|
||||
ROLQ $31, val \
|
||||
IMULQ R13, val \
|
||||
XORQ val, acc \
|
||||
IMULQ R13, acc \
|
||||
ADDQ DI, acc
|
||||
// round0 performs the operation x = round(0, x).
|
||||
#define round0(x) \
|
||||
IMULQ prime2, x \
|
||||
ROLQ $31, x \
|
||||
IMULQ prime1, x
|
||||
|
||||
// mergeRound applies a merge round on the two registers acc and x.
|
||||
// It assumes that prime1, prime2, and prime4 have been loaded.
|
||||
#define mergeRound(acc, x) \
|
||||
round0(x) \
|
||||
XORQ x, acc \
|
||||
IMULQ prime1, acc \
|
||||
ADDQ prime4, acc
|
||||
|
||||
// blockLoop processes as many 32-byte blocks as possible,
|
||||
// updating v1, v2, v3, and v4. It assumes that there is at least one block
|
||||
// to process.
|
||||
#define blockLoop() \
|
||||
loop: \
|
||||
MOVQ +0(p), x \
|
||||
round(v1, x) \
|
||||
MOVQ +8(p), x \
|
||||
round(v2, x) \
|
||||
MOVQ +16(p), x \
|
||||
round(v3, x) \
|
||||
MOVQ +24(p), x \
|
||||
round(v4, x) \
|
||||
ADDQ $32, p \
|
||||
CMPQ p, end \
|
||||
JLE loop
|
||||
|
||||
// func Sum64(b []byte) uint64
|
||||
TEXT ·Sum64(SB), NOSPLIT, $0-32
|
||||
TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32
|
||||
// Load fixed primes.
|
||||
MOVQ ·prime1v(SB), R13
|
||||
MOVQ ·prime2v(SB), R14
|
||||
MOVQ ·prime4v(SB), DI
|
||||
MOVQ ·primes+0(SB), prime1
|
||||
MOVQ ·primes+8(SB), prime2
|
||||
MOVQ ·primes+24(SB), prime4
|
||||
|
||||
// Load slice.
|
||||
MOVQ b_base+0(FP), SI
|
||||
MOVQ b_len+8(FP), DX
|
||||
LEAQ (SI)(DX*1), BX
|
||||
MOVQ b_base+0(FP), p
|
||||
MOVQ b_len+8(FP), n
|
||||
LEAQ (p)(n*1), end
|
||||
|
||||
// The first loop limit will be len(b)-32.
|
||||
SUBQ $32, BX
|
||||
SUBQ $32, end
|
||||
|
||||
// Check whether we have at least one block.
|
||||
CMPQ DX, $32
|
||||
CMPQ n, $32
|
||||
JLT noBlocks
|
||||
|
||||
// Set up initial state (v1, v2, v3, v4).
|
||||
MOVQ R13, R8
|
||||
ADDQ R14, R8
|
||||
MOVQ R14, R9
|
||||
XORQ R10, R10
|
||||
XORQ R11, R11
|
||||
SUBQ R13, R11
|
||||
MOVQ prime1, v1
|
||||
ADDQ prime2, v1
|
||||
MOVQ prime2, v2
|
||||
XORQ v3, v3
|
||||
XORQ v4, v4
|
||||
SUBQ prime1, v4
|
||||
|
||||
// Loop until SI > BX.
|
||||
blockLoop:
|
||||
round(R8)
|
||||
round(R9)
|
||||
round(R10)
|
||||
round(R11)
|
||||
blockLoop()
|
||||
|
||||
CMPQ SI, BX
|
||||
JLE blockLoop
|
||||
MOVQ v1, h
|
||||
ROLQ $1, h
|
||||
MOVQ v2, x
|
||||
ROLQ $7, x
|
||||
ADDQ x, h
|
||||
MOVQ v3, x
|
||||
ROLQ $12, x
|
||||
ADDQ x, h
|
||||
MOVQ v4, x
|
||||
ROLQ $18, x
|
||||
ADDQ x, h
|
||||
|
||||
MOVQ R8, AX
|
||||
ROLQ $1, AX
|
||||
MOVQ R9, R12
|
||||
ROLQ $7, R12
|
||||
ADDQ R12, AX
|
||||
MOVQ R10, R12
|
||||
ROLQ $12, R12
|
||||
ADDQ R12, AX
|
||||
MOVQ R11, R12
|
||||
ROLQ $18, R12
|
||||
ADDQ R12, AX
|
||||
|
||||
mergeRound(AX, R8)
|
||||
mergeRound(AX, R9)
|
||||
mergeRound(AX, R10)
|
||||
mergeRound(AX, R11)
|
||||
mergeRound(h, v1)
|
||||
mergeRound(h, v2)
|
||||
mergeRound(h, v3)
|
||||
mergeRound(h, v4)
|
||||
|
||||
JMP afterBlocks
|
||||
|
||||
noBlocks:
|
||||
MOVQ ·prime5v(SB), AX
|
||||
MOVQ ·primes+32(SB), h
|
||||
|
||||
afterBlocks:
|
||||
ADDQ DX, AX
|
||||
ADDQ n, h
|
||||
|
||||
// Right now BX has len(b)-32, and we want to loop until SI > len(b)-8.
|
||||
ADDQ $24, BX
|
||||
ADDQ $24, end
|
||||
CMPQ p, end
|
||||
JG try4
|
||||
|
||||
CMPQ SI, BX
|
||||
JG fourByte
|
||||
loop8:
|
||||
MOVQ (p), x
|
||||
ADDQ $8, p
|
||||
round0(x)
|
||||
XORQ x, h
|
||||
ROLQ $27, h
|
||||
IMULQ prime1, h
|
||||
ADDQ prime4, h
|
||||
|
||||
wordLoop:
|
||||
// Calculate k1.
|
||||
MOVQ (SI), R8
|
||||
ADDQ $8, SI
|
||||
IMULQ R14, R8
|
||||
ROLQ $31, R8
|
||||
IMULQ R13, R8
|
||||
CMPQ p, end
|
||||
JLE loop8
|
||||
|
||||
XORQ R8, AX
|
||||
ROLQ $27, AX
|
||||
IMULQ R13, AX
|
||||
ADDQ DI, AX
|
||||
try4:
|
||||
ADDQ $4, end
|
||||
CMPQ p, end
|
||||
JG try1
|
||||
|
||||
CMPQ SI, BX
|
||||
JLE wordLoop
|
||||
MOVL (p), x
|
||||
ADDQ $4, p
|
||||
IMULQ prime1, x
|
||||
XORQ x, h
|
||||
|
||||
fourByte:
|
||||
ADDQ $4, BX
|
||||
CMPQ SI, BX
|
||||
JG singles
|
||||
ROLQ $23, h
|
||||
IMULQ prime2, h
|
||||
ADDQ ·primes+16(SB), h
|
||||
|
||||
MOVL (SI), R8
|
||||
ADDQ $4, SI
|
||||
IMULQ R13, R8
|
||||
XORQ R8, AX
|
||||
|
||||
ROLQ $23, AX
|
||||
IMULQ R14, AX
|
||||
ADDQ ·prime3v(SB), AX
|
||||
|
||||
singles:
|
||||
ADDQ $4, BX
|
||||
CMPQ SI, BX
|
||||
try1:
|
||||
ADDQ $4, end
|
||||
CMPQ p, end
|
||||
JGE finalize
|
||||
|
||||
singlesLoop:
|
||||
MOVBQZX (SI), R12
|
||||
ADDQ $1, SI
|
||||
IMULQ ·prime5v(SB), R12
|
||||
XORQ R12, AX
|
||||
loop1:
|
||||
MOVBQZX (p), x
|
||||
ADDQ $1, p
|
||||
IMULQ ·primes+32(SB), x
|
||||
XORQ x, h
|
||||
ROLQ $11, h
|
||||
IMULQ prime1, h
|
||||
|
||||
ROLQ $11, AX
|
||||
IMULQ R13, AX
|
||||
|
||||
CMPQ SI, BX
|
||||
JL singlesLoop
|
||||
CMPQ p, end
|
||||
JL loop1
|
||||
|
||||
finalize:
|
||||
MOVQ AX, R12
|
||||
SHRQ $33, R12
|
||||
XORQ R12, AX
|
||||
IMULQ R14, AX
|
||||
MOVQ AX, R12
|
||||
SHRQ $29, R12
|
||||
XORQ R12, AX
|
||||
IMULQ ·prime3v(SB), AX
|
||||
MOVQ AX, R12
|
||||
SHRQ $32, R12
|
||||
XORQ R12, AX
|
||||
MOVQ h, x
|
||||
SHRQ $33, x
|
||||
XORQ x, h
|
||||
IMULQ prime2, h
|
||||
MOVQ h, x
|
||||
SHRQ $29, x
|
||||
XORQ x, h
|
||||
IMULQ ·primes+16(SB), h
|
||||
MOVQ h, x
|
||||
SHRQ $32, x
|
||||
XORQ x, h
|
||||
|
||||
MOVQ AX, ret+24(FP)
|
||||
MOVQ h, ret+24(FP)
|
||||
RET
|
||||
|
||||
// writeBlocks uses the same registers as above except that it uses AX to store
|
||||
// the d pointer.
|
||||
|
||||
// func writeBlocks(d *Digest, b []byte) int
|
||||
TEXT ·writeBlocks(SB), NOSPLIT, $0-40
|
||||
TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40
|
||||
// Load fixed primes needed for round.
|
||||
MOVQ ·prime1v(SB), R13
|
||||
MOVQ ·prime2v(SB), R14
|
||||
MOVQ ·primes+0(SB), prime1
|
||||
MOVQ ·primes+8(SB), prime2
|
||||
|
||||
// Load slice.
|
||||
MOVQ b_base+8(FP), SI
|
||||
MOVQ b_len+16(FP), DX
|
||||
LEAQ (SI)(DX*1), BX
|
||||
SUBQ $32, BX
|
||||
MOVQ b_base+8(FP), p
|
||||
MOVQ b_len+16(FP), n
|
||||
LEAQ (p)(n*1), end
|
||||
SUBQ $32, end
|
||||
|
||||
// Load vN from d.
|
||||
MOVQ d+0(FP), AX
|
||||
MOVQ 0(AX), R8 // v1
|
||||
MOVQ 8(AX), R9 // v2
|
||||
MOVQ 16(AX), R10 // v3
|
||||
MOVQ 24(AX), R11 // v4
|
||||
MOVQ s+0(FP), d
|
||||
MOVQ 0(d), v1
|
||||
MOVQ 8(d), v2
|
||||
MOVQ 16(d), v3
|
||||
MOVQ 24(d), v4
|
||||
|
||||
// We don't need to check the loop condition here; this function is
|
||||
// always called with at least one block of data to process.
|
||||
blockLoop:
|
||||
round(R8)
|
||||
round(R9)
|
||||
round(R10)
|
||||
round(R11)
|
||||
|
||||
CMPQ SI, BX
|
||||
JLE blockLoop
|
||||
blockLoop()
|
||||
|
||||
// Copy vN back to d.
|
||||
MOVQ R8, 0(AX)
|
||||
MOVQ R9, 8(AX)
|
||||
MOVQ R10, 16(AX)
|
||||
MOVQ R11, 24(AX)
|
||||
MOVQ v1, 0(d)
|
||||
MOVQ v2, 8(d)
|
||||
MOVQ v3, 16(d)
|
||||
MOVQ v4, 24(d)
|
||||
|
||||
// The number of bytes written is SI minus the old base pointer.
|
||||
SUBQ b_base+8(FP), SI
|
||||
MOVQ SI, ret+32(FP)
|
||||
// The number of bytes written is p minus the old base pointer.
|
||||
SUBQ b_base+8(FP), p
|
||||
MOVQ p, ret+32(FP)
|
||||
|
||||
RET
|
||||
|
||||
136
vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s
generated
vendored
136
vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s
generated
vendored
@ -1,13 +1,17 @@
|
||||
// +build gc,!purego,!noasm
|
||||
//go:build !appengine && gc && !purego && !noasm
|
||||
// +build !appengine
|
||||
// +build gc
|
||||
// +build !purego
|
||||
// +build !noasm
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
// Register allocation.
|
||||
// Registers:
|
||||
#define digest R1
|
||||
#define h R2 // Return value.
|
||||
#define p R3 // Input pointer.
|
||||
#define len R4
|
||||
#define nblocks R5 // len / 32.
|
||||
#define h R2 // return value
|
||||
#define p R3 // input pointer
|
||||
#define n R4 // input length
|
||||
#define nblocks R5 // n / 32
|
||||
#define prime1 R7
|
||||
#define prime2 R8
|
||||
#define prime3 R9
|
||||
@ -25,60 +29,52 @@
|
||||
#define round(acc, x) \
|
||||
MADD prime2, acc, x, acc \
|
||||
ROR $64-31, acc \
|
||||
MUL prime1, acc \
|
||||
MUL prime1, acc
|
||||
|
||||
// x = round(0, x).
|
||||
// round0 performs the operation x = round(0, x).
|
||||
#define round0(x) \
|
||||
MUL prime2, x \
|
||||
ROR $64-31, x \
|
||||
MUL prime1, x \
|
||||
MUL prime1, x
|
||||
|
||||
#define mergeRound(x) \
|
||||
round0(x) \
|
||||
EOR x, h \
|
||||
MADD h, prime4, prime1, h \
|
||||
#define mergeRound(acc, x) \
|
||||
round0(x) \
|
||||
EOR x, acc \
|
||||
MADD acc, prime4, prime1, acc
|
||||
|
||||
// Update v[1-4] with 32-byte blocks. Assumes len >= 32.
|
||||
#define blocksLoop() \
|
||||
LSR $5, len, nblocks \
|
||||
PCALIGN $16 \
|
||||
loop: \
|
||||
LDP.P 32(p), (x1, x2) \
|
||||
round(v1, x1) \
|
||||
LDP -16(p), (x3, x4) \
|
||||
round(v2, x2) \
|
||||
SUB $1, nblocks \
|
||||
round(v3, x3) \
|
||||
round(v4, x4) \
|
||||
CBNZ nblocks, loop \
|
||||
|
||||
// The primes are repeated here to ensure that they're stored
|
||||
// in a contiguous array, so we can load them with LDP.
|
||||
DATA primes<> +0(SB)/8, $11400714785074694791
|
||||
DATA primes<> +8(SB)/8, $14029467366897019727
|
||||
DATA primes<>+16(SB)/8, $1609587929392839161
|
||||
DATA primes<>+24(SB)/8, $9650029242287828579
|
||||
DATA primes<>+32(SB)/8, $2870177450012600261
|
||||
GLOBL primes<>(SB), NOPTR+RODATA, $40
|
||||
// blockLoop processes as many 32-byte blocks as possible,
|
||||
// updating v1, v2, v3, and v4. It assumes that n >= 32.
|
||||
#define blockLoop() \
|
||||
LSR $5, n, nblocks \
|
||||
PCALIGN $16 \
|
||||
loop: \
|
||||
LDP.P 16(p), (x1, x2) \
|
||||
LDP.P 16(p), (x3, x4) \
|
||||
round(v1, x1) \
|
||||
round(v2, x2) \
|
||||
round(v3, x3) \
|
||||
round(v4, x4) \
|
||||
SUB $1, nblocks \
|
||||
CBNZ nblocks, loop
|
||||
|
||||
// func Sum64(b []byte) uint64
|
||||
TEXT ·Sum64(SB), NOFRAME+NOSPLIT, $0-32
|
||||
LDP b_base+0(FP), (p, len)
|
||||
TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32
|
||||
LDP b_base+0(FP), (p, n)
|
||||
|
||||
LDP primes<> +0(SB), (prime1, prime2)
|
||||
LDP primes<>+16(SB), (prime3, prime4)
|
||||
MOVD primes<>+32(SB), prime5
|
||||
LDP ·primes+0(SB), (prime1, prime2)
|
||||
LDP ·primes+16(SB), (prime3, prime4)
|
||||
MOVD ·primes+32(SB), prime5
|
||||
|
||||
CMP $32, len
|
||||
CSEL LO, prime5, ZR, h // if len < 32 { h = prime5 } else { h = 0 }
|
||||
BLO afterLoop
|
||||
CMP $32, n
|
||||
CSEL LT, prime5, ZR, h // if n < 32 { h = prime5 } else { h = 0 }
|
||||
BLT afterLoop
|
||||
|
||||
ADD prime1, prime2, v1
|
||||
MOVD prime2, v2
|
||||
MOVD $0, v3
|
||||
NEG prime1, v4
|
||||
|
||||
blocksLoop()
|
||||
blockLoop()
|
||||
|
||||
ROR $64-1, v1, x1
|
||||
ROR $64-7, v2, x2
|
||||
@ -88,71 +84,75 @@ TEXT ·Sum64(SB), NOFRAME+NOSPLIT, $0-32
|
||||
ADD x3, x4
|
||||
ADD x2, x4, h
|
||||
|
||||
mergeRound(v1)
|
||||
mergeRound(v2)
|
||||
mergeRound(v3)
|
||||
mergeRound(v4)
|
||||
mergeRound(h, v1)
|
||||
mergeRound(h, v2)
|
||||
mergeRound(h, v3)
|
||||
mergeRound(h, v4)
|
||||
|
||||
afterLoop:
|
||||
ADD len, h
|
||||
ADD n, h
|
||||
|
||||
TBZ $4, len, try8
|
||||
TBZ $4, n, try8
|
||||
LDP.P 16(p), (x1, x2)
|
||||
|
||||
round0(x1)
|
||||
|
||||
// NOTE: here and below, sequencing the EOR after the ROR (using a
|
||||
// rotated register) is worth a small but measurable speedup for small
|
||||
// inputs.
|
||||
ROR $64-27, h
|
||||
EOR x1 @> 64-27, h, h
|
||||
MADD h, prime4, prime1, h
|
||||
|
||||
round0(x2)
|
||||
ROR $64-27, h
|
||||
EOR x2 @> 64-27, h
|
||||
EOR x2 @> 64-27, h, h
|
||||
MADD h, prime4, prime1, h
|
||||
|
||||
try8:
|
||||
TBZ $3, len, try4
|
||||
TBZ $3, n, try4
|
||||
MOVD.P 8(p), x1
|
||||
|
||||
round0(x1)
|
||||
ROR $64-27, h
|
||||
EOR x1 @> 64-27, h
|
||||
EOR x1 @> 64-27, h, h
|
||||
MADD h, prime4, prime1, h
|
||||
|
||||
try4:
|
||||
TBZ $2, len, try2
|
||||
TBZ $2, n, try2
|
||||
MOVWU.P 4(p), x2
|
||||
|
||||
MUL prime1, x2
|
||||
ROR $64-23, h
|
||||
EOR x2 @> 64-23, h
|
||||
EOR x2 @> 64-23, h, h
|
||||
MADD h, prime3, prime2, h
|
||||
|
||||
try2:
|
||||
TBZ $1, len, try1
|
||||
TBZ $1, n, try1
|
||||
MOVHU.P 2(p), x3
|
||||
AND $255, x3, x1
|
||||
LSR $8, x3, x2
|
||||
|
||||
MUL prime5, x1
|
||||
ROR $64-11, h
|
||||
EOR x1 @> 64-11, h
|
||||
EOR x1 @> 64-11, h, h
|
||||
MUL prime1, h
|
||||
|
||||
MUL prime5, x2
|
||||
ROR $64-11, h
|
||||
EOR x2 @> 64-11, h
|
||||
EOR x2 @> 64-11, h, h
|
||||
MUL prime1, h
|
||||
|
||||
try1:
|
||||
TBZ $0, len, end
|
||||
TBZ $0, n, finalize
|
||||
MOVBU (p), x4
|
||||
|
||||
MUL prime5, x4
|
||||
ROR $64-11, h
|
||||
EOR x4 @> 64-11, h
|
||||
EOR x4 @> 64-11, h, h
|
||||
MUL prime1, h
|
||||
|
||||
end:
|
||||
finalize:
|
||||
EOR h >> 33, h
|
||||
MUL prime2, h
|
||||
EOR h >> 29, h
|
||||
@ -163,24 +163,22 @@ end:
|
||||
RET
|
||||
|
||||
// func writeBlocks(d *Digest, b []byte) int
|
||||
//
|
||||
// Assumes len(b) >= 32.
|
||||
TEXT ·writeBlocks(SB), NOFRAME+NOSPLIT, $0-40
|
||||
LDP primes<>(SB), (prime1, prime2)
|
||||
TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40
|
||||
LDP ·primes+0(SB), (prime1, prime2)
|
||||
|
||||
// Load state. Assume v[1-4] are stored contiguously.
|
||||
MOVD d+0(FP), digest
|
||||
LDP 0(digest), (v1, v2)
|
||||
LDP 16(digest), (v3, v4)
|
||||
|
||||
LDP b_base+8(FP), (p, len)
|
||||
LDP b_base+8(FP), (p, n)
|
||||
|
||||
blocksLoop()
|
||||
blockLoop()
|
||||
|
||||
// Store updated state.
|
||||
STP (v1, v2), 0(digest)
|
||||
STP (v3, v4), 16(digest)
|
||||
|
||||
BIC $31, len
|
||||
MOVD len, ret+32(FP)
|
||||
BIC $31, n
|
||||
MOVD n, ret+32(FP)
|
||||
RET
|
||||
|
||||
2
vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go
generated
vendored
2
vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go
generated
vendored
@ -13,4 +13,4 @@ package xxhash
|
||||
func Sum64(b []byte) uint64
|
||||
|
||||
//go:noescape
|
||||
func writeBlocks(d *Digest, b []byte) int
|
||||
func writeBlocks(s *Digest, b []byte) int
|
||||
|
||||
19
vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go
generated
vendored
19
vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go
generated
vendored
@ -15,10 +15,10 @@ func Sum64(b []byte) uint64 {
|
||||
var h uint64
|
||||
|
||||
if n >= 32 {
|
||||
v1 := prime1v + prime2
|
||||
v1 := primes[0] + prime2
|
||||
v2 := prime2
|
||||
v3 := uint64(0)
|
||||
v4 := -prime1v
|
||||
v4 := -primes[0]
|
||||
for len(b) >= 32 {
|
||||
v1 = round(v1, u64(b[0:8:len(b)]))
|
||||
v2 = round(v2, u64(b[8:16:len(b)]))
|
||||
@ -37,19 +37,18 @@ func Sum64(b []byte) uint64 {
|
||||
|
||||
h += uint64(n)
|
||||
|
||||
i, end := 0, len(b)
|
||||
for ; i+8 <= end; i += 8 {
|
||||
k1 := round(0, u64(b[i:i+8:len(b)]))
|
||||
for ; len(b) >= 8; b = b[8:] {
|
||||
k1 := round(0, u64(b[:8]))
|
||||
h ^= k1
|
||||
h = rol27(h)*prime1 + prime4
|
||||
}
|
||||
if i+4 <= end {
|
||||
h ^= uint64(u32(b[i:i+4:len(b)])) * prime1
|
||||
if len(b) >= 4 {
|
||||
h ^= uint64(u32(b[:4])) * prime1
|
||||
h = rol23(h)*prime2 + prime3
|
||||
i += 4
|
||||
b = b[4:]
|
||||
}
|
||||
for ; i < end; i++ {
|
||||
h ^= uint64(b[i]) * prime5
|
||||
for ; len(b) > 0; b = b[1:] {
|
||||
h ^= uint64(b[0]) * prime5
|
||||
h = rol11(h) * prime1
|
||||
}
|
||||
|
||||
|
||||
22
vendor/github.com/klauspost/compress/zstd/seqdec.go
generated
vendored
22
vendor/github.com/klauspost/compress/zstd/seqdec.go
generated
vendored
@ -99,6 +99,21 @@ func (s *sequenceDecs) initialize(br *bitReader, hist *history, out []byte) erro
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *sequenceDecs) freeDecoders() {
|
||||
if f := s.litLengths.fse; f != nil && !f.preDefined {
|
||||
fseDecoderPool.Put(f)
|
||||
s.litLengths.fse = nil
|
||||
}
|
||||
if f := s.offsets.fse; f != nil && !f.preDefined {
|
||||
fseDecoderPool.Put(f)
|
||||
s.offsets.fse = nil
|
||||
}
|
||||
if f := s.matchLengths.fse; f != nil && !f.preDefined {
|
||||
fseDecoderPool.Put(f)
|
||||
s.matchLengths.fse = nil
|
||||
}
|
||||
}
|
||||
|
||||
// execute will execute the decoded sequence with the provided history.
|
||||
// The sequence must be evaluated before being sent.
|
||||
func (s *sequenceDecs) execute(seqs []seqVals, hist []byte) error {
|
||||
@ -299,7 +314,10 @@ func (s *sequenceDecs) decodeSync(hist []byte) error {
|
||||
}
|
||||
size := ll + ml + len(out)
|
||||
if size-startSize > maxBlockSize {
|
||||
return fmt.Errorf("output (%d) bigger than max block size (%d)", size-startSize, maxBlockSize)
|
||||
if size-startSize == 424242 {
|
||||
panic("here")
|
||||
}
|
||||
return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize)
|
||||
}
|
||||
if size > cap(out) {
|
||||
// Not enough size, which can happen under high volume block streaming conditions
|
||||
@ -411,7 +429,7 @@ func (s *sequenceDecs) decodeSync(hist []byte) error {
|
||||
|
||||
// Check if space for literals
|
||||
if size := len(s.literals) + len(s.out) - startSize; size > maxBlockSize {
|
||||
return fmt.Errorf("output (%d) bigger than max block size (%d)", size, maxBlockSize)
|
||||
return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize)
|
||||
}
|
||||
|
||||
// Add final literals
|
||||
|
||||
43
vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go
generated
vendored
43
vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go
generated
vendored
@ -32,18 +32,22 @@ type decodeSyncAsmContext struct {
|
||||
// sequenceDecs_decodeSync_amd64 implements the main loop of sequenceDecs.decodeSync in x86 asm.
|
||||
//
|
||||
// Please refer to seqdec_generic.go for the reference implementation.
|
||||
//
|
||||
//go:noescape
|
||||
func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int
|
||||
|
||||
// sequenceDecs_decodeSync_bmi2 implements the main loop of sequenceDecs.decodeSync in x86 asm with BMI2 extensions.
|
||||
//
|
||||
//go:noescape
|
||||
func sequenceDecs_decodeSync_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int
|
||||
|
||||
// sequenceDecs_decodeSync_safe_amd64 does the same as above, but does not write more than output buffer.
|
||||
//
|
||||
//go:noescape
|
||||
func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int
|
||||
|
||||
// sequenceDecs_decodeSync_safe_bmi2 does the same as above, but does not write more than output buffer.
|
||||
//
|
||||
//go:noescape
|
||||
func sequenceDecs_decodeSync_safe_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int
|
||||
|
||||
@ -55,16 +59,22 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) {
|
||||
if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSize {
|
||||
return false, nil
|
||||
}
|
||||
useSafe := false
|
||||
if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSizeAlloc {
|
||||
useSafe = true
|
||||
}
|
||||
if s.maxSyncLen > 0 && cap(s.out)-len(s.out)-compressedBlockOverAlloc < int(s.maxSyncLen) {
|
||||
useSafe = true
|
||||
}
|
||||
if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc {
|
||||
useSafe = true
|
||||
}
|
||||
|
||||
// FIXME: Using unsafe memory copies leads to rare, random crashes
|
||||
// with fuzz testing. It is therefore disabled for now.
|
||||
const useSafe = true
|
||||
/*
|
||||
useSafe := false
|
||||
if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSizeAlloc {
|
||||
useSafe = true
|
||||
}
|
||||
if s.maxSyncLen > 0 && cap(s.out)-len(s.out)-compressedBlockOverAlloc < int(s.maxSyncLen) {
|
||||
useSafe = true
|
||||
}
|
||||
if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc {
|
||||
useSafe = true
|
||||
}
|
||||
*/
|
||||
|
||||
br := s.br
|
||||
|
||||
@ -129,7 +139,7 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) {
|
||||
if debugDecoder {
|
||||
println("msl:", s.maxSyncLen, "cap", cap(s.out), "bef:", startSize, "sz:", size-startSize, "mbs:", maxBlockSize, "outsz:", cap(s.out)-startSize)
|
||||
}
|
||||
return true, fmt.Errorf("output (%d) bigger than max block size (%d)", size-startSize, maxBlockSize)
|
||||
return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize)
|
||||
|
||||
default:
|
||||
return true, fmt.Errorf("sequenceDecs_decode returned erronous code %d", errCode)
|
||||
@ -137,7 +147,8 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) {
|
||||
|
||||
s.seqSize += ctx.litRemain
|
||||
if s.seqSize > maxBlockSize {
|
||||
return true, fmt.Errorf("output (%d) bigger than max block size (%d)", s.seqSize, maxBlockSize)
|
||||
return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize)
|
||||
|
||||
}
|
||||
err := br.close()
|
||||
if err != nil {
|
||||
@ -195,20 +206,24 @@ const errorNotEnoughSpace = 5
|
||||
// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm.
|
||||
//
|
||||
// Please refer to seqdec_generic.go for the reference implementation.
|
||||
//
|
||||
//go:noescape
|
||||
func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
|
||||
|
||||
// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm.
|
||||
//
|
||||
// Please refer to seqdec_generic.go for the reference implementation.
|
||||
//
|
||||
//go:noescape
|
||||
func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
|
||||
|
||||
// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm with BMI2 extensions.
|
||||
//
|
||||
//go:noescape
|
||||
func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
|
||||
|
||||
// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm with BMI2 extensions.
|
||||
//
|
||||
//go:noescape
|
||||
func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
|
||||
|
||||
@ -275,7 +290,7 @@ func (s *sequenceDecs) decode(seqs []seqVals) error {
|
||||
|
||||
s.seqSize += ctx.litRemain
|
||||
if s.seqSize > maxBlockSize {
|
||||
return fmt.Errorf("output (%d) bigger than max block size (%d)", s.seqSize, maxBlockSize)
|
||||
return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize)
|
||||
}
|
||||
err := br.close()
|
||||
if err != nil {
|
||||
@ -302,10 +317,12 @@ type executeAsmContext struct {
|
||||
// Returns false if a match offset is too big.
|
||||
//
|
||||
// Please refer to seqdec_generic.go for the reference implementation.
|
||||
//
|
||||
//go:noescape
|
||||
func sequenceDecs_executeSimple_amd64(ctx *executeAsmContext) bool
|
||||
|
||||
// Same as above, but with safe memcopies
|
||||
//
|
||||
//go:noescape
|
||||
func sequenceDecs_executeSimple_safe_amd64(ctx *executeAsmContext) bool
|
||||
|
||||
|
||||
489
vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s
generated
vendored
489
vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s
generated
vendored
@ -1,7 +1,6 @@
|
||||
// Code generated by command: go run gen.go -out ../seqdec_amd64.s -pkg=zstd. DO NOT EDIT.
|
||||
|
||||
//go:build !appengine && !noasm && gc && !noasm
|
||||
// +build !appengine,!noasm,gc,!noasm
|
||||
|
||||
// func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
|
||||
// Requires: CMOV
|
||||
@ -52,34 +51,46 @@ sequenceDecs_decode_amd64_fill_byte_by_byte:
|
||||
|
||||
sequenceDecs_decode_amd64_fill_end:
|
||||
// Update offset
|
||||
MOVQ R9, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R15
|
||||
SHLQ CL, R15
|
||||
MOVB AH, CL
|
||||
ADDQ CX, BX
|
||||
NEGL CX
|
||||
SHRQ CL, R15
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
CMOVQEQ CX, R15
|
||||
ADDQ R15, AX
|
||||
MOVQ AX, 16(R10)
|
||||
MOVQ R9, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R15
|
||||
SHLQ CL, R15
|
||||
MOVB AH, CL
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
JZ sequenceDecs_decode_amd64_of_update_zero
|
||||
ADDQ CX, BX
|
||||
CMPQ BX, $0x40
|
||||
JA sequenceDecs_decode_amd64_of_update_zero
|
||||
CMPQ CX, $0x40
|
||||
JAE sequenceDecs_decode_amd64_of_update_zero
|
||||
NEGQ CX
|
||||
SHRQ CL, R15
|
||||
ADDQ R15, AX
|
||||
|
||||
sequenceDecs_decode_amd64_of_update_zero:
|
||||
MOVQ AX, 16(R10)
|
||||
|
||||
// Update match length
|
||||
MOVQ R8, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R15
|
||||
SHLQ CL, R15
|
||||
MOVB AH, CL
|
||||
ADDQ CX, BX
|
||||
NEGL CX
|
||||
SHRQ CL, R15
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
CMOVQEQ CX, R15
|
||||
ADDQ R15, AX
|
||||
MOVQ AX, 8(R10)
|
||||
MOVQ R8, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R15
|
||||
SHLQ CL, R15
|
||||
MOVB AH, CL
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
JZ sequenceDecs_decode_amd64_ml_update_zero
|
||||
ADDQ CX, BX
|
||||
CMPQ BX, $0x40
|
||||
JA sequenceDecs_decode_amd64_ml_update_zero
|
||||
CMPQ CX, $0x40
|
||||
JAE sequenceDecs_decode_amd64_ml_update_zero
|
||||
NEGQ CX
|
||||
SHRQ CL, R15
|
||||
ADDQ R15, AX
|
||||
|
||||
sequenceDecs_decode_amd64_ml_update_zero:
|
||||
MOVQ AX, 8(R10)
|
||||
|
||||
// Fill bitreader to have enough for the remaining
|
||||
CMPQ SI, $0x08
|
||||
@ -107,19 +118,25 @@ sequenceDecs_decode_amd64_fill_2_byte_by_byte:
|
||||
|
||||
sequenceDecs_decode_amd64_fill_2_end:
|
||||
// Update literal length
|
||||
MOVQ DI, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R15
|
||||
SHLQ CL, R15
|
||||
MOVB AH, CL
|
||||
ADDQ CX, BX
|
||||
NEGL CX
|
||||
SHRQ CL, R15
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
CMOVQEQ CX, R15
|
||||
ADDQ R15, AX
|
||||
MOVQ AX, (R10)
|
||||
MOVQ DI, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R15
|
||||
SHLQ CL, R15
|
||||
MOVB AH, CL
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
JZ sequenceDecs_decode_amd64_ll_update_zero
|
||||
ADDQ CX, BX
|
||||
CMPQ BX, $0x40
|
||||
JA sequenceDecs_decode_amd64_ll_update_zero
|
||||
CMPQ CX, $0x40
|
||||
JAE sequenceDecs_decode_amd64_ll_update_zero
|
||||
NEGQ CX
|
||||
SHRQ CL, R15
|
||||
ADDQ R15, AX
|
||||
|
||||
sequenceDecs_decode_amd64_ll_update_zero:
|
||||
MOVQ AX, (R10)
|
||||
|
||||
// Fill bitreader for state updates
|
||||
MOVQ R14, (SP)
|
||||
@ -198,7 +215,7 @@ sequenceDecs_decode_amd64_skip_update:
|
||||
MOVQ R12, R13
|
||||
MOVQ R11, R12
|
||||
MOVQ CX, R11
|
||||
JMP sequenceDecs_decode_amd64_adjust_end
|
||||
JMP sequenceDecs_decode_amd64_after_adjust
|
||||
|
||||
sequenceDecs_decode_amd64_adjust_offsetB_1_or_0:
|
||||
CMPQ (R10), $0x00000000
|
||||
@ -210,7 +227,7 @@ sequenceDecs_decode_amd64_adjust_offset_maybezero:
|
||||
TESTQ CX, CX
|
||||
JNZ sequenceDecs_decode_amd64_adjust_offset_nonzero
|
||||
MOVQ R11, CX
|
||||
JMP sequenceDecs_decode_amd64_adjust_end
|
||||
JMP sequenceDecs_decode_amd64_after_adjust
|
||||
|
||||
sequenceDecs_decode_amd64_adjust_offset_nonzero:
|
||||
CMPQ CX, $0x01
|
||||
@ -247,7 +264,7 @@ sequenceDecs_decode_amd64_adjust_temp_valid:
|
||||
MOVQ AX, R11
|
||||
MOVQ AX, CX
|
||||
|
||||
sequenceDecs_decode_amd64_adjust_end:
|
||||
sequenceDecs_decode_amd64_after_adjust:
|
||||
MOVQ CX, 16(R10)
|
||||
|
||||
// Check values
|
||||
@ -303,10 +320,6 @@ error_not_enough_literals:
|
||||
MOVQ $0x00000004, ret+24(FP)
|
||||
RET
|
||||
|
||||
// Return with not enough output space error
|
||||
MOVQ $0x00000005, ret+24(FP)
|
||||
RET
|
||||
|
||||
// func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
|
||||
// Requires: CMOV
|
||||
TEXT ·sequenceDecs_decode_56_amd64(SB), $8-32
|
||||
@ -356,49 +369,67 @@ sequenceDecs_decode_56_amd64_fill_byte_by_byte:
|
||||
|
||||
sequenceDecs_decode_56_amd64_fill_end:
|
||||
// Update offset
|
||||
MOVQ R9, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R15
|
||||
SHLQ CL, R15
|
||||
MOVB AH, CL
|
||||
ADDQ CX, BX
|
||||
NEGL CX
|
||||
SHRQ CL, R15
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
CMOVQEQ CX, R15
|
||||
ADDQ R15, AX
|
||||
MOVQ AX, 16(R10)
|
||||
MOVQ R9, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R15
|
||||
SHLQ CL, R15
|
||||
MOVB AH, CL
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
JZ sequenceDecs_decode_56_amd64_of_update_zero
|
||||
ADDQ CX, BX
|
||||
CMPQ BX, $0x40
|
||||
JA sequenceDecs_decode_56_amd64_of_update_zero
|
||||
CMPQ CX, $0x40
|
||||
JAE sequenceDecs_decode_56_amd64_of_update_zero
|
||||
NEGQ CX
|
||||
SHRQ CL, R15
|
||||
ADDQ R15, AX
|
||||
|
||||
sequenceDecs_decode_56_amd64_of_update_zero:
|
||||
MOVQ AX, 16(R10)
|
||||
|
||||
// Update match length
|
||||
MOVQ R8, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R15
|
||||
SHLQ CL, R15
|
||||
MOVB AH, CL
|
||||
ADDQ CX, BX
|
||||
NEGL CX
|
||||
SHRQ CL, R15
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
CMOVQEQ CX, R15
|
||||
ADDQ R15, AX
|
||||
MOVQ AX, 8(R10)
|
||||
MOVQ R8, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R15
|
||||
SHLQ CL, R15
|
||||
MOVB AH, CL
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
JZ sequenceDecs_decode_56_amd64_ml_update_zero
|
||||
ADDQ CX, BX
|
||||
CMPQ BX, $0x40
|
||||
JA sequenceDecs_decode_56_amd64_ml_update_zero
|
||||
CMPQ CX, $0x40
|
||||
JAE sequenceDecs_decode_56_amd64_ml_update_zero
|
||||
NEGQ CX
|
||||
SHRQ CL, R15
|
||||
ADDQ R15, AX
|
||||
|
||||
sequenceDecs_decode_56_amd64_ml_update_zero:
|
||||
MOVQ AX, 8(R10)
|
||||
|
||||
// Update literal length
|
||||
MOVQ DI, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R15
|
||||
SHLQ CL, R15
|
||||
MOVB AH, CL
|
||||
ADDQ CX, BX
|
||||
NEGL CX
|
||||
SHRQ CL, R15
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
CMOVQEQ CX, R15
|
||||
ADDQ R15, AX
|
||||
MOVQ AX, (R10)
|
||||
MOVQ DI, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R15
|
||||
SHLQ CL, R15
|
||||
MOVB AH, CL
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
JZ sequenceDecs_decode_56_amd64_ll_update_zero
|
||||
ADDQ CX, BX
|
||||
CMPQ BX, $0x40
|
||||
JA sequenceDecs_decode_56_amd64_ll_update_zero
|
||||
CMPQ CX, $0x40
|
||||
JAE sequenceDecs_decode_56_amd64_ll_update_zero
|
||||
NEGQ CX
|
||||
SHRQ CL, R15
|
||||
ADDQ R15, AX
|
||||
|
||||
sequenceDecs_decode_56_amd64_ll_update_zero:
|
||||
MOVQ AX, (R10)
|
||||
|
||||
// Fill bitreader for state updates
|
||||
MOVQ R14, (SP)
|
||||
@ -477,7 +508,7 @@ sequenceDecs_decode_56_amd64_skip_update:
|
||||
MOVQ R12, R13
|
||||
MOVQ R11, R12
|
||||
MOVQ CX, R11
|
||||
JMP sequenceDecs_decode_56_amd64_adjust_end
|
||||
JMP sequenceDecs_decode_56_amd64_after_adjust
|
||||
|
||||
sequenceDecs_decode_56_amd64_adjust_offsetB_1_or_0:
|
||||
CMPQ (R10), $0x00000000
|
||||
@ -489,7 +520,7 @@ sequenceDecs_decode_56_amd64_adjust_offset_maybezero:
|
||||
TESTQ CX, CX
|
||||
JNZ sequenceDecs_decode_56_amd64_adjust_offset_nonzero
|
||||
MOVQ R11, CX
|
||||
JMP sequenceDecs_decode_56_amd64_adjust_end
|
||||
JMP sequenceDecs_decode_56_amd64_after_adjust
|
||||
|
||||
sequenceDecs_decode_56_amd64_adjust_offset_nonzero:
|
||||
CMPQ CX, $0x01
|
||||
@ -526,7 +557,7 @@ sequenceDecs_decode_56_amd64_adjust_temp_valid:
|
||||
MOVQ AX, R11
|
||||
MOVQ AX, CX
|
||||
|
||||
sequenceDecs_decode_56_amd64_adjust_end:
|
||||
sequenceDecs_decode_56_amd64_after_adjust:
|
||||
MOVQ CX, 16(R10)
|
||||
|
||||
// Check values
|
||||
@ -582,10 +613,6 @@ error_not_enough_literals:
|
||||
MOVQ $0x00000004, ret+24(FP)
|
||||
RET
|
||||
|
||||
// Return with not enough output space error
|
||||
MOVQ $0x00000005, ret+24(FP)
|
||||
RET
|
||||
|
||||
// func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
|
||||
// Requires: BMI, BMI2, CMOV
|
||||
TEXT ·sequenceDecs_decode_bmi2(SB), $8-32
|
||||
@ -757,7 +784,7 @@ sequenceDecs_decode_bmi2_skip_update:
|
||||
MOVQ R11, R12
|
||||
MOVQ R10, R11
|
||||
MOVQ CX, R10
|
||||
JMP sequenceDecs_decode_bmi2_adjust_end
|
||||
JMP sequenceDecs_decode_bmi2_after_adjust
|
||||
|
||||
sequenceDecs_decode_bmi2_adjust_offsetB_1_or_0:
|
||||
CMPQ (R9), $0x00000000
|
||||
@ -769,7 +796,7 @@ sequenceDecs_decode_bmi2_adjust_offset_maybezero:
|
||||
TESTQ CX, CX
|
||||
JNZ sequenceDecs_decode_bmi2_adjust_offset_nonzero
|
||||
MOVQ R10, CX
|
||||
JMP sequenceDecs_decode_bmi2_adjust_end
|
||||
JMP sequenceDecs_decode_bmi2_after_adjust
|
||||
|
||||
sequenceDecs_decode_bmi2_adjust_offset_nonzero:
|
||||
CMPQ CX, $0x01
|
||||
@ -806,7 +833,7 @@ sequenceDecs_decode_bmi2_adjust_temp_valid:
|
||||
MOVQ R13, R10
|
||||
MOVQ R13, CX
|
||||
|
||||
sequenceDecs_decode_bmi2_adjust_end:
|
||||
sequenceDecs_decode_bmi2_after_adjust:
|
||||
MOVQ CX, 16(R9)
|
||||
|
||||
// Check values
|
||||
@ -862,10 +889,6 @@ error_not_enough_literals:
|
||||
MOVQ $0x00000004, ret+24(FP)
|
||||
RET
|
||||
|
||||
// Return with not enough output space error
|
||||
MOVQ $0x00000005, ret+24(FP)
|
||||
RET
|
||||
|
||||
// func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
|
||||
// Requires: BMI, BMI2, CMOV
|
||||
TEXT ·sequenceDecs_decode_56_bmi2(SB), $8-32
|
||||
@ -1012,7 +1035,7 @@ sequenceDecs_decode_56_bmi2_skip_update:
|
||||
MOVQ R11, R12
|
||||
MOVQ R10, R11
|
||||
MOVQ CX, R10
|
||||
JMP sequenceDecs_decode_56_bmi2_adjust_end
|
||||
JMP sequenceDecs_decode_56_bmi2_after_adjust
|
||||
|
||||
sequenceDecs_decode_56_bmi2_adjust_offsetB_1_or_0:
|
||||
CMPQ (R9), $0x00000000
|
||||
@ -1024,7 +1047,7 @@ sequenceDecs_decode_56_bmi2_adjust_offset_maybezero:
|
||||
TESTQ CX, CX
|
||||
JNZ sequenceDecs_decode_56_bmi2_adjust_offset_nonzero
|
||||
MOVQ R10, CX
|
||||
JMP sequenceDecs_decode_56_bmi2_adjust_end
|
||||
JMP sequenceDecs_decode_56_bmi2_after_adjust
|
||||
|
||||
sequenceDecs_decode_56_bmi2_adjust_offset_nonzero:
|
||||
CMPQ CX, $0x01
|
||||
@ -1061,7 +1084,7 @@ sequenceDecs_decode_56_bmi2_adjust_temp_valid:
|
||||
MOVQ R13, R10
|
||||
MOVQ R13, CX
|
||||
|
||||
sequenceDecs_decode_56_bmi2_adjust_end:
|
||||
sequenceDecs_decode_56_bmi2_after_adjust:
|
||||
MOVQ CX, 16(R9)
|
||||
|
||||
// Check values
|
||||
@ -1117,10 +1140,6 @@ error_not_enough_literals:
|
||||
MOVQ $0x00000004, ret+24(FP)
|
||||
RET
|
||||
|
||||
// Return with not enough output space error
|
||||
MOVQ $0x00000005, ret+24(FP)
|
||||
RET
|
||||
|
||||
// func sequenceDecs_executeSimple_amd64(ctx *executeAsmContext) bool
|
||||
// Requires: SSE
|
||||
TEXT ·sequenceDecs_executeSimple_amd64(SB), $8-9
|
||||
@ -1354,8 +1373,7 @@ loop_finished:
|
||||
MOVQ ctx+0(FP), AX
|
||||
MOVQ DX, 24(AX)
|
||||
MOVQ DI, 104(AX)
|
||||
MOVQ 80(AX), CX
|
||||
SUBQ CX, SI
|
||||
SUBQ 80(AX), SI
|
||||
MOVQ SI, 112(AX)
|
||||
RET
|
||||
|
||||
@ -1367,8 +1385,7 @@ error_match_off_too_big:
|
||||
MOVQ ctx+0(FP), AX
|
||||
MOVQ DX, 24(AX)
|
||||
MOVQ DI, 104(AX)
|
||||
MOVQ 80(AX), CX
|
||||
SUBQ CX, SI
|
||||
SUBQ 80(AX), SI
|
||||
MOVQ SI, 112(AX)
|
||||
RET
|
||||
|
||||
@ -1712,8 +1729,7 @@ loop_finished:
|
||||
MOVQ ctx+0(FP), AX
|
||||
MOVQ DX, 24(AX)
|
||||
MOVQ DI, 104(AX)
|
||||
MOVQ 80(AX), CX
|
||||
SUBQ CX, SI
|
||||
SUBQ 80(AX), SI
|
||||
MOVQ SI, 112(AX)
|
||||
RET
|
||||
|
||||
@ -1725,8 +1741,7 @@ error_match_off_too_big:
|
||||
MOVQ ctx+0(FP), AX
|
||||
MOVQ DX, 24(AX)
|
||||
MOVQ DI, 104(AX)
|
||||
MOVQ 80(AX), CX
|
||||
SUBQ CX, SI
|
||||
SUBQ 80(AX), SI
|
||||
MOVQ SI, 112(AX)
|
||||
RET
|
||||
|
||||
@ -1749,6 +1764,10 @@ TEXT ·sequenceDecs_decodeSync_amd64(SB), $64-32
|
||||
MOVQ 72(AX), DI
|
||||
MOVQ 80(AX), R8
|
||||
MOVQ 88(AX), R9
|
||||
XORQ CX, CX
|
||||
MOVQ CX, 8(SP)
|
||||
MOVQ CX, 16(SP)
|
||||
MOVQ CX, 24(SP)
|
||||
MOVQ 112(AX), R10
|
||||
MOVQ 128(AX), CX
|
||||
MOVQ CX, 32(SP)
|
||||
@ -1798,34 +1817,46 @@ sequenceDecs_decodeSync_amd64_fill_byte_by_byte:
|
||||
|
||||
sequenceDecs_decodeSync_amd64_fill_end:
|
||||
// Update offset
|
||||
MOVQ R9, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R14
|
||||
SHLQ CL, R14
|
||||
MOVB AH, CL
|
||||
ADDQ CX, BX
|
||||
NEGL CX
|
||||
SHRQ CL, R14
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
CMOVQEQ CX, R14
|
||||
ADDQ R14, AX
|
||||
MOVQ AX, 8(SP)
|
||||
MOVQ R9, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R14
|
||||
SHLQ CL, R14
|
||||
MOVB AH, CL
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
JZ sequenceDecs_decodeSync_amd64_of_update_zero
|
||||
ADDQ CX, BX
|
||||
CMPQ BX, $0x40
|
||||
JA sequenceDecs_decodeSync_amd64_of_update_zero
|
||||
CMPQ CX, $0x40
|
||||
JAE sequenceDecs_decodeSync_amd64_of_update_zero
|
||||
NEGQ CX
|
||||
SHRQ CL, R14
|
||||
ADDQ R14, AX
|
||||
|
||||
sequenceDecs_decodeSync_amd64_of_update_zero:
|
||||
MOVQ AX, 8(SP)
|
||||
|
||||
// Update match length
|
||||
MOVQ R8, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R14
|
||||
SHLQ CL, R14
|
||||
MOVB AH, CL
|
||||
ADDQ CX, BX
|
||||
NEGL CX
|
||||
SHRQ CL, R14
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
CMOVQEQ CX, R14
|
||||
ADDQ R14, AX
|
||||
MOVQ AX, 16(SP)
|
||||
MOVQ R8, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R14
|
||||
SHLQ CL, R14
|
||||
MOVB AH, CL
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
JZ sequenceDecs_decodeSync_amd64_ml_update_zero
|
||||
ADDQ CX, BX
|
||||
CMPQ BX, $0x40
|
||||
JA sequenceDecs_decodeSync_amd64_ml_update_zero
|
||||
CMPQ CX, $0x40
|
||||
JAE sequenceDecs_decodeSync_amd64_ml_update_zero
|
||||
NEGQ CX
|
||||
SHRQ CL, R14
|
||||
ADDQ R14, AX
|
||||
|
||||
sequenceDecs_decodeSync_amd64_ml_update_zero:
|
||||
MOVQ AX, 16(SP)
|
||||
|
||||
// Fill bitreader to have enough for the remaining
|
||||
CMPQ SI, $0x08
|
||||
@ -1853,19 +1884,25 @@ sequenceDecs_decodeSync_amd64_fill_2_byte_by_byte:
|
||||
|
||||
sequenceDecs_decodeSync_amd64_fill_2_end:
|
||||
// Update literal length
|
||||
MOVQ DI, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R14
|
||||
SHLQ CL, R14
|
||||
MOVB AH, CL
|
||||
ADDQ CX, BX
|
||||
NEGL CX
|
||||
SHRQ CL, R14
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
CMOVQEQ CX, R14
|
||||
ADDQ R14, AX
|
||||
MOVQ AX, 24(SP)
|
||||
MOVQ DI, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R14
|
||||
SHLQ CL, R14
|
||||
MOVB AH, CL
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
JZ sequenceDecs_decodeSync_amd64_ll_update_zero
|
||||
ADDQ CX, BX
|
||||
CMPQ BX, $0x40
|
||||
JA sequenceDecs_decodeSync_amd64_ll_update_zero
|
||||
CMPQ CX, $0x40
|
||||
JAE sequenceDecs_decodeSync_amd64_ll_update_zero
|
||||
NEGQ CX
|
||||
SHRQ CL, R14
|
||||
ADDQ R14, AX
|
||||
|
||||
sequenceDecs_decodeSync_amd64_ll_update_zero:
|
||||
MOVQ AX, 24(SP)
|
||||
|
||||
// Fill bitreader for state updates
|
||||
MOVQ R13, (SP)
|
||||
@ -1945,7 +1982,7 @@ sequenceDecs_decodeSync_amd64_skip_update:
|
||||
MOVUPS 144(CX), X0
|
||||
MOVQ R13, 144(CX)
|
||||
MOVUPS X0, 152(CX)
|
||||
JMP sequenceDecs_decodeSync_amd64_adjust_end
|
||||
JMP sequenceDecs_decodeSync_amd64_after_adjust
|
||||
|
||||
sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0:
|
||||
CMPQ 24(SP), $0x00000000
|
||||
@ -1957,7 +1994,7 @@ sequenceDecs_decodeSync_amd64_adjust_offset_maybezero:
|
||||
TESTQ R13, R13
|
||||
JNZ sequenceDecs_decodeSync_amd64_adjust_offset_nonzero
|
||||
MOVQ 144(CX), R13
|
||||
JMP sequenceDecs_decodeSync_amd64_adjust_end
|
||||
JMP sequenceDecs_decodeSync_amd64_after_adjust
|
||||
|
||||
sequenceDecs_decodeSync_amd64_adjust_offset_nonzero:
|
||||
MOVQ R13, AX
|
||||
@ -1966,8 +2003,7 @@ sequenceDecs_decodeSync_amd64_adjust_offset_nonzero:
|
||||
CMPQ R13, $0x03
|
||||
CMOVQEQ R14, AX
|
||||
CMOVQEQ R15, R14
|
||||
LEAQ 144(CX), R15
|
||||
ADDQ (R15)(AX*8), R14
|
||||
ADDQ 144(CX)(AX*8), R14
|
||||
JNZ sequenceDecs_decodeSync_amd64_adjust_temp_valid
|
||||
MOVQ $0x00000001, R14
|
||||
|
||||
@ -1983,7 +2019,7 @@ sequenceDecs_decodeSync_amd64_adjust_skip:
|
||||
MOVQ R14, 144(CX)
|
||||
MOVQ R14, R13
|
||||
|
||||
sequenceDecs_decodeSync_amd64_adjust_end:
|
||||
sequenceDecs_decodeSync_amd64_after_adjust:
|
||||
MOVQ R13, 8(SP)
|
||||
|
||||
// Check values
|
||||
@ -2280,6 +2316,10 @@ TEXT ·sequenceDecs_decodeSync_bmi2(SB), $64-32
|
||||
MOVQ 72(CX), SI
|
||||
MOVQ 80(CX), DI
|
||||
MOVQ 88(CX), R8
|
||||
XORQ R9, R9
|
||||
MOVQ R9, 8(SP)
|
||||
MOVQ R9, 16(SP)
|
||||
MOVQ R9, 24(SP)
|
||||
MOVQ 112(CX), R9
|
||||
MOVQ 128(CX), R10
|
||||
MOVQ R10, 32(SP)
|
||||
@ -2452,7 +2492,7 @@ sequenceDecs_decodeSync_bmi2_skip_update:
|
||||
MOVUPS 144(CX), X0
|
||||
MOVQ R13, 144(CX)
|
||||
MOVUPS X0, 152(CX)
|
||||
JMP sequenceDecs_decodeSync_bmi2_adjust_end
|
||||
JMP sequenceDecs_decodeSync_bmi2_after_adjust
|
||||
|
||||
sequenceDecs_decodeSync_bmi2_adjust_offsetB_1_or_0:
|
||||
CMPQ 24(SP), $0x00000000
|
||||
@ -2464,7 +2504,7 @@ sequenceDecs_decodeSync_bmi2_adjust_offset_maybezero:
|
||||
TESTQ R13, R13
|
||||
JNZ sequenceDecs_decodeSync_bmi2_adjust_offset_nonzero
|
||||
MOVQ 144(CX), R13
|
||||
JMP sequenceDecs_decodeSync_bmi2_adjust_end
|
||||
JMP sequenceDecs_decodeSync_bmi2_after_adjust
|
||||
|
||||
sequenceDecs_decodeSync_bmi2_adjust_offset_nonzero:
|
||||
MOVQ R13, R12
|
||||
@ -2473,8 +2513,7 @@ sequenceDecs_decodeSync_bmi2_adjust_offset_nonzero:
|
||||
CMPQ R13, $0x03
|
||||
CMOVQEQ R14, R12
|
||||
CMOVQEQ R15, R14
|
||||
LEAQ 144(CX), R15
|
||||
ADDQ (R15)(R12*8), R14
|
||||
ADDQ 144(CX)(R12*8), R14
|
||||
JNZ sequenceDecs_decodeSync_bmi2_adjust_temp_valid
|
||||
MOVQ $0x00000001, R14
|
||||
|
||||
@ -2490,7 +2529,7 @@ sequenceDecs_decodeSync_bmi2_adjust_skip:
|
||||
MOVQ R14, 144(CX)
|
||||
MOVQ R14, R13
|
||||
|
||||
sequenceDecs_decodeSync_bmi2_adjust_end:
|
||||
sequenceDecs_decodeSync_bmi2_after_adjust:
|
||||
MOVQ R13, 8(SP)
|
||||
|
||||
// Check values
|
||||
@ -2787,6 +2826,10 @@ TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $64-32
|
||||
MOVQ 72(AX), DI
|
||||
MOVQ 80(AX), R8
|
||||
MOVQ 88(AX), R9
|
||||
XORQ CX, CX
|
||||
MOVQ CX, 8(SP)
|
||||
MOVQ CX, 16(SP)
|
||||
MOVQ CX, 24(SP)
|
||||
MOVQ 112(AX), R10
|
||||
MOVQ 128(AX), CX
|
||||
MOVQ CX, 32(SP)
|
||||
@ -2836,34 +2879,46 @@ sequenceDecs_decodeSync_safe_amd64_fill_byte_by_byte:
|
||||
|
||||
sequenceDecs_decodeSync_safe_amd64_fill_end:
|
||||
// Update offset
|
||||
MOVQ R9, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R14
|
||||
SHLQ CL, R14
|
||||
MOVB AH, CL
|
||||
ADDQ CX, BX
|
||||
NEGL CX
|
||||
SHRQ CL, R14
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
CMOVQEQ CX, R14
|
||||
ADDQ R14, AX
|
||||
MOVQ AX, 8(SP)
|
||||
MOVQ R9, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R14
|
||||
SHLQ CL, R14
|
||||
MOVB AH, CL
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
JZ sequenceDecs_decodeSync_safe_amd64_of_update_zero
|
||||
ADDQ CX, BX
|
||||
CMPQ BX, $0x40
|
||||
JA sequenceDecs_decodeSync_safe_amd64_of_update_zero
|
||||
CMPQ CX, $0x40
|
||||
JAE sequenceDecs_decodeSync_safe_amd64_of_update_zero
|
||||
NEGQ CX
|
||||
SHRQ CL, R14
|
||||
ADDQ R14, AX
|
||||
|
||||
sequenceDecs_decodeSync_safe_amd64_of_update_zero:
|
||||
MOVQ AX, 8(SP)
|
||||
|
||||
// Update match length
|
||||
MOVQ R8, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R14
|
||||
SHLQ CL, R14
|
||||
MOVB AH, CL
|
||||
ADDQ CX, BX
|
||||
NEGL CX
|
||||
SHRQ CL, R14
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
CMOVQEQ CX, R14
|
||||
ADDQ R14, AX
|
||||
MOVQ AX, 16(SP)
|
||||
MOVQ R8, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R14
|
||||
SHLQ CL, R14
|
||||
MOVB AH, CL
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
JZ sequenceDecs_decodeSync_safe_amd64_ml_update_zero
|
||||
ADDQ CX, BX
|
||||
CMPQ BX, $0x40
|
||||
JA sequenceDecs_decodeSync_safe_amd64_ml_update_zero
|
||||
CMPQ CX, $0x40
|
||||
JAE sequenceDecs_decodeSync_safe_amd64_ml_update_zero
|
||||
NEGQ CX
|
||||
SHRQ CL, R14
|
||||
ADDQ R14, AX
|
||||
|
||||
sequenceDecs_decodeSync_safe_amd64_ml_update_zero:
|
||||
MOVQ AX, 16(SP)
|
||||
|
||||
// Fill bitreader to have enough for the remaining
|
||||
CMPQ SI, $0x08
|
||||
@ -2891,19 +2946,25 @@ sequenceDecs_decodeSync_safe_amd64_fill_2_byte_by_byte:
|
||||
|
||||
sequenceDecs_decodeSync_safe_amd64_fill_2_end:
|
||||
// Update literal length
|
||||
MOVQ DI, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R14
|
||||
SHLQ CL, R14
|
||||
MOVB AH, CL
|
||||
ADDQ CX, BX
|
||||
NEGL CX
|
||||
SHRQ CL, R14
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
CMOVQEQ CX, R14
|
||||
ADDQ R14, AX
|
||||
MOVQ AX, 24(SP)
|
||||
MOVQ DI, AX
|
||||
MOVQ BX, CX
|
||||
MOVQ DX, R14
|
||||
SHLQ CL, R14
|
||||
MOVB AH, CL
|
||||
SHRQ $0x20, AX
|
||||
TESTQ CX, CX
|
||||
JZ sequenceDecs_decodeSync_safe_amd64_ll_update_zero
|
||||
ADDQ CX, BX
|
||||
CMPQ BX, $0x40
|
||||
JA sequenceDecs_decodeSync_safe_amd64_ll_update_zero
|
||||
CMPQ CX, $0x40
|
||||
JAE sequenceDecs_decodeSync_safe_amd64_ll_update_zero
|
||||
NEGQ CX
|
||||
SHRQ CL, R14
|
||||
ADDQ R14, AX
|
||||
|
||||
sequenceDecs_decodeSync_safe_amd64_ll_update_zero:
|
||||
MOVQ AX, 24(SP)
|
||||
|
||||
// Fill bitreader for state updates
|
||||
MOVQ R13, (SP)
|
||||
@ -2983,7 +3044,7 @@ sequenceDecs_decodeSync_safe_amd64_skip_update:
|
||||
MOVUPS 144(CX), X0
|
||||
MOVQ R13, 144(CX)
|
||||
MOVUPS X0, 152(CX)
|
||||
JMP sequenceDecs_decodeSync_safe_amd64_adjust_end
|
||||
JMP sequenceDecs_decodeSync_safe_amd64_after_adjust
|
||||
|
||||
sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0:
|
||||
CMPQ 24(SP), $0x00000000
|
||||
@ -2995,7 +3056,7 @@ sequenceDecs_decodeSync_safe_amd64_adjust_offset_maybezero:
|
||||
TESTQ R13, R13
|
||||
JNZ sequenceDecs_decodeSync_safe_amd64_adjust_offset_nonzero
|
||||
MOVQ 144(CX), R13
|
||||
JMP sequenceDecs_decodeSync_safe_amd64_adjust_end
|
||||
JMP sequenceDecs_decodeSync_safe_amd64_after_adjust
|
||||
|
||||
sequenceDecs_decodeSync_safe_amd64_adjust_offset_nonzero:
|
||||
MOVQ R13, AX
|
||||
@ -3004,8 +3065,7 @@ sequenceDecs_decodeSync_safe_amd64_adjust_offset_nonzero:
|
||||
CMPQ R13, $0x03
|
||||
CMOVQEQ R14, AX
|
||||
CMOVQEQ R15, R14
|
||||
LEAQ 144(CX), R15
|
||||
ADDQ (R15)(AX*8), R14
|
||||
ADDQ 144(CX)(AX*8), R14
|
||||
JNZ sequenceDecs_decodeSync_safe_amd64_adjust_temp_valid
|
||||
MOVQ $0x00000001, R14
|
||||
|
||||
@ -3021,7 +3081,7 @@ sequenceDecs_decodeSync_safe_amd64_adjust_skip:
|
||||
MOVQ R14, 144(CX)
|
||||
MOVQ R14, R13
|
||||
|
||||
sequenceDecs_decodeSync_safe_amd64_adjust_end:
|
||||
sequenceDecs_decodeSync_safe_amd64_after_adjust:
|
||||
MOVQ R13, 8(SP)
|
||||
|
||||
// Check values
|
||||
@ -3420,6 +3480,10 @@ TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $64-32
|
||||
MOVQ 72(CX), SI
|
||||
MOVQ 80(CX), DI
|
||||
MOVQ 88(CX), R8
|
||||
XORQ R9, R9
|
||||
MOVQ R9, 8(SP)
|
||||
MOVQ R9, 16(SP)
|
||||
MOVQ R9, 24(SP)
|
||||
MOVQ 112(CX), R9
|
||||
MOVQ 128(CX), R10
|
||||
MOVQ R10, 32(SP)
|
||||
@ -3592,7 +3656,7 @@ sequenceDecs_decodeSync_safe_bmi2_skip_update:
|
||||
MOVUPS 144(CX), X0
|
||||
MOVQ R13, 144(CX)
|
||||
MOVUPS X0, 152(CX)
|
||||
JMP sequenceDecs_decodeSync_safe_bmi2_adjust_end
|
||||
JMP sequenceDecs_decodeSync_safe_bmi2_after_adjust
|
||||
|
||||
sequenceDecs_decodeSync_safe_bmi2_adjust_offsetB_1_or_0:
|
||||
CMPQ 24(SP), $0x00000000
|
||||
@ -3604,7 +3668,7 @@ sequenceDecs_decodeSync_safe_bmi2_adjust_offset_maybezero:
|
||||
TESTQ R13, R13
|
||||
JNZ sequenceDecs_decodeSync_safe_bmi2_adjust_offset_nonzero
|
||||
MOVQ 144(CX), R13
|
||||
JMP sequenceDecs_decodeSync_safe_bmi2_adjust_end
|
||||
JMP sequenceDecs_decodeSync_safe_bmi2_after_adjust
|
||||
|
||||
sequenceDecs_decodeSync_safe_bmi2_adjust_offset_nonzero:
|
||||
MOVQ R13, R12
|
||||
@ -3613,8 +3677,7 @@ sequenceDecs_decodeSync_safe_bmi2_adjust_offset_nonzero:
|
||||
CMPQ R13, $0x03
|
||||
CMOVQEQ R14, R12
|
||||
CMOVQEQ R15, R14
|
||||
LEAQ 144(CX), R15
|
||||
ADDQ (R15)(R12*8), R14
|
||||
ADDQ 144(CX)(R12*8), R14
|
||||
JNZ sequenceDecs_decodeSync_safe_bmi2_adjust_temp_valid
|
||||
MOVQ $0x00000001, R14
|
||||
|
||||
@ -3630,7 +3693,7 @@ sequenceDecs_decodeSync_safe_bmi2_adjust_skip:
|
||||
MOVQ R14, 144(CX)
|
||||
MOVQ R14, R13
|
||||
|
||||
sequenceDecs_decodeSync_safe_bmi2_adjust_end:
|
||||
sequenceDecs_decodeSync_safe_bmi2_after_adjust:
|
||||
MOVQ R13, 8(SP)
|
||||
|
||||
// Check values
|
||||
|
||||
4
vendor/github.com/klauspost/compress/zstd/seqdec_generic.go
generated
vendored
4
vendor/github.com/klauspost/compress/zstd/seqdec_generic.go
generated
vendored
@ -111,7 +111,7 @@ func (s *sequenceDecs) decode(seqs []seqVals) error {
|
||||
}
|
||||
s.seqSize += ll + ml
|
||||
if s.seqSize > maxBlockSize {
|
||||
return fmt.Errorf("output (%d) bigger than max block size (%d)", s.seqSize, maxBlockSize)
|
||||
return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize)
|
||||
}
|
||||
litRemain -= ll
|
||||
if litRemain < 0 {
|
||||
@ -149,7 +149,7 @@ func (s *sequenceDecs) decode(seqs []seqVals) error {
|
||||
}
|
||||
s.seqSize += litRemain
|
||||
if s.seqSize > maxBlockSize {
|
||||
return fmt.Errorf("output (%d) bigger than max block size (%d)", s.seqSize, maxBlockSize)
|
||||
return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize)
|
||||
}
|
||||
err := br.close()
|
||||
if err != nil {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user