fix(s3): preserve trailing-slash objects

internal/db/db.go

@@ -25,7 +25,10 @@ func Init(d *gorm.DB) {
 	if err := migrateWebDAVProperties(); err != nil {
 		log.Fatalf("failed migrate webdav_properties: %s", err.Error())
 	}
-	err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.SharingDB), new(model.WebDAVProperty), new(model.MediaItem), new(model.MediaConfig), new(model.MediaScanPath), new(model.MediaAppKey), new(model.MediaUserData), new(model.MediaPlaylist), new(model.MediaPlaylistItem), new(model.EmbyDevice), new(model.MediaAnnotation))
+	if err := migrateS3Markers(); err != nil {
+		log.Fatalf("failed migrate s3_markers: %s", err.Error())
+	}
+	err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.SharingDB), new(model.WebDAVProperty), new(model.MediaItem), new(model.MediaConfig), new(model.MediaScanPath), new(model.MediaAppKey), new(model.MediaUserData), new(model.MediaPlaylist), new(model.MediaPlaylistItem), new(model.EmbyDevice), new(model.MediaAnnotation), new(model.S3Marker))
 	if err != nil {
 		// 典型场景：未确认清库时，旧 media_items 数据违反新建的唯一索引。
 		// 这里把解决办法直接写进日志，避免用户只看到一句 failed migrate database。

internal/db/s3_marker_migrate.go

@@ -0,0 +1,58 @@
+package db
+
+import (
+	"fmt"
+
+	"github.com/OpenListTeam/OpenList/v4/internal/model"
+	"gorm.io/gorm"
+)
+
+// migrateS3Markers upgrades the early key-only hashes before adding the unique
+// bucket-aware hash index.
+func migrateS3Markers() error {
+	stmt := &gorm.Statement{DB: db}
+	if err := stmt.Parse(&model.S3Marker{}); err != nil {
+		return err
+	}
+	table := stmt.Schema.Table
+	m := db.Migrator()
+	if !m.HasTable(table) {
+		return nil
+	}
+
+	var rows []model.S3Marker
+	if err := db.Model(&model.S3Marker{}).Select("id, bucket, key, key_hash").Order("id").Find(&rows).Error; err != nil {
+		return err
+	}
+	identities := make(map[string]uint, len(rows))
+	hashes := make(map[string]string, len(rows))
+	for _, row := range rows {
+		identity := row.Bucket + "\x00" + row.Key
+		hash := model.HashS3MarkerKey(row.Bucket, row.Key)
+		if previousIdentity, ok := hashes[hash]; ok && previousIdentity != identity {
+			return fmt.Errorf("s3 marker hash collision between %q and %q", previousIdentity, identity)
+		}
+		hashes[hash] = identity
+	}
+	for _, row := range rows {
+		identity := row.Bucket + "\x00" + row.Key
+		if previousID, ok := identities[identity]; ok {
+			if err := db.Delete(&model.S3Marker{}, previousID).Error; err != nil {
+				return err
+			}
+		}
+		identities[identity] = row.ID
+		hash := model.HashS3MarkerKey(row.Bucket, row.Key)
+		if row.KeyHash != hash {
+			if err := db.Model(&model.S3Marker{}).Where("id = ?", row.ID).UpdateColumn("key_hash", hash).Error; err != nil {
+				return err
+			}
+		}
+	}
+
+	const oldIndex = "idx_s3_marker_bucket_key_hash"
+	if m.HasIndex(table, oldIndex) {
+		return m.DropIndex(table, oldIndex)
+	}
+	return nil
+}

internal/db/s3_marker_migrate_test.go

@@ -0,0 +1,72 @@
+package db
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"testing"
+
+	"github.com/OpenListTeam/OpenList/v4/internal/model"
+)
+
+func TestMigrateS3MarkersBackfillsHashesAndRebuildsIdentity(t *testing.T) {
+	d := openTestDB(t, "ol_")
+	db = d
+	if err := d.Exec(`CREATE TABLE ol_s3_markers (
+		id integer primary key autoincrement,
+		bucket text,
+		key text,
+		key_hash text,
+		metadata text,
+		modified datetime
+	)`).Error; err != nil {
+		t.Fatal(err)
+	}
+	if err := d.Exec(`CREATE INDEX idx_s3_marker_bucket_key_hash ON ol_s3_markers (bucket, key_hash)`).Error; err != nil {
+		t.Fatal(err)
+	}
+	insert := func(bucket, key, metadata string) {
+		t.Helper()
+		hash := sha256.Sum256([]byte(key))
+		if err := d.Exec("INSERT INTO ol_s3_markers (bucket, key, key_hash, metadata) VALUES (?, ?, ?, ?)", bucket, key, hex.EncodeToString(hash[:]), metadata).Error; err != nil {
+			t.Fatal(err)
+		}
+	}
+	insert("first", "shared/", "old")
+	insert("second", "shared/", "other bucket")
+	insert("first", "shared/", "latest")
+
+	if err := migrateS3Markers(); err != nil {
+		t.Fatal(err)
+	}
+	if err := d.AutoMigrate(&model.S3Marker{}); err != nil {
+		t.Fatal(err)
+	}
+	if d.Migrator().HasIndex("ol_s3_markers", "idx_s3_marker_bucket_key_hash") {
+		t.Fatal("legacy composite index was not removed")
+	}
+	if !d.Migrator().HasIndex("ol_s3_markers", "idx_s3_marker_key_hash") {
+		t.Fatal("bucket-aware unique hash index was not created")
+	}
+	var rows []model.S3Marker
+	if err := d.Order("bucket").Find(&rows).Error; err != nil {
+		t.Fatal(err)
+	}
+	if len(rows) != 2 {
+		t.Fatalf("migrated rows = %d, want one row per bucket/key pair", len(rows))
+	}
+	for _, row := range rows {
+		if row.KeyHash != model.HashS3MarkerKey(row.Bucket, row.Key) {
+			t.Errorf("migrated hash for %s/%s = %q, want bucket-aware hash", row.Bucket, row.Key, row.KeyHash)
+		}
+	}
+	if rows[0].Metadata != "latest" {
+		t.Fatalf("duplicate marker metadata = %q, want latest", rows[0].Metadata)
+	}
+	duplicate := model.S3Marker{Bucket: "first", Key: "shared/", KeyHash: model.HashS3MarkerKey("first", "shared/")}
+	if err := d.Create(&duplicate).Error; err == nil {
+		t.Fatal("migrated index accepted a duplicate bucket/key pair")
+	}
+	if err := migrateS3Markers(); err != nil {
+		t.Fatalf("repeat migration: %v", err)
+	}
+}

internal/model/s3.go

@@ -0,0 +1,23 @@
+package model
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"time"
+)
+
+// S3Marker stores a trailing-slash S3 object without collapsing it into a directory path.
+type S3Marker struct {
+	ID       uint   `gorm:"primaryKey"`
+	Bucket   string `gorm:"size:191;not null;index:idx_s3_marker_bucket"`
+	Key      string `gorm:"type:text;not null"`
+	KeyHash  string `gorm:"size:64;not null;uniqueIndex:idx_s3_marker_key_hash"`
+	Metadata string `gorm:"type:text"`
+	Modified time.Time
+}
+
+// HashS3MarkerKey returns the indexed identity of an exact bucket/key pair.
+func HashS3MarkerKey(bucket, key string) string {
+	hash := sha256.Sum256([]byte(bucket + "\x00" + key))
+	return hex.EncodeToString(hash[:])
+}

server/s3/backend.go

@@ -5,6 +5,7 @@ package s3
 import (
 	"context"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
 	"path"
@@ -33,6 +34,8 @@ var (
 	timeFormat  = "Mon, 2 Jan 2006 15:04:05 GMT"
 )
 
+const emptyObjectETag = `"d41d8cd98f00b204e9800998ecf8427e"`
+
 // s3Backend implements the gofakes3.Backend interface to make an S3
 // backend for gofakes3. It also implements gofakes3.MultipartBackend so that
 // multipart uploads are streamed to local temp files part-by-part and
@@ -94,7 +97,11 @@ func (b *s3Backend) ListBucket(ctx context.Context, bucketName string, prefix *g
 
 	path, remaining := prefixParser(prefix)
 
-	response, err := b.listPage(ctx, bucketPath, path, remaining, prefix.HasDelimiter, page)
+	markers, err := listS3Markers(bucketName)
+	if err != nil {
+		return nil, err
+	}
+	response, err := b.listPageWithMarkers(ctx, bucketPath, path, remaining, prefix.HasDelimiter, page, markers)
 	if err == gofakes3.ErrNoSuchKey {
 		// AWS just returns an empty list
 		response = gofakes3.NewObjectList()
@@ -115,6 +122,22 @@ func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName strin
 	}
 	bucketPath := bucket.Path
 
+	if marker, markerErr := getS3MarkerForRequest(ctx, bucketName, objectName); markerErr != nil {
+		return nil, markerErr
+	} else if marker != nil {
+		metadata := s3MarkerMetadata(marker, marker.Key)
+		return &gofakes3.Object{
+			Name:     marker.Key,
+			Metadata: metadata,
+			Size:     0,
+			Contents: noOpReadCloser{},
+		}, nil
+	}
+
+	if isExactTrailingSlashRequest(ctx, objectName) {
+		return nil, gofakes3.KeyNotFound(objectName)
+	}
+
 	fp := path.Join(bucketPath, objectName)
 	fmeta, _ := op.GetNearestMeta(fp)
 	node, err := fs.Get(context.WithValue(ctx, conf.MetaKey, fmeta), fp, &fs.GetArgs{})
@@ -160,6 +183,25 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string
 	}
 	bucketPath := bucket.Path
 
+	if marker, markerErr := getS3MarkerForRequest(ctx, bucketName, objectName); markerErr != nil {
+		return nil, markerErr
+	} else if marker != nil {
+		if rangeRequest != nil {
+			return nil, gofakes3.ErrInvalidRange
+		}
+		metadata := s3MarkerMetadata(marker, marker.Key)
+		return &gofakes3.Object{
+			Name:     marker.Key,
+			Metadata: metadata,
+			Size:     0,
+			Contents: io.NopCloser(strings.NewReader("")),
+		}, nil
+	}
+
+	if isExactTrailingSlashRequest(ctx, objectName) {
+		return nil, gofakes3.KeyNotFound(objectName)
+	}
+
 	fp := path.Join(bucketPath, objectName)
 	fmeta, _ := op.GetNearestMeta(fp)
 	node, err := fs.Get(context.WithValue(ctx, conf.MetaKey, fmeta), fp, &fs.GetArgs{})
@@ -259,22 +301,41 @@ func (b *s3Backend) putStream(
 	}
 	bucketPath := bucket.Path
 
-	isDir := strings.HasSuffix(objectName, "/")
-	log.Debugf("isDir: %v", isDir)
+	if strings.HasSuffix(objectName, "/") {
+		if size != 0 {
+			return gofakes3.ErrorMessagef(gofakes3.ErrInvalidArgument, "trailing-slash objects must be empty")
+		}
+		if input != nil {
+			var extra [1]byte
+			n, readErr := io.ReadFull(input, extra[:])
+			if n != 0 {
+				return gofakes3.ErrorMessagef(gofakes3.ErrInvalidArgument, "trailing-slash objects must be empty")
+			}
+			if readErr != io.EOF {
+				return readErr
+			}
+		}
+		encodedMeta, err := json.Marshal(meta)
+		if err != nil {
+			return err
+		}
+		return saveS3Marker(&model.S3Marker{
+			Bucket:   bucketName,
+			Key:      objectName,
+			KeyHash:  s3KeyHash(bucketName, objectName),
+			Metadata: string(encodedMeta),
+			Modified: time.Now(),
+		})
+	}
 
 	fp := path.Join(bucketPath, objectName)
 	log.Debugf("fp: %s, bucketPath: %s, objectName: %s", fp, bucketPath, objectName)
 
 	var reqPath string
-	if isDir {
-		reqPath = fp + "/"
-	} else {
-		reqPath = path.Dir(fp)
-	}
+	reqPath = path.Dir(fp)
 	log.Debugf("reqPath: %s", reqPath)
 	fmeta, _ := op.GetNearestMeta(fp)
 	ctx = context.WithValue(ctx, conf.MetaKey, fmeta)
-
 	_, err = fs.Get(ctx, reqPath, &fs.GetArgs{})
 	if err != nil {
 		if errs.IsObjectNotFound(err) && strings.Contains(objectName, "/") {
@@ -288,10 +349,6 @@ func (b *s3Backend) putStream(
 		}
 	}
 
-	if isDir {
-		return nil
-	}
-
 	var ti time.Time
 
 	if val, ok := meta["X-Amz-Meta-Mtime"]; ok {
@@ -367,6 +424,14 @@ func (b *s3Backend) deleteObject(ctx context.Context, bucketName, objectName str
 	if err != nil {
 		return err
 	}
+	if marker, markerErr := getS3MarkerForRequest(ctx, bucketName, objectName); markerErr != nil {
+		return markerErr
+	} else if marker != nil {
+		return deleteS3Marker(marker.Bucket, marker.Key)
+	}
+	if isExactTrailingSlashRequest(ctx, objectName) {
+		return nil
+	}
 	bucketPath := bucket.Path
 
 	fp := path.Join(bucketPath, objectName)
@@ -418,6 +483,31 @@ func (b *s3Backend) CopyObject(ctx context.Context, srcBucket, srcKey, dstBucket
 	}
 	srcBucketPath := srcB.Path
 
+	if marker, markerErr := getS3Marker(srcBucket, srcKey); markerErr != nil {
+		return result, markerErr
+	} else if marker != nil {
+		c, getErr := b.GetObject(ctx, srcBucket, srcKey, nil)
+		if getErr != nil {
+			return result, getErr
+		}
+		defer func() { _ = c.Contents.Close() }()
+		if meta == nil {
+			meta = make(map[string]string)
+		}
+		for k, v := range c.Metadata {
+			if _, found := meta[k]; !found && k != "X-Amz-Acl" {
+				meta[k] = v
+			}
+		}
+		if _, putErr := b.PutObject(ctx, dstBucket, dstKey, meta, c.Contents, c.Size); putErr != nil {
+			return result, putErr
+		}
+		return gofakes3.CopyObjectResult{
+			ETag:         emptyObjectETag,
+			LastModified: gofakes3.NewContentTime(marker.Modified),
+		}, nil
+	}
+
 	srcFp := path.Join(srcBucketPath, srcKey)
 	fmeta, _ := op.GetNearestMeta(srcFp)
 	srcNode, err := fs.Get(context.WithValue(ctx, conf.MetaKey, fmeta), srcFp, &fs.GetArgs{})