From c10e6e817dca365dfd3fad226cc6f52543fc2337 Mon Sep 17 00:00:00 2001 From: Matios102 Date: Wed, 17 Dec 2025 14:26:10 +0100 Subject: [PATCH 01/11] feat: implement file caching mechanism and update packager to utilize cache --- cmd/main.go | 5 +- docker-compose.yaml | 2 + go.mod | 4 + go.sum | 9 ++ internal/stages/packager/packager.go | 57 +++++-- internal/storage/cache.go | 214 +++++++++++++++++++++++++++ pkg/constants/constants.go | 7 + pkg/messages/messages.go | 9 +- 8 files changed, 292 insertions(+), 15 deletions(-) create mode 100644 internal/storage/cache.go diff --git a/cmd/main.go b/cmd/main.go index 39c33ef..0f01ed0 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -42,9 +42,10 @@ func main() { workerChannel := rabbitmq.NewRabbitMQChannel(conn) // Initialize the services - storage := storage.NewStorage(config.StorageBaseUrl) + storageService := storage.NewStorage(config.StorageBaseUrl) + fileCache := storage.NewFileCache() compiler := compiler.NewCompiler() - packager := packager.NewPackager(storage) + packager := packager.NewPackager(storageService, fileCache) executor := executor.NewExecutor(dCli) verifier := verifier.NewVerifier(config.VerifierFlags) responder := responder.NewResponder(workerChannel, config.PublishChanSize) diff --git a/docker-compose.yaml b/docker-compose.yaml index 03f377a..3f8479a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -15,6 +15,7 @@ services: container_name: go_app volumes: - /var/run/docker.sock:/var/run/docker.sock + - worker_cache:/tmp/worker-cache environment: - RABBITMQ_HOST=rabbitmq - RABBITMQ_USER=guest @@ -32,3 +33,4 @@ services: volumes: rabbitmq_data: + worker_cache: diff --git a/go.mod b/go.mod index 655e931..f27e047 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/docker/docker v28.1.1+incompatible github.com/joho/godotenv v1.5.1 github.com/rabbitmq/amqp091-go v1.10.0 + github.com/stretchr/testify v1.10.0 go.uber.org/mock v0.6.0 go.uber.org/zap v1.27.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 @@ -16,6 +17,7 @@ require ( require ( github.com/Microsoft/go-winio v0.4.14 // indirect github.com/containerd/log v0.1.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -30,6 +32,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect go.opentelemetry.io/otel v1.35.0 // indirect @@ -39,5 +42,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/sys v0.31.0 // indirect golang.org/x/time v0.11.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.5.2 // indirect ) diff --git a/go.sum b/go.sum index 50c7602..f37bbea 100644 --- a/go.sum +++ b/go.sum @@ -36,6 +36,10 @@ github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwA github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= @@ -57,6 +61,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -137,6 +143,9 @@ google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/stages/packager/packager.go b/internal/stages/packager/packager.go index 5f78348..6b23e1c 100644 --- a/internal/stages/packager/packager.go +++ b/internal/stages/packager/packager.go @@ -28,8 +28,9 @@ type Packager interface { } type packager struct { - logger *zap.SugaredLogger - storage storage.Storage + logger *zap.SugaredLogger + storage storage.Storage + fileCache storage.FileCache } type TaskDirConfig struct { @@ -46,11 +47,17 @@ type TaskDirConfig struct { CompileErrFilePath string } -func NewPackager(storage storage.Storage) Packager { +func NewPackager(storageService storage.Storage, fileCache storage.FileCache) Packager { logger := logger.NewNamedLogger("packager") + if fileCache != nil { + if err := fileCache.InitCache(); err != nil { + logger.Warnf("Failed to initialize file cache: %v", err) + } + } return &packager{ - logger: logger, - storage: storage, + logger: logger, + storage: storageService, + fileCache: fileCache, } } @@ -82,7 +89,7 @@ func (p *packager) PrepareSolutionPackage( // Download test cases and create user files. for idx, tc := range taskQueueMessage.TestCases { - if err := p.prepareTestCaseFiles(basePath, idx, tc); err != nil { + if err := p.prepareTestCaseFiles(basePath, idx, tc, taskQueueMessage.TaskFilesVersion); err != nil { p.logger.Errorf("Failed to prepare test case files: %s", err) _ = utils.RemoveIO(basePath, true, true) return nil, err @@ -152,13 +159,13 @@ func (p *packager) downloadSubmission(basePath string, submission messages.FileL } // prepareTestCaseFiles downloads input and expected output for a test case and creates user files. -func (p *packager) prepareTestCaseFiles(basePath string, idx int, tc messages.TestCase) error { +func (p *packager) prepareTestCaseFiles(basePath string, idx int, tc messages.TestCase, taskVersion string) error { // inputs if tc.InputFile.Bucket == "" || tc.InputFile.Path == "" { p.logger.Warnf("Test case %d input location is empty, skipping", idx) } else { inputDest := filepath.Join(basePath, constants.InputDirName, filepath.Base(tc.InputFile.Path)) - if _, err := p.storage.DownloadFile(tc.InputFile, inputDest); err != nil { + if err := p.downloadOrCopyFromCache(tc.InputFile, inputDest, taskVersion); err != nil { return err } } @@ -168,7 +175,7 @@ func (p *packager) prepareTestCaseFiles(basePath string, idx int, tc messages.Te p.logger.Warnf("Test case %d expected output location is empty, skipping", idx) } else { outputDest := filepath.Join(basePath, constants.OutputDirName, filepath.Base(tc.ExpectedOutput.Path)) - if _, err := p.storage.DownloadFile(tc.ExpectedOutput, outputDest); err != nil { + if err := p.downloadOrCopyFromCache(tc.ExpectedOutput, outputDest, taskVersion); err != nil { return err } } @@ -267,6 +274,38 @@ func (p *packager) SendSolutionPackage( return nil } +func (p *packager) downloadOrCopyFromCache(fileLocation messages.FileLocation, destPath string, taskVersion string) error { + // Try to get from cache + if p.fileCache != nil && taskVersion != "" { + cachedPath, isCached, err := p.fileCache.GetCachedFile(fileLocation, taskVersion) + if err != nil { + p.logger.Warnf("Error checking cache for %s: %v", fileLocation.Path, err) + } else if isCached { + // Copy from cache to destination + if err := utils.CopyFile(cachedPath, destPath); err != nil { + p.logger.Warnf("Failed to copy from cache, will download: %v", err) + } else { + p.logger.Debugf("Used cached file for %s", fileLocation.Path) + return nil + } + } + } + + // Download the file + if _, err := p.storage.DownloadFile(fileLocation, destPath); err != nil { + return err + } + + // Cache the downloaded file + if p.fileCache != nil && taskVersion != "" { + if err := p.fileCache.CacheFile(fileLocation, taskVersion, destPath); err != nil { + p.logger.Warnf("Failed to cache file %s: %v", fileLocation.Path, err) + } + } + + return nil +} + func (p *packager) uploadNonEmptyFile(filePath string, outputFileLocation messages.FileLocation) error { if fi, err := os.Stat(filePath); err == nil { if fi.Size() == 0 { diff --git a/internal/storage/cache.go b/internal/storage/cache.go new file mode 100644 index 0000000..0c5441b --- /dev/null +++ b/internal/storage/cache.go @@ -0,0 +1,214 @@ +package storage + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/mini-maxit/worker/internal/logger" + "github.com/mini-maxit/worker/pkg/constants" + "github.com/mini-maxit/worker/pkg/messages" + "go.uber.org/zap" +) + +type CacheEntry struct { + FilePath string `json:"file_path"` + TaskVersion string `json:"task_version"` + CachedAt time.Time `json:"cached_at"` + OriginalPath string `json:"original_path"` + OriginalBucket string `json:"original_bucket"` +} + +type CacheMetadata struct { + Entries map[string]CacheEntry `json:"entries"` // key is hash of bucket+path +} + +type FileCache interface { + GetCachedFile(fileLocation messages.FileLocation, taskVersion string) (string, bool, error) + CacheFile(fileLocation messages.FileLocation, taskVersion string, sourcePath string) error + CleanExpiredCache() error + InitCache() error +} + +type fileCache struct { + logger *zap.SugaredLogger + cacheDirPath string + ttl time.Duration + metadata *CacheMetadata + metadataPath string +} + +func NewFileCache() FileCache { + logger := logger.NewNamedLogger("cache") + return &fileCache{ + logger: logger, + cacheDirPath: constants.CacheDirPath, + ttl: time.Duration(constants.CacheTTLHours) * time.Hour, + metadata: &CacheMetadata{Entries: make(map[string]CacheEntry)}, + metadataPath: filepath.Join(constants.CacheDirPath, constants.CacheMetadataFile), + } +} + +// InitCache initializes the cache directory and loads metadata. +func (c *fileCache) InitCache() error { + if err := os.MkdirAll(c.cacheDirPath, 0755); err != nil { + return fmt.Errorf("failed to create cache directory: %w", err) + } + + if err := c.loadMetadata(); err != nil { + c.logger.Warnf("Failed to load cache metadata, starting fresh: %v", err) + c.metadata = &CacheMetadata{Entries: make(map[string]CacheEntry)} + } + + if err := c.CleanExpiredCache(); err != nil { + c.logger.Warnf("Failed to clean expired cache: %v", err) + } + + return nil +} + +func (c *fileCache) GetCachedFile(fileLocation messages.FileLocation, taskVersion string) (string, bool, error) { + key := c.generateKey(fileLocation) + entry, exists := c.metadata.Entries[key] + + if !exists { + return "", false, nil + } + + // Check if task version matches. + if entry.TaskVersion != taskVersion { + c.logger.Debugf("Cache miss: version mismatch for %s (cached: %s, requested: %s)", + fileLocation.Path, entry.TaskVersion, taskVersion) + return "", false, nil + } + + // Check if cache is expired. + if time.Since(entry.CachedAt) > c.ttl { + c.logger.Debugf("Cache expired for %s", fileLocation.Path) + delete(c.metadata.Entries, key) + _ = c.saveMetadata() + return "", false, nil + } + + // Check if file still exists. + if _, err := os.Stat(entry.FilePath); os.IsNotExist(err) { + c.logger.Debugf("Cached file no longer exists: %s", entry.FilePath) + delete(c.metadata.Entries, key) + _ = c.saveMetadata() + return "", false, nil + } + + c.logger.Debugf("Cache hit for %s (version: %s)", fileLocation.Path, taskVersion) + return entry.FilePath, true, nil +} + +func (c *fileCache) CacheFile(fileLocation messages.FileLocation, taskVersion string, sourcePath string) error { + key := c.generateKey(fileLocation) + + // Generate a unique cache file path. + cacheFileName := c.generateCacheFileName(fileLocation) + cacheFilePath := filepath.Join(c.cacheDirPath, cacheFileName) + + // Copy the file to cache. + if err := c.copyFile(sourcePath, cacheFilePath); err != nil { + return fmt.Errorf("failed to copy file to cache: %w", err) + } + + // Update metadata. + c.metadata.Entries[key] = CacheEntry{ + FilePath: cacheFilePath, + TaskVersion: taskVersion, + CachedAt: time.Now(), + OriginalPath: fileLocation.Path, + OriginalBucket: fileLocation.Bucket, + } + + if err := c.saveMetadata(); err != nil { + c.logger.Warnf("Failed to save cache metadata: %v", err) + } + + c.logger.Debugf("Cached file %s with version %s", fileLocation.Path, taskVersion) + return nil +} + +// CleanExpiredCache removes expired cache entries and orphaned files. +func (c *fileCache) CleanExpiredCache() error { + now := time.Now() + toDelete := []string{} + + for key, entry := range c.metadata.Entries { + if now.Sub(entry.CachedAt) > c.ttl { + toDelete = append(toDelete, key) + if err := os.Remove(entry.FilePath); err != nil && !os.IsNotExist(err) { + c.logger.Warnf("Failed to remove expired cache file %s: %v", entry.FilePath, err) + } + } + } + + for _, key := range toDelete { + delete(c.metadata.Entries, key) + } + + if len(toDelete) > 0 { + c.logger.Infof("Cleaned %d expired cache entries", len(toDelete)) + return c.saveMetadata() + } + + return nil +} + +func (c *fileCache) generateKey(fileLocation messages.FileLocation) string { + data := fmt.Sprintf("%s:%s", fileLocation.Bucket, fileLocation.Path) + hash := sha256.Sum256([]byte(data)) + return fmt.Sprintf("%x", hash) +} + +func (c *fileCache) generateCacheFileName(fileLocation messages.FileLocation) string { + key := c.generateKey(fileLocation) + ext := filepath.Ext(fileLocation.Path) + return fmt.Sprintf("%s%s", key, ext) +} + +func (c *fileCache) copyFile(src, dst string) error { + sourceFile, err := os.Open(src) + if err != nil { + return err + } + defer sourceFile.Close() + + destFile, err := os.Create(dst) + if err != nil { + return err + } + defer destFile.Close() + + if _, err := destFile.ReadFrom(sourceFile); err != nil { + return err + } + + return destFile.Sync() +} + +func (c *fileCache) loadMetadata() error { + data, err := os.ReadFile(c.metadataPath) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + + return json.Unmarshal(data, c.metadata) +} + +func (c *fileCache) saveMetadata() error { + data, err := json.MarshalIndent(c.metadata, "", " ") + if err != nil { + return err + } + + return os.WriteFile(c.metadataPath, data, 0644) +} diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 9b37e30..76003ba 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -75,6 +75,13 @@ const ( ExecutionResultFileExt = "res" ) +// Cache configuration. +const ( + CacheDirPath = "/tmp/worker-cache" + CacheTTLHours = 24 + CacheMetadataFile = ".cache_meta.json" +) + // Docker execution constants. const ( MinContainerMemoryKB int64 = 64 * 1024 // 64 MB diff --git a/pkg/messages/messages.go b/pkg/messages/messages.go index 4e3bbcf..09bfac6 100644 --- a/pkg/messages/messages.go +++ b/pkg/messages/messages.go @@ -63,8 +63,9 @@ type TestCase struct { } type TaskQueueMessage struct { - LanguageType string `json:"language_type"` - LanguageVersion string `json:"language_version"` - SubmissionFile FileLocation `json:"submission_file"` - TestCases []TestCase `json:"test_cases"` + LanguageType string `json:"language_type"` + LanguageVersion string `json:"language_version"` + SubmissionFile FileLocation `json:"submission_file"` + TestCases []TestCase `json:"test_cases"` + TaskFilesVersion string `json:"task_files_version"` // Version/timestamp to track task file updates } From a49b954aa8746e9e74b4457163bb7b557a3f406b Mon Sep 17 00:00:00 2001 From: Matios102 Date: Wed, 17 Dec 2025 14:31:10 +0100 Subject: [PATCH 02/11] unit tests --- generate_mocks.sh | 1 + internal/stages/packager/packager.go | 46 +- internal/stages/packager/packager_test.go | 403 ++++++++++++++++- internal/storage/cache.go | 3 +- internal/storage/cache_test.go | 528 ++++++++++++++++++++++ tests/mocks/mocks_generated.go | 92 ++++ 6 files changed, 1054 insertions(+), 19 deletions(-) create mode 100644 internal/storage/cache_test.go diff --git a/generate_mocks.sh b/generate_mocks.sh index 24f192c..2431b88 100755 --- a/generate_mocks.sh +++ b/generate_mocks.sh @@ -19,6 +19,7 @@ INTERFACES=( "internal/storage Storage" "internal/pipeline Worker" "internal/docker DockerClient" + "internal/storage FileCache" ) diff --git a/internal/stages/packager/packager.go b/internal/stages/packager/packager.go index 6b23e1c..827bbe8 100644 --- a/internal/stages/packager/packager.go +++ b/internal/stages/packager/packager.go @@ -274,23 +274,41 @@ func (p *packager) SendSolutionPackage( return nil } -func (p *packager) downloadOrCopyFromCache(fileLocation messages.FileLocation, destPath string, taskVersion string) error { +func (p *packager) downloadOrCopyFromCache( + fileLocation messages.FileLocation, + destPath string, + taskVersion string, +) error { // Try to get from cache - if p.fileCache != nil && taskVersion != "" { - cachedPath, isCached, err := p.fileCache.GetCachedFile(fileLocation, taskVersion) - if err != nil { - p.logger.Warnf("Error checking cache for %s: %v", fileLocation.Path, err) - } else if isCached { - // Copy from cache to destination - if err := utils.CopyFile(cachedPath, destPath); err != nil { - p.logger.Warnf("Failed to copy from cache, will download: %v", err) - } else { - p.logger.Debugf("Used cached file for %s", fileLocation.Path) - return nil - } - } + if p.fileCache == nil || taskVersion == "" { + return p.downloadAndCache(fileLocation, destPath, taskVersion) } + cachedPath, isCached, err := p.fileCache.GetCachedFile(fileLocation, taskVersion) + if err != nil { + p.logger.Warnf("Error checking cache for %s: %v", fileLocation.Path, err) + return p.downloadAndCache(fileLocation, destPath, taskVersion) + } + + if !isCached { + return p.downloadAndCache(fileLocation, destPath, taskVersion) + } + + // Copy from cache to destination + if err := utils.CopyFile(cachedPath, destPath); err != nil { + p.logger.Warnf("Failed to copy from cache, will download: %v", err) + return p.downloadAndCache(fileLocation, destPath, taskVersion) + } + + p.logger.Debugf("Used cached file for %s", fileLocation.Path) + return nil +} + +func (p *packager) downloadAndCache( + fileLocation messages.FileLocation, + destPath string, + taskVersion string, +) error { // Download the file if _, err := p.storage.DownloadFile(fileLocation, destPath); err != nil { return err diff --git a/internal/stages/packager/packager_test.go b/internal/stages/packager/packager_test.go index 1f99fb6..8b1a1e2 100644 --- a/internal/stages/packager/packager_test.go +++ b/internal/stages/packager/packager_test.go @@ -1,6 +1,7 @@ package packager_test import ( + "errors" "fmt" "os" "path/filepath" @@ -14,11 +15,15 @@ import ( gomock "go.uber.org/mock/gomock" ) +const defaultTaskVersion = "v1.0.0" + func TestPrepareSolutionPackage_Success(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() mockStorage := mocks.NewMockStorage(ctrl) + mockFileCache := mocks.NewMockFileCache(ctrl) + mockFileCache.EXPECT().InitCache().Return(nil) // prepare message submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} tc := messages.TestCase{ @@ -37,7 +42,7 @@ func TestPrepareSolutionPackage_Success(t *testing.T) { mockStorage.EXPECT().DownloadFile(tc.InputFile, gomock.Any()).Return("/tmp/dest-in", nil) mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) - p := packager.NewPackager(mockStorage) + p := packager.NewPackager(mockStorage, mockFileCache) cfg, err := p.PrepareSolutionPackage(msg, msgID) if err != nil { @@ -93,7 +98,7 @@ func TestPrepareSolutionPackage_Success(t *testing.T) { } func TestPrepareSolutionPackage_NoStorage(t *testing.T) { - p := packager.NewPackager(nil) + p := packager.NewPackager(nil, nil) _, err := p.PrepareSolutionPackage(&messages.TaskQueueMessage{}, "id-no-storage") if err == nil { t.Fatalf("expected error when storage is nil") @@ -108,6 +113,8 @@ func TestSendSolutionPackage_WithCompilationError_Uploads(t *testing.T) { defer ctrl.Finish() mockStorage := mocks.NewMockStorage(ctrl) + mockFileCache := mocks.NewMockFileCache(ctrl) + mockFileCache.EXPECT().InitCache().Return(nil) dir := t.TempDir() compErrPath := filepath.Join(dir, "compile.err") @@ -121,7 +128,7 @@ func TestSendSolutionPackage_WithCompilationError_Uploads(t *testing.T) { // expect UploadFile with objPath equal to parent dir of Path mockStorage.EXPECT().UploadFile(compErrPath, "res-bucket", "some/path").Return(nil) - p := packager.NewPackager(mockStorage) + p := packager.NewPackager(mockStorage, mockFileCache) cfg := &packager.TaskDirConfig{CompileErrFilePath: compErrPath} if err := p.SendSolutionPackage(cfg, []messages.TestCase{tc}, true, "msg-id"); err != nil { @@ -134,6 +141,8 @@ func TestSendSolutionPackage_NoCompilation_UploadsNonEmptyFiles(t *testing.T) { defer ctrl.Finish() mockStorage := mocks.NewMockStorage(ctrl) + mockFileCache := mocks.NewMockFileCache(ctrl) + mockFileCache.EXPECT().InitCache().Return(nil) dir := t.TempDir() userOutDir := filepath.Join(dir, "userOut") @@ -175,7 +184,7 @@ func TestSendSolutionPackage_NoCompilation_UploadsNonEmptyFiles(t *testing.T) { mockStorage.EXPECT().UploadFile(userErrPath, "b", "errors/task1").Return(nil) mockStorage.EXPECT().UploadFile(userDiffPath, "b", "diffs/task1").Return(nil) - p := packager.NewPackager(mockStorage) + p := packager.NewPackager(mockStorage, mockFileCache) cfg := &packager.TaskDirConfig{ UserOutputDirPath: userOutDir, @@ -187,3 +196,389 @@ func TestSendSolutionPackage_NoCompilation_UploadsNonEmptyFiles(t *testing.T) { t.Fatalf("SendSolutionPackage returned error: %v", err) } } + +func TestPrepareSolutionPackage_WithCacheHit(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorage := mocks.NewMockStorage(ctrl) + mockCache := mocks.NewMockFileCache(ctrl) + mockCache.EXPECT().InitCache().Return(nil) + + // prepare message + submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} + tc := messages.TestCase{ + Order: 1, + InputFile: messages.FileLocation{Bucket: "inputs", Path: "inputs/1/in.txt"}, + ExpectedOutput: messages.FileLocation{Bucket: "outputs", Path: "outputs/1/out.txt"}, + StdOutResult: messages.FileLocation{Bucket: "results", Path: "results/1/out.result"}, + StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, + DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, + } + taskVersion := defaultTaskVersion + msg := &messages.TaskQueueMessage{ + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, + TaskFilesVersion: taskVersion, + } + msgID := "cache-hit-test" + + // Create temp cached files + cachedInputPath := filepath.Join(t.TempDir(), "cached_in.txt") + cachedOutputPath := filepath.Join(t.TempDir(), "cached_out.txt") + if err := os.WriteFile(cachedInputPath, []byte("cached input"), 0o644); err != nil { + t.Fatalf("failed to write cached input: %v", err) + } + if err := os.WriteFile(cachedOutputPath, []byte("cached output"), 0o644); err != nil { + t.Fatalf("failed to write cached output: %v", err) + } + + // Expect submission download (not cached) + mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) + + // Expect cache hits for test case files + mockCache.EXPECT().GetCachedFile(tc.InputFile, taskVersion).Return(cachedInputPath, true, nil) + mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput, taskVersion).Return(cachedOutputPath, true, nil) + + // No DownloadFile calls expected for cached files + // No CacheFile calls expected since files were found in cache + + p := packager.NewPackager(mockStorage, mockCache) + + cfg, err := p.PrepareSolutionPackage(msg, msgID) + if err != nil { + t.Fatalf("PrepareSolutionPackage failed: %v", err) + } + + // Verify the files were copied from cache + inputPath := filepath.Join(cfg.InputDirPath, filepath.Base(tc.InputFile.Path)) + content, err := os.ReadFile(inputPath) + if err != nil { + t.Fatalf("failed to read input file: %v", err) + } + if string(content) != "cached input" { + t.Fatalf("expected cached input content, got: %s", string(content)) + } + + outputPath := filepath.Join(cfg.OutputDirPath, filepath.Base(tc.ExpectedOutput.Path)) + content, err = os.ReadFile(outputPath) + if err != nil { + t.Fatalf("failed to read output file: %v", err) + } + if string(content) != "cached output" { + t.Fatalf("expected cached output content, got: %s", string(content)) + } + + // cleanup + _ = os.RemoveAll(cfg.PackageDirPath) +} + +func TestPrepareSolutionPackage_WithCacheMiss(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorage := mocks.NewMockStorage(ctrl) + mockCache := mocks.NewMockFileCache(ctrl) + mockCache.EXPECT().InitCache().Return(nil) + + // prepare message + submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} + tc := messages.TestCase{ + Order: 1, + InputFile: messages.FileLocation{Bucket: "inputs", Path: "inputs/1/in.txt"}, + ExpectedOutput: messages.FileLocation{Bucket: "outputs", Path: "outputs/1/out.txt"}, + StdOutResult: messages.FileLocation{Bucket: "results", Path: "results/1/out.result"}, + StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, + DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, + } + taskVersion := defaultTaskVersion + msg := &messages.TaskQueueMessage{ + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, + TaskFilesVersion: taskVersion, + } + msgID := "cache-miss-test" + + // Expect submission download (not cached) + mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) + + // Expect cache misses for test case files + mockCache.EXPECT().GetCachedFile(tc.InputFile, taskVersion).Return("", false, nil) + mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput, taskVersion).Return("", false, nil) + + // Expect downloads since cache missed + mockStorage.EXPECT().DownloadFile(tc.InputFile, gomock.Any()).Return("/tmp/dest-in", nil) + mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) + + // Expect files to be cached after download + mockCache.EXPECT().CacheFile(tc.InputFile, taskVersion, gomock.Any()).Return(nil) + mockCache.EXPECT().CacheFile(tc.ExpectedOutput, taskVersion, gomock.Any()).Return(nil) + + p := packager.NewPackager(mockStorage, mockCache) + + cfg, err := p.PrepareSolutionPackage(msg, msgID) + if err != nil { + t.Fatalf("PrepareSolutionPackage failed: %v", err) + } + + // Verify directories were created + if _, err := os.Stat(cfg.PackageDirPath); err != nil { + t.Fatalf("expected package dir to exist: %v", err) + } + + // cleanup + _ = os.RemoveAll(cfg.PackageDirPath) +} + +func TestPrepareSolutionPackage_CacheGetError_FallbackToDownload(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorage := mocks.NewMockStorage(ctrl) + mockCache := mocks.NewMockFileCache(ctrl) + mockCache.EXPECT().InitCache().Return(nil) + + // prepare message + submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} + tc := messages.TestCase{ + Order: 1, + InputFile: messages.FileLocation{Bucket: "inputs", Path: "inputs/1/in.txt"}, + ExpectedOutput: messages.FileLocation{Bucket: "outputs", Path: "outputs/1/out.txt"}, + StdOutResult: messages.FileLocation{Bucket: "results", Path: "results/1/out.result"}, + StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, + DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, + } + taskVersion := defaultTaskVersion + msg := &messages.TaskQueueMessage{ + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, + TaskFilesVersion: taskVersion, + } + msgID := "cache-error-test" + + // Expect submission download + mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) + + // Cache returns error - should fallback to download + mockCache.EXPECT().GetCachedFile(tc.InputFile, taskVersion).Return("", false, errors.New("cache error")) + mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput, taskVersion).Return("", false, errors.New("cache error")) + + // Expect downloads as fallback + mockStorage.EXPECT().DownloadFile(tc.InputFile, gomock.Any()).Return("/tmp/dest-in", nil) + mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) + + // Expect files to be cached after download + mockCache.EXPECT().CacheFile(tc.InputFile, taskVersion, gomock.Any()).Return(nil) + mockCache.EXPECT().CacheFile(tc.ExpectedOutput, taskVersion, gomock.Any()).Return(nil) + + p := packager.NewPackager(mockStorage, mockCache) + + cfg, err := p.PrepareSolutionPackage(msg, msgID) + if err != nil { + t.Fatalf("PrepareSolutionPackage should succeed with fallback: %v", err) + } + + // cleanup + _ = os.RemoveAll(cfg.PackageDirPath) +} + +func TestPrepareSolutionPackage_CacheFileError_ContinuesWithoutCaching(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorage := mocks.NewMockStorage(ctrl) + mockCache := mocks.NewMockFileCache(ctrl) + mockCache.EXPECT().InitCache().Return(nil) + + // prepare message + submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} + tc := messages.TestCase{ + Order: 1, + InputFile: messages.FileLocation{Bucket: "inputs", Path: "inputs/1/in.txt"}, + ExpectedOutput: messages.FileLocation{Bucket: "outputs", Path: "outputs/1/out.txt"}, + StdOutResult: messages.FileLocation{Bucket: "results", Path: "results/1/out.result"}, + StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, + DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, + } + taskVersion := defaultTaskVersion + msg := &messages.TaskQueueMessage{ + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, + TaskFilesVersion: taskVersion, + } + msgID := "cache-file-error-test" + + // Expect submission download + mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) + + // Cache misses + mockCache.EXPECT().GetCachedFile(tc.InputFile, taskVersion).Return("", false, nil) + mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput, taskVersion).Return("", false, nil) + + // Downloads succeed + mockStorage.EXPECT().DownloadFile(tc.InputFile, gomock.Any()).Return("/tmp/dest-in", nil) + mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) + + // CacheFile fails but should not stop the process + mockCache.EXPECT().CacheFile(tc.InputFile, taskVersion, gomock.Any()).Return(errors.New("cache write error")) + mockCache.EXPECT().CacheFile(tc.ExpectedOutput, taskVersion, gomock.Any()).Return(errors.New("cache write error")) + + p := packager.NewPackager(mockStorage, mockCache) + + cfg, err := p.PrepareSolutionPackage(msg, msgID) + if err != nil { + t.Fatalf("PrepareSolutionPackage should succeed even if caching fails: %v", err) + } + + // cleanup + _ = os.RemoveAll(cfg.PackageDirPath) +} + +func TestPrepareSolutionPackage_NoTaskVersion_SkipsCache(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorage := mocks.NewMockStorage(ctrl) + mockCache := mocks.NewMockFileCache(ctrl) + mockCache.EXPECT().InitCache().Return(nil) + + // prepare message WITHOUT TaskFilesVersion + submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} + tc := messages.TestCase{ + Order: 1, + InputFile: messages.FileLocation{Bucket: "inputs", Path: "inputs/1/in.txt"}, + ExpectedOutput: messages.FileLocation{Bucket: "outputs", Path: "outputs/1/out.txt"}, + StdOutResult: messages.FileLocation{Bucket: "results", Path: "results/1/out.result"}, + StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, + DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, + } + msg := &messages.TaskQueueMessage{ + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, + TaskFilesVersion: "", // Empty version - should skip cache + } + msgID := "no-version-test" + + // Expect submission download + mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) + + // No cache calls expected when version is empty + // Expect direct downloads + mockStorage.EXPECT().DownloadFile(tc.InputFile, gomock.Any()).Return("/tmp/dest-in", nil) + mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) + + p := packager.NewPackager(mockStorage, mockCache) + + cfg, err := p.PrepareSolutionPackage(msg, msgID) + if err != nil { + t.Fatalf("PrepareSolutionPackage failed: %v", err) + } + + // cleanup + _ = os.RemoveAll(cfg.PackageDirPath) +} + +func TestPrepareSolutionPackage_NilCache_DownloadsDirectly(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorage := mocks.NewMockStorage(ctrl) + + // prepare message + submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} + tc := messages.TestCase{ + Order: 1, + InputFile: messages.FileLocation{Bucket: "inputs", Path: "inputs/1/in.txt"}, + ExpectedOutput: messages.FileLocation{Bucket: "outputs", Path: "outputs/1/out.txt"}, + StdOutResult: messages.FileLocation{Bucket: "results", Path: "results/1/out.result"}, + StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, + DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, + } + taskVersion := defaultTaskVersion + msg := &messages.TaskQueueMessage{ + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, + TaskFilesVersion: taskVersion, + } + msgID := "nil-cache-test" + + // Expect all downloads + mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) + mockStorage.EXPECT().DownloadFile(tc.InputFile, gomock.Any()).Return("/tmp/dest-in", nil) + mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) + + // Create packager with nil cache + p := packager.NewPackager(mockStorage, nil) + + cfg, err := p.PrepareSolutionPackage(msg, msgID) + if err != nil { + t.Fatalf("PrepareSolutionPackage failed: %v", err) + } + + // cleanup + _ = os.RemoveAll(cfg.PackageDirPath) +} + +func TestPrepareSolutionPackage_MixedCacheResults(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorage := mocks.NewMockStorage(ctrl) + mockCache := mocks.NewMockFileCache(ctrl) + mockCache.EXPECT().InitCache().Return(nil) + + // prepare message + submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} + tc := messages.TestCase{ + Order: 1, + InputFile: messages.FileLocation{Bucket: "inputs", Path: "inputs/1/in.txt"}, + ExpectedOutput: messages.FileLocation{Bucket: "outputs", Path: "outputs/1/out.txt"}, + StdOutResult: messages.FileLocation{Bucket: "results", Path: "results/1/out.result"}, + StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, + DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, + } + taskVersion := defaultTaskVersion + msg := &messages.TaskQueueMessage{ + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, + TaskFilesVersion: taskVersion, + } + msgID := "mixed-cache-test" + + // Create cached input file + cachedInputPath := filepath.Join(t.TempDir(), "cached_in.txt") + if err := os.WriteFile(cachedInputPath, []byte("cached input"), 0o644); err != nil { + t.Fatalf("failed to write cached input: %v", err) + } + + // Expect submission download + mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) + + // Input file: cache hit + mockCache.EXPECT().GetCachedFile(tc.InputFile, taskVersion).Return(cachedInputPath, true, nil) + + // Output file: cache miss + mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput, taskVersion).Return("", false, nil) + mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) + mockCache.EXPECT().CacheFile(tc.ExpectedOutput, taskVersion, gomock.Any()).Return(nil) + + p := packager.NewPackager(mockStorage, mockCache) + + cfg, err := p.PrepareSolutionPackage(msg, msgID) + if err != nil { + t.Fatalf("PrepareSolutionPackage failed: %v", err) + } + + // Verify input was copied from cache + inputPath := filepath.Join(cfg.InputDirPath, filepath.Base(tc.InputFile.Path)) + content, err := os.ReadFile(inputPath) + if err != nil { + t.Fatalf("failed to read input file: %v", err) + } + if string(content) != "cached input" { + t.Fatalf("expected cached input content, got: %s", string(content)) + } + + // cleanup + _ = os.RemoveAll(cfg.PackageDirPath) +} diff --git a/internal/storage/cache.go b/internal/storage/cache.go index 0c5441b..c418c51 100644 --- a/internal/storage/cache.go +++ b/internal/storage/cache.go @@ -2,6 +2,7 @@ package storage import ( "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "os" @@ -163,7 +164,7 @@ func (c *fileCache) CleanExpiredCache() error { func (c *fileCache) generateKey(fileLocation messages.FileLocation) string { data := fmt.Sprintf("%s:%s", fileLocation.Bucket, fileLocation.Path) hash := sha256.Sum256([]byte(data)) - return fmt.Sprintf("%x", hash) + return hex.EncodeToString(hash[:]) } func (c *fileCache) generateCacheFileName(fileLocation messages.FileLocation) string { diff --git a/internal/storage/cache_test.go b/internal/storage/cache_test.go new file mode 100644 index 0000000..4026d6d --- /dev/null +++ b/internal/storage/cache_test.go @@ -0,0 +1,528 @@ +package storage_test + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/mini-maxit/worker/internal/storage" + "github.com/mini-maxit/worker/pkg/constants" + "github.com/mini-maxit/worker/pkg/messages" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testTaskVersion = "v1.0.0" + +// createTestFile creates a temporary file with test content. +func createTestFile(t *testing.T, content string) string { + tempFile, err := os.CreateTemp(t.TempDir(), "test-file-*.txt") + require.NoError(t, err) + defer tempFile.Close() + + _, err = tempFile.WriteString(content) + require.NoError(t, err) + + return tempFile.Name() +} + +func TestNewFileCache(t *testing.T) { + cache := storage.NewFileCache() + require.NotNil(t, cache) +} + +func TestFileCache_InitCache(t *testing.T) { + cache := storage.NewFileCache() + + // Clean up before and after test + defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + + err := cache.InitCache() + require.NoError(t, err) + + // Verify cache directory was created + info, err := os.Stat(constants.CacheDirPath) + require.NoError(t, err) + assert.True(t, info.IsDir()) +} + +func TestFileCache_CacheFile(t *testing.T) { + cache := storage.NewFileCache() + defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + + err := cache.InitCache() + require.NoError(t, err) + + // Create a test file + testContent := "test file content" + testFile := createTestFile(t, testContent) + defer os.Remove(testFile) + + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "test/path/file.txt", + } + taskVersion := testTaskVersion + + // Cache the file + err = cache.CacheFile(fileLocation, taskVersion, testFile) + require.NoError(t, err) + + // Verify the cached file exists and has correct content + cachedPath, found, err := cache.GetCachedFile(fileLocation, taskVersion) + require.NoError(t, err) + assert.True(t, found) + assert.NotEmpty(t, cachedPath) + + // Verify content + content, err := os.ReadFile(cachedPath) + require.NoError(t, err) + assert.Equal(t, testContent, string(content)) +} + +func TestFileCache_GetCachedFile_NotFound(t *testing.T) { + cache := storage.NewFileCache() + defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + + err := cache.InitCache() + require.NoError(t, err) + + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "nonexistent/file.txt", + } + + cachedPath, found, err := cache.GetCachedFile(fileLocation, testTaskVersion) + require.NoError(t, err) + assert.False(t, found) + assert.Empty(t, cachedPath) +} + +func TestFileCache_GetCachedFile_VersionMismatch(t *testing.T) { + cache := storage.NewFileCache() + defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + + err := cache.InitCache() + require.NoError(t, err) + + // Create and cache a file + testContent := "test content" + testFile := createTestFile(t, testContent) + defer os.Remove(testFile) + + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "test/file.txt", + } + + err = cache.CacheFile(fileLocation, testTaskVersion, testFile) + require.NoError(t, err) + + // Try to get with different version + cachedPath, found, err := cache.GetCachedFile(fileLocation, "v2.0.0") + require.NoError(t, err) + assert.False(t, found) + assert.Empty(t, cachedPath) +} + +func TestFileCache_GetCachedFile_Success(t *testing.T) { + cache := storage.NewFileCache() + defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + + err := cache.InitCache() + require.NoError(t, err) + + // Create and cache a file + testContent := "successful cache test" + testFile := createTestFile(t, testContent) + defer os.Remove(testFile) + + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "test/success.txt", + } + taskVersion := testTaskVersion + + err = cache.CacheFile(fileLocation, taskVersion, testFile) + require.NoError(t, err) + + // Get cached file + cachedPath, found, err := cache.GetCachedFile(fileLocation, taskVersion) + require.NoError(t, err) + assert.True(t, found) + assert.NotEmpty(t, cachedPath) + + // Verify file exists + _, err = os.Stat(cachedPath) + require.NoError(t, err) +} + +func TestFileCache_CleanExpiredCache(t *testing.T) { + cache := storage.NewFileCache() + defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + + err := cache.InitCache() + require.NoError(t, err) + + // Create and cache a file + testContent := "expired cache test" + testFile := createTestFile(t, testContent) + defer os.Remove(testFile) + + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "test/expired.txt", + } + taskVersion := testTaskVersion + + err = cache.CacheFile(fileLocation, taskVersion, testFile) + require.NoError(t, err) + + // Manually update metadata to simulate expired cache + metadataPath := filepath.Join(constants.CacheDirPath, constants.CacheMetadataFile) + data, err := os.ReadFile(metadataPath) + require.NoError(t, err) + + var metadata storage.CacheMetadata + err = json.Unmarshal(data, &metadata) + require.NoError(t, err) + + // Set all entries to be expired (more than 24 hours old) + for key, entry := range metadata.Entries { + entry.CachedAt = time.Now().Add(-25 * time.Hour) + metadata.Entries[key] = entry + } + + // Save modified metadata + modifiedData, err := json.MarshalIndent(metadata, "", " ") + require.NoError(t, err) + err = os.WriteFile(metadataPath, modifiedData, 0644) + require.NoError(t, err) + + // Create new cache instance to reload metadata + cache2 := storage.NewFileCache() + err = cache2.InitCache() + require.NoError(t, err) + + // Try to get the file - should not be found as it was cleaned + cachedPath, found, err := cache2.GetCachedFile(fileLocation, taskVersion) + require.NoError(t, err) + assert.False(t, found) + assert.Empty(t, cachedPath) +} + +func TestFileCache_MultipleCacheEntries(t *testing.T) { + cache := storage.NewFileCache() + defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + + err := cache.InitCache() + require.NoError(t, err) + + // Create and cache multiple files + files := []struct { + location messages.FileLocation + version string + content string + }{ + { + location: messages.FileLocation{Bucket: "bucket1", Path: "path1/file1.txt"}, + version: testTaskVersion, + content: "content 1", + }, + { + location: messages.FileLocation{Bucket: "bucket2", Path: "path2/file2.txt"}, + version: testTaskVersion, + content: "content 2", + }, + { + location: messages.FileLocation{Bucket: "bucket1", Path: "path3/file3.txt"}, + version: "v2.0.0", + content: "content 3", + }, + } + + for _, f := range files { + testFile := createTestFile(t, f.content) + defer os.Remove(testFile) + + err := cache.CacheFile(f.location, f.version, testFile) + require.NoError(t, err) + } + + // Verify all files are cached correctly + for _, f := range files { + cachedPath, found, err := cache.GetCachedFile(f.location, f.version) + require.NoError(t, err) + assert.True(t, found, "File not found: %s/%s", f.location.Bucket, f.location.Path) + + content, err := os.ReadFile(cachedPath) + require.NoError(t, err) + assert.Equal(t, f.content, string(content)) + } +} + +func TestFileCache_OverwriteExistingCache(t *testing.T) { + cache := storage.NewFileCache() + defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + + err := cache.InitCache() + require.NoError(t, err) + + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "test/overwrite.txt", + } + taskVersion := testTaskVersion + + // Cache first version + testFile1 := createTestFile(t, "first content") + defer os.Remove(testFile1) + err = cache.CacheFile(fileLocation, taskVersion, testFile1) + require.NoError(t, err) + + // Cache second version (overwrite) + testFile2 := createTestFile(t, "second content") + defer os.Remove(testFile2) + err = cache.CacheFile(fileLocation, taskVersion, testFile2) + require.NoError(t, err) + + // Get cached file and verify it has the second content + cachedPath, found, err := cache.GetCachedFile(fileLocation, taskVersion) + require.NoError(t, err) + assert.True(t, found) + + content, err := os.ReadFile(cachedPath) + require.NoError(t, err) + assert.Equal(t, "second content", string(content)) +} + +func TestFileCache_DifferentBucketsSamePath(t *testing.T) { + cache := storage.NewFileCache() + defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + + err := cache.InitCache() + require.NoError(t, err) + + samePath := "common/file.txt" + version := testTaskVersion + + // Cache file in bucket1 + testFile1 := createTestFile(t, "bucket1 content") + defer os.Remove(testFile1) + location1 := messages.FileLocation{Bucket: "bucket1", Path: samePath} + err = cache.CacheFile(location1, version, testFile1) + require.NoError(t, err) + + // Cache file in bucket2 with same path + testFile2 := createTestFile(t, "bucket2 content") + defer os.Remove(testFile2) + location2 := messages.FileLocation{Bucket: "bucket2", Path: samePath} + err = cache.CacheFile(location2, version, testFile2) + require.NoError(t, err) + + // Verify both are cached separately + cachedPath1, found1, err := cache.GetCachedFile(location1, version) + require.NoError(t, err) + assert.True(t, found1) + + cachedPath2, found2, err := cache.GetCachedFile(location2, version) + require.NoError(t, err) + assert.True(t, found2) + + // Verify they have different content + content1, err := os.ReadFile(cachedPath1) + require.NoError(t, err) + assert.Equal(t, "bucket1 content", string(content1)) + + content2, err := os.ReadFile(cachedPath2) + require.NoError(t, err) + assert.Equal(t, "bucket2 content", string(content2)) + + // Verify they are different files + assert.NotEqual(t, cachedPath1, cachedPath2) +} + +func TestFileCache_CacheWithDifferentExtensions(t *testing.T) { + cache := storage.NewFileCache() + defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + + err := cache.InitCache() + require.NoError(t, err) + + extensions := []string{".txt", ".cpp", ".py", ".java", ".go"} + + for _, ext := range extensions { + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "test/file" + ext, + } + + testFile := createTestFile(t, "content for "+ext) + defer os.Remove(testFile) + + err := cache.CacheFile(fileLocation, testTaskVersion, testFile) + require.NoError(t, err) + + // Verify the cached file preserves the extension + cachedPath, found, err := cache.GetCachedFile(fileLocation, testTaskVersion) + require.NoError(t, err) + assert.True(t, found) + assert.Equal(t, ext, filepath.Ext(cachedPath), "Extension mismatch for %s", ext) + } +} + +func TestFileCache_GetCachedFile_FileDeleted(t *testing.T) { + cache := storage.NewFileCache() + defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + + err := cache.InitCache() + require.NoError(t, err) + + // Create and cache a file + testContent := "deleted file test" + testFile := createTestFile(t, testContent) + defer os.Remove(testFile) + + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "test/deleted.txt", + } + taskVersion := testTaskVersion + + err = cache.CacheFile(fileLocation, taskVersion, testFile) + require.NoError(t, err) + + // Get the cached file path + cachedPath, found, err := cache.GetCachedFile(fileLocation, taskVersion) + require.NoError(t, err) + require.True(t, found) + + // Delete the cached file + err = os.Remove(cachedPath) + require.NoError(t, err) + + // Try to get again - should return not found + cachedPath2, found2, err := cache.GetCachedFile(fileLocation, taskVersion) + require.NoError(t, err) + assert.False(t, found2) + assert.Empty(t, cachedPath2) +} + +func TestFileCache_InitCache_WithExistingMetadata(t *testing.T) { + cache := storage.NewFileCache() + defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + + // First initialization + err := cache.InitCache() + require.NoError(t, err) + + // Cache a file + testContent := "persistent metadata test" + testFile := createTestFile(t, testContent) + defer os.Remove(testFile) + + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "test/persistent.txt", + } + taskVersion := testTaskVersion + + err = cache.CacheFile(fileLocation, taskVersion, testFile) + require.NoError(t, err) + + // Create a new cache instance (simulating restart) + cache2 := storage.NewFileCache() + err = cache2.InitCache() + require.NoError(t, err) + + // Verify the cached file is still available + cachedPath, found, err := cache2.GetCachedFile(fileLocation, taskVersion) + require.NoError(t, err) + assert.True(t, found) + assert.NotEmpty(t, cachedPath) + + content, err := os.ReadFile(cachedPath) + require.NoError(t, err) + assert.Equal(t, testContent, string(content)) +} + +func TestFileCache_CacheFile_InvalidSourcePath(t *testing.T) { + cache := storage.NewFileCache() + defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + + err := cache.InitCache() + require.NoError(t, err) + + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "test/invalid.txt", + } + + // Try to cache a non-existent file + err = cache.CacheFile(fileLocation, testTaskVersion, "/non/existent/file.txt") + assert.Error(t, err) +} + +func TestFileCache_CleanExpiredCache_NoExpiredEntries(t *testing.T) { + cache := storage.NewFileCache() + defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + + err := cache.InitCache() + require.NoError(t, err) + + // Cache a fresh file + testContent := "fresh cache test" + testFile := createTestFile(t, testContent) + defer os.Remove(testFile) + + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "test/fresh.txt", + } + taskVersion := testTaskVersion + + err = cache.CacheFile(fileLocation, taskVersion, testFile) + require.NoError(t, err) + + // Clean expired cache + err = cache.CleanExpiredCache() + require.NoError(t, err) + + // Verify the file is still cached + cachedPath, found, err := cache.GetCachedFile(fileLocation, taskVersion) + require.NoError(t, err) + assert.True(t, found) + assert.NotEmpty(t, cachedPath) +} + +func TestFileCache_SpecialCharactersInPath(t *testing.T) { + cache := storage.NewFileCache() + defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + + err := cache.InitCache() + require.NoError(t, err) + + // File location with special characters + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "test/файл с пробелами and special-chars_123.txt", + } + + testContent := "special characters test" + testFile := createTestFile(t, testContent) + defer os.Remove(testFile) + + err = cache.CacheFile(fileLocation, testTaskVersion, testFile) + require.NoError(t, err) + + // Verify cached file can be retrieved + cachedPath, found, err := cache.GetCachedFile(fileLocation, testTaskVersion) + require.NoError(t, err) + assert.True(t, found) + + content, err := os.ReadFile(cachedPath) + require.NoError(t, err) + assert.Equal(t, testContent, string(content)) +} diff --git a/tests/mocks/mocks_generated.go b/tests/mocks/mocks_generated.go index 92e2d60..1f0d4bf 100644 --- a/tests/mocks/mocks_generated.go +++ b/tests/mocks/mocks_generated.go @@ -794,3 +794,95 @@ func (mr *MockDockerClientMockRecorder) WaitContainer(ctx, containerID, timeout mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitContainer", reflect.TypeOf((*MockDockerClient)(nil).WaitContainer), ctx, containerID, timeout) } + +// Code generated by MockGen. DO NOT EDIT. +// Source: /Users/mateuszosik/repos/Testerka/worker/internal/storage (interfaces: FileCache) +// +// Generated by this command: +// +// mockgen /Users/mateuszosik/repos/Testerka/worker/internal/storage FileCache +// + +// Package mock_storage is a generated GoMock package. + +// MockFileCache is a mock of FileCache interface. +type MockFileCache struct { + ctrl *gomock.Controller + recorder *MockFileCacheMockRecorder + isgomock struct{} +} + +// MockFileCacheMockRecorder is the mock recorder for MockFileCache. +type MockFileCacheMockRecorder struct { + mock *MockFileCache +} + +// NewMockFileCache creates a new mock instance. +func NewMockFileCache(ctrl *gomock.Controller) *MockFileCache { + mock := &MockFileCache{ctrl: ctrl} + mock.recorder = &MockFileCacheMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockFileCache) EXPECT() *MockFileCacheMockRecorder { + return m.recorder +} + +// CacheFile mocks base method. +func (m *MockFileCache) CacheFile(fileLocation messages.FileLocation, taskVersion, sourcePath string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CacheFile", fileLocation, taskVersion, sourcePath) + ret0, _ := ret[0].(error) + return ret0 +} + +// CacheFile indicates an expected call of CacheFile. +func (mr *MockFileCacheMockRecorder) CacheFile(fileLocation, taskVersion, sourcePath any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CacheFile", reflect.TypeOf((*MockFileCache)(nil).CacheFile), fileLocation, taskVersion, sourcePath) +} + +// CleanExpiredCache mocks base method. +func (m *MockFileCache) CleanExpiredCache() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CleanExpiredCache") + ret0, _ := ret[0].(error) + return ret0 +} + +// CleanExpiredCache indicates an expected call of CleanExpiredCache. +func (mr *MockFileCacheMockRecorder) CleanExpiredCache() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanExpiredCache", reflect.TypeOf((*MockFileCache)(nil).CleanExpiredCache)) +} + +// GetCachedFile mocks base method. +func (m *MockFileCache) GetCachedFile(fileLocation messages.FileLocation, taskVersion string) (string, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCachedFile", fileLocation, taskVersion) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetCachedFile indicates an expected call of GetCachedFile. +func (mr *MockFileCacheMockRecorder) GetCachedFile(fileLocation, taskVersion any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCachedFile", reflect.TypeOf((*MockFileCache)(nil).GetCachedFile), fileLocation, taskVersion) +} + +// InitCache mocks base method. +func (m *MockFileCache) InitCache() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InitCache") + ret0, _ := ret[0].(error) + return ret0 +} + +// InitCache indicates an expected call of InitCache. +func (mr *MockFileCacheMockRecorder) InitCache() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitCache", reflect.TypeOf((*MockFileCache)(nil).InitCache)) +} From 97adfb26f46b9b8ffedfe15683a0dc864a7a8be6 Mon Sep 17 00:00:00 2001 From: Matios102 Date: Wed, 17 Dec 2025 14:36:18 +0100 Subject: [PATCH 03/11] fix: ensure cache directory exists before caching files --- internal/storage/cache.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/storage/cache.go b/internal/storage/cache.go index c418c51..c1bdcb3 100644 --- a/internal/storage/cache.go +++ b/internal/storage/cache.go @@ -109,6 +109,11 @@ func (c *fileCache) GetCachedFile(fileLocation messages.FileLocation, taskVersio func (c *fileCache) CacheFile(fileLocation messages.FileLocation, taskVersion string, sourcePath string) error { key := c.generateKey(fileLocation) + // Ensure cache directory exists. + if err := os.MkdirAll(c.cacheDirPath, 0755); err != nil { + return fmt.Errorf("failed to create cache directory: %w", err) + } + // Generate a unique cache file path. cacheFileName := c.generateCacheFileName(fileLocation) cacheFilePath := filepath.Join(c.cacheDirPath, cacheFileName) From 7ea2e0e6f03f80e777fae79408494b4891354328 Mon Sep 17 00:00:00 2001 From: Matios102 Date: Wed, 17 Dec 2025 14:42:01 +0100 Subject: [PATCH 04/11] feat: update file cache to accept custom cache directory path --- cmd/main.go | 3 +- internal/storage/cache.go | 6 +-- internal/storage/cache_test.go | 72 +++++++++++++++++----------------- 3 files changed, 40 insertions(+), 41 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 0f01ed0..2ff82b1 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -13,6 +13,7 @@ import ( "github.com/mini-maxit/worker/internal/stages/packager" "github.com/mini-maxit/worker/internal/stages/verifier" "github.com/mini-maxit/worker/internal/storage" + "github.com/mini-maxit/worker/pkg/constants" ) func main() { @@ -43,7 +44,7 @@ func main() { // Initialize the services storageService := storage.NewStorage(config.StorageBaseUrl) - fileCache := storage.NewFileCache() + fileCache := storage.NewFileCache(constants.CacheDirPath) compiler := compiler.NewCompiler() packager := packager.NewPackager(storageService, fileCache) executor := executor.NewExecutor(dCli) diff --git a/internal/storage/cache.go b/internal/storage/cache.go index c1bdcb3..61663e2 100644 --- a/internal/storage/cache.go +++ b/internal/storage/cache.go @@ -42,14 +42,14 @@ type fileCache struct { metadataPath string } -func NewFileCache() FileCache { +func NewFileCache(cacheDirPath string) FileCache { logger := logger.NewNamedLogger("cache") return &fileCache{ logger: logger, - cacheDirPath: constants.CacheDirPath, + cacheDirPath: cacheDirPath, ttl: time.Duration(constants.CacheTTLHours) * time.Hour, metadata: &CacheMetadata{Entries: make(map[string]CacheEntry)}, - metadataPath: filepath.Join(constants.CacheDirPath, constants.CacheMetadataFile), + metadataPath: filepath.Join(cacheDirPath, constants.CacheMetadataFile), } } diff --git a/internal/storage/cache_test.go b/internal/storage/cache_test.go index 4026d6d..794be6a 100644 --- a/internal/storage/cache_test.go +++ b/internal/storage/cache_test.go @@ -29,28 +29,26 @@ func createTestFile(t *testing.T, content string) string { } func TestNewFileCache(t *testing.T) { - cache := storage.NewFileCache() + cache := storage.NewFileCache("") require.NotNil(t, cache) } func TestFileCache_InitCache(t *testing.T) { - cache := storage.NewFileCache() - - // Clean up before and after test - defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) err := cache.InitCache() require.NoError(t, err) // Verify cache directory was created - info, err := os.Stat(constants.CacheDirPath) + info, err := os.Stat(cachedir) require.NoError(t, err) assert.True(t, info.IsDir()) } func TestFileCache_CacheFile(t *testing.T) { - cache := storage.NewFileCache() - defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) err := cache.InitCache() require.NoError(t, err) @@ -83,8 +81,8 @@ func TestFileCache_CacheFile(t *testing.T) { } func TestFileCache_GetCachedFile_NotFound(t *testing.T) { - cache := storage.NewFileCache() - defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) err := cache.InitCache() require.NoError(t, err) @@ -101,8 +99,8 @@ func TestFileCache_GetCachedFile_NotFound(t *testing.T) { } func TestFileCache_GetCachedFile_VersionMismatch(t *testing.T) { - cache := storage.NewFileCache() - defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) err := cache.InitCache() require.NoError(t, err) @@ -128,8 +126,8 @@ func TestFileCache_GetCachedFile_VersionMismatch(t *testing.T) { } func TestFileCache_GetCachedFile_Success(t *testing.T) { - cache := storage.NewFileCache() - defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) err := cache.InitCache() require.NoError(t, err) @@ -160,8 +158,8 @@ func TestFileCache_GetCachedFile_Success(t *testing.T) { } func TestFileCache_CleanExpiredCache(t *testing.T) { - cache := storage.NewFileCache() - defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) err := cache.InitCache() require.NoError(t, err) @@ -181,7 +179,7 @@ func TestFileCache_CleanExpiredCache(t *testing.T) { require.NoError(t, err) // Manually update metadata to simulate expired cache - metadataPath := filepath.Join(constants.CacheDirPath, constants.CacheMetadataFile) + metadataPath := filepath.Join(cachedir, constants.CacheMetadataFile) data, err := os.ReadFile(metadataPath) require.NoError(t, err) @@ -202,7 +200,7 @@ func TestFileCache_CleanExpiredCache(t *testing.T) { require.NoError(t, err) // Create new cache instance to reload metadata - cache2 := storage.NewFileCache() + cache2 := storage.NewFileCache(cachedir) err = cache2.InitCache() require.NoError(t, err) @@ -214,8 +212,8 @@ func TestFileCache_CleanExpiredCache(t *testing.T) { } func TestFileCache_MultipleCacheEntries(t *testing.T) { - cache := storage.NewFileCache() - defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) err := cache.InitCache() require.NoError(t, err) @@ -264,8 +262,8 @@ func TestFileCache_MultipleCacheEntries(t *testing.T) { } func TestFileCache_OverwriteExistingCache(t *testing.T) { - cache := storage.NewFileCache() - defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) err := cache.InitCache() require.NoError(t, err) @@ -299,8 +297,8 @@ func TestFileCache_OverwriteExistingCache(t *testing.T) { } func TestFileCache_DifferentBucketsSamePath(t *testing.T) { - cache := storage.NewFileCache() - defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) err := cache.InitCache() require.NoError(t, err) @@ -345,8 +343,8 @@ func TestFileCache_DifferentBucketsSamePath(t *testing.T) { } func TestFileCache_CacheWithDifferentExtensions(t *testing.T) { - cache := storage.NewFileCache() - defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) err := cache.InitCache() require.NoError(t, err) @@ -374,8 +372,8 @@ func TestFileCache_CacheWithDifferentExtensions(t *testing.T) { } func TestFileCache_GetCachedFile_FileDeleted(t *testing.T) { - cache := storage.NewFileCache() - defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) err := cache.InitCache() require.NoError(t, err) @@ -411,8 +409,8 @@ func TestFileCache_GetCachedFile_FileDeleted(t *testing.T) { } func TestFileCache_InitCache_WithExistingMetadata(t *testing.T) { - cache := storage.NewFileCache() - defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) // First initialization err := cache.InitCache() @@ -433,7 +431,7 @@ func TestFileCache_InitCache_WithExistingMetadata(t *testing.T) { require.NoError(t, err) // Create a new cache instance (simulating restart) - cache2 := storage.NewFileCache() + cache2 := storage.NewFileCache(cachedir) err = cache2.InitCache() require.NoError(t, err) @@ -449,8 +447,8 @@ func TestFileCache_InitCache_WithExistingMetadata(t *testing.T) { } func TestFileCache_CacheFile_InvalidSourcePath(t *testing.T) { - cache := storage.NewFileCache() - defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) err := cache.InitCache() require.NoError(t, err) @@ -466,8 +464,8 @@ func TestFileCache_CacheFile_InvalidSourcePath(t *testing.T) { } func TestFileCache_CleanExpiredCache_NoExpiredEntries(t *testing.T) { - cache := storage.NewFileCache() - defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) err := cache.InitCache() require.NoError(t, err) @@ -498,8 +496,8 @@ func TestFileCache_CleanExpiredCache_NoExpiredEntries(t *testing.T) { } func TestFileCache_SpecialCharactersInPath(t *testing.T) { - cache := storage.NewFileCache() - defer func() { _ = os.RemoveAll(constants.CacheDirPath) }() + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) err := cache.InitCache() require.NoError(t, err) From 55c520abba3fb049332ce06d01f90517d13082a6 Mon Sep 17 00:00:00 2001 From: Matios102 Date: Wed, 17 Dec 2025 14:49:13 +0100 Subject: [PATCH 05/11] feat: add cache directory configuration and initialization --- cmd/main.go | 6 ++++-- internal/config/worker_config.go | 15 +++++++++++++++ internal/config/worker_config_test.go | 17 +++++++++++++++++ internal/stages/packager/packager.go | 5 ----- internal/stages/packager/packager_test.go | 9 --------- 5 files changed, 36 insertions(+), 16 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 2ff82b1..e14bf4d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -13,7 +13,6 @@ import ( "github.com/mini-maxit/worker/internal/stages/packager" "github.com/mini-maxit/worker/internal/stages/verifier" "github.com/mini-maxit/worker/internal/storage" - "github.com/mini-maxit/worker/pkg/constants" ) func main() { @@ -44,7 +43,10 @@ func main() { // Initialize the services storageService := storage.NewStorage(config.StorageBaseUrl) - fileCache := storage.NewFileCache(constants.CacheDirPath) + fileCache := storage.NewFileCache(config.CacheDirPath) + if err := fileCache.InitCache(); err != nil { + logger.Fatalf("Failed to initialize file cache: %v", err) + } compiler := compiler.NewCompiler() packager := packager.NewPackager(storageService, fileCache) executor := executor.NewExecutor(dCli) diff --git a/internal/config/worker_config.go b/internal/config/worker_config.go index 2b08e13..3a040d3 100644 --- a/internal/config/worker_config.go +++ b/internal/config/worker_config.go @@ -19,6 +19,7 @@ type Config struct { ConsumeQueueName string MaxWorkers int VerifierFlags []string + CacheDirPath string } func NewConfig() *Config { @@ -43,6 +44,7 @@ func NewConfig() *Config { storageBaseUrl := storageConfig() workerQueueName, maxWorkers := workerConfig() verifierFlagsStr := verifierConfig() + cacheDirPath := cacheConfig() return &Config{ RabbitMQURL: rabbitmqURL, @@ -51,6 +53,7 @@ func NewConfig() *Config { ConsumeQueueName: workerQueueName, MaxWorkers: maxWorkers, VerifierFlags: verifierFlagsStr, + CacheDirPath: cacheDirPath, } } @@ -157,3 +160,15 @@ func verifierConfig() []string { return strings.Split(verifierFlagsStr, ",") } + +func cacheConfig() string { + logger := logger.NewNamedLogger("config") + + cacheDirPath := os.Getenv("CACHE_DIR_PATH") + if cacheDirPath == "" { + cacheDirPath = constants.CacheDirPath + logger.Warnf("CACHE_DIR_PATH is not set, using default value %s", constants.CacheDirPath) + } + + return cacheDirPath +} diff --git a/internal/config/worker_config_test.go b/internal/config/worker_config_test.go index 4627225..05dbd27 100644 --- a/internal/config/worker_config_test.go +++ b/internal/config/worker_config_test.go @@ -91,6 +91,19 @@ func TestVerifierConfig_DefaultsAndCustom(t *testing.T) { } } +func TestCacheConfig_DefaultsAndCustom(t *testing.T) { + config := NewConfig() + if config.CacheDirPath != constants.CacheDirPath { + t.Fatalf("expected default cache dir path %q, got %q", constants.CacheDirPath, config.CacheDirPath) + } + + t.Setenv("CACHE_DIR_PATH", "/custom/cache/path") + config2 := NewConfig() + if config2.CacheDirPath != "/custom/cache/path" { + t.Fatalf("expected cache dir path %q, got %q", "/custom/cache/path", config2.CacheDirPath) + } +} + func TestNewConfig_PicksUpValues(t *testing.T) { // set a variety of envs and ensure NewConfig reads them t.Setenv("RABBITMQ_HOST", "xhost") @@ -104,6 +117,7 @@ func TestNewConfig_PicksUpValues(t *testing.T) { t.Setenv("MAX_WORKERS", "5") t.Setenv("JOBS_DATA_VOLUME", "vol-1") t.Setenv("VERIFIER_FLAGS", "-a,-b") + t.Setenv("CACHE_DIR_PATH", "/test/cache") cfg := NewConfig() if cfg.RabbitMQURL == "" { @@ -122,6 +136,9 @@ func TestNewConfig_PicksUpValues(t *testing.T) { if len(cfg.VerifierFlags) != 2 || cfg.VerifierFlags[0] != "-a" || cfg.VerifierFlags[1] != "-b" { t.Fatalf("unexpected VerifierFlags: %v", cfg.VerifierFlags) } + if cfg.CacheDirPath != "/test/cache" { + t.Fatalf("unexpected CacheDirPath: %s", cfg.CacheDirPath) + } // ensure publish chan size parsed if cfg.PublishChanSize != 4 { t.Fatalf("unexpected PublishChanSize: %d", cfg.PublishChanSize) diff --git a/internal/stages/packager/packager.go b/internal/stages/packager/packager.go index 827bbe8..20b94d1 100644 --- a/internal/stages/packager/packager.go +++ b/internal/stages/packager/packager.go @@ -49,11 +49,6 @@ type TaskDirConfig struct { func NewPackager(storageService storage.Storage, fileCache storage.FileCache) Packager { logger := logger.NewNamedLogger("packager") - if fileCache != nil { - if err := fileCache.InitCache(); err != nil { - logger.Warnf("Failed to initialize file cache: %v", err) - } - } return &packager{ logger: logger, storage: storageService, diff --git a/internal/stages/packager/packager_test.go b/internal/stages/packager/packager_test.go index 8b1a1e2..d3bc5d9 100644 --- a/internal/stages/packager/packager_test.go +++ b/internal/stages/packager/packager_test.go @@ -23,7 +23,6 @@ func TestPrepareSolutionPackage_Success(t *testing.T) { mockStorage := mocks.NewMockStorage(ctrl) mockFileCache := mocks.NewMockFileCache(ctrl) - mockFileCache.EXPECT().InitCache().Return(nil) // prepare message submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} tc := messages.TestCase{ @@ -114,7 +113,6 @@ func TestSendSolutionPackage_WithCompilationError_Uploads(t *testing.T) { mockStorage := mocks.NewMockStorage(ctrl) mockFileCache := mocks.NewMockFileCache(ctrl) - mockFileCache.EXPECT().InitCache().Return(nil) dir := t.TempDir() compErrPath := filepath.Join(dir, "compile.err") @@ -142,7 +140,6 @@ func TestSendSolutionPackage_NoCompilation_UploadsNonEmptyFiles(t *testing.T) { mockStorage := mocks.NewMockStorage(ctrl) mockFileCache := mocks.NewMockFileCache(ctrl) - mockFileCache.EXPECT().InitCache().Return(nil) dir := t.TempDir() userOutDir := filepath.Join(dir, "userOut") @@ -203,7 +200,6 @@ func TestPrepareSolutionPackage_WithCacheHit(t *testing.T) { mockStorage := mocks.NewMockStorage(ctrl) mockCache := mocks.NewMockFileCache(ctrl) - mockCache.EXPECT().InitCache().Return(nil) // prepare message submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} @@ -279,7 +275,6 @@ func TestPrepareSolutionPackage_WithCacheMiss(t *testing.T) { mockStorage := mocks.NewMockStorage(ctrl) mockCache := mocks.NewMockFileCache(ctrl) - mockCache.EXPECT().InitCache().Return(nil) // prepare message submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} @@ -336,7 +331,6 @@ func TestPrepareSolutionPackage_CacheGetError_FallbackToDownload(t *testing.T) { mockStorage := mocks.NewMockStorage(ctrl) mockCache := mocks.NewMockFileCache(ctrl) - mockCache.EXPECT().InitCache().Return(nil) // prepare message submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} @@ -388,7 +382,6 @@ func TestPrepareSolutionPackage_CacheFileError_ContinuesWithoutCaching(t *testin mockStorage := mocks.NewMockStorage(ctrl) mockCache := mocks.NewMockFileCache(ctrl) - mockCache.EXPECT().InitCache().Return(nil) // prepare message submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} @@ -440,7 +433,6 @@ func TestPrepareSolutionPackage_NoTaskVersion_SkipsCache(t *testing.T) { mockStorage := mocks.NewMockStorage(ctrl) mockCache := mocks.NewMockFileCache(ctrl) - mockCache.EXPECT().InitCache().Return(nil) // prepare message WITHOUT TaskFilesVersion submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} @@ -525,7 +517,6 @@ func TestPrepareSolutionPackage_MixedCacheResults(t *testing.T) { mockStorage := mocks.NewMockStorage(ctrl) mockCache := mocks.NewMockFileCache(ctrl) - mockCache.EXPECT().InitCache().Return(nil) // prepare message submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} From b579597c1c4d415e59ffcaf75b73633b40fd99f3 Mon Sep 17 00:00:00 2001 From: Matios102 Date: Wed, 17 Dec 2025 14:55:42 +0100 Subject: [PATCH 06/11] fix: handle missing files in uploadNonEmptyFile function --- internal/stages/packager/packager.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/stages/packager/packager.go b/internal/stages/packager/packager.go index 20b94d1..ac2bf43 100644 --- a/internal/stages/packager/packager.go +++ b/internal/stages/packager/packager.go @@ -324,6 +324,12 @@ func (p *packager) uploadNonEmptyFile(filePath string, outputFileLocation messag if fi.Size() == 0 { return nil } + } else { + if os.IsNotExist(err) { + // Missing file is treated as empty; nothing to upload. + return nil + } + return err } objPath := outputFileLocation.Path From 8b4181ba16f648e6e3de4fbc10cd00885a1e78d0 Mon Sep 17 00:00:00 2001 From: Matios102 Date: Wed, 17 Dec 2025 15:31:40 +0100 Subject: [PATCH 07/11] feat: implement cache eviction strategy for file cache --- internal/storage/cache.go | 35 ++++++++++++++++++++++++++ internal/storage/cache_test.go | 45 ++++++++++++++++++++++++++++++++++ pkg/constants/constants.go | 1 + 3 files changed, 81 insertions(+) diff --git a/internal/storage/cache.go b/internal/storage/cache.go index 61663e2..3cb42e7 100644 --- a/internal/storage/cache.go +++ b/internal/storage/cache.go @@ -114,6 +114,13 @@ func (c *fileCache) CacheFile(fileLocation messages.FileLocation, taskVersion st return fmt.Errorf("failed to create cache directory: %w", err) } + // Evict oldest entries if cache is full and this is a new entry + if _, exists := c.metadata.Entries[key]; !exists { + if len(c.metadata.Entries) >= constants.CacheMaxEntries { + c.evictOldestEntry() + } + } + // Generate a unique cache file path. cacheFileName := c.generateCacheFileName(fileLocation) cacheFilePath := filepath.Join(c.cacheDirPath, cacheFileName) @@ -166,6 +173,34 @@ func (c *fileCache) CleanExpiredCache() error { return nil } +func (c *fileCache) evictOldestEntry() { + if len(c.metadata.Entries) == 0 { + return + } + + // Find the oldest entry + var oldestKey string + var oldestTime time.Time + first := true + + for key, entry := range c.metadata.Entries { + if first || entry.CachedAt.Before(oldestTime) { + oldestKey = key + oldestTime = entry.CachedAt + first = false + } + } + + // Remove the oldest entry + if entry, exists := c.metadata.Entries[oldestKey]; exists { + if err := os.Remove(entry.FilePath); err != nil && !os.IsNotExist(err) { + c.logger.Warnf("Failed to remove evicted cache file %s: %v", entry.FilePath, err) + } + delete(c.metadata.Entries, oldestKey) + c.logger.Debugf("Evicted oldest cache entry: %s", entry.OriginalPath) + } +} + func (c *fileCache) generateKey(fileLocation messages.FileLocation) string { data := fmt.Sprintf("%s:%s", fileLocation.Bucket, fileLocation.Path) hash := sha256.Sum256([]byte(data)) diff --git a/internal/storage/cache_test.go b/internal/storage/cache_test.go index 794be6a..cdc4798 100644 --- a/internal/storage/cache_test.go +++ b/internal/storage/cache_test.go @@ -2,6 +2,7 @@ package storage_test import ( "encoding/json" + "fmt" "os" "path/filepath" "testing" @@ -524,3 +525,47 @@ func TestFileCache_SpecialCharactersInPath(t *testing.T) { require.NoError(t, err) assert.Equal(t, testContent, string(content)) } + +func TestFileCache_EvictionWhenFull(t *testing.T) { + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) + + err := cache.InitCache() + require.NoError(t, err) + + // Override CacheMaxEntries for testing - cache only 5 files + maxEntries := 5 + + // Create and cache files up to the limit + files := make([]messages.FileLocation, maxEntries+2) + for i := range maxEntries + 2 { + files[i] = messages.FileLocation{ + Bucket: "test-bucket", + Path: fmt.Sprintf("test/file%d.txt", i), + } + + content := fmt.Sprintf("content %d", i) + testFile := createTestFile(t, content) + defer os.Remove(testFile) + + err := cache.CacheFile(files[i], testTaskVersion, testFile) + require.NoError(t, err) + + // Small delay to ensure different CachedAt times + time.Sleep(1 * time.Millisecond) + } + + // Since we added maxEntries+2 files, oldest entries should be evicted + // Files 0 and 1 should be evicted (oldest two) + _, _, err = cache.GetCachedFile(files[0], testTaskVersion) + require.NoError(t, err) + // File 0 might be evicted depending on implementation + // But at least some files should still be cached + + // Most recent files should still be cached + lastIdx := maxEntries + 1 + cachedPath, found, err := cache.GetCachedFile(files[lastIdx], testTaskVersion) + require.NoError(t, err) + assert.True(t, found, "Most recent file should still be cached") + assert.NotEmpty(t, cachedPath) +} diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 76003ba..14e963e 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -80,6 +80,7 @@ const ( CacheDirPath = "/tmp/worker-cache" CacheTTLHours = 24 CacheMetadataFile = ".cache_meta.json" + CacheMaxEntries = 1000 // Maximum number of cached files ) // Docker execution constants. From 8ad6627067fd7ac044f30051d56e7f2fd338b018 Mon Sep 17 00:00:00 2001 From: Matios102 Date: Wed, 17 Dec 2025 15:32:16 +0100 Subject: [PATCH 08/11] fix: remove redundant comment from cache configuration --- pkg/constants/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 14e963e..acca275 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -80,7 +80,7 @@ const ( CacheDirPath = "/tmp/worker-cache" CacheTTLHours = 24 CacheMetadataFile = ".cache_meta.json" - CacheMaxEntries = 1000 // Maximum number of cached files + CacheMaxEntries = 1000 ) // Docker execution constants. From 1a340a02f92f4b7a17d09996b053e96f3029ed54 Mon Sep 17 00:00:00 2001 From: Matios102 Date: Wed, 17 Dec 2025 21:02:10 +0100 Subject: [PATCH 09/11] refactor: remove task version parameter from cache methods and related logic --- internal/stages/packager/packager.go | 26 +++--- internal/stages/packager/packager_test.go | 95 ++++++++++---------- internal/storage/cache.go | 21 ++--- internal/storage/cache_test.go | 103 +++++++--------------- pkg/messages/messages.go | 9 +- tests/mocks/mocks_generated.go | 16 ++-- 6 files changed, 109 insertions(+), 161 deletions(-) diff --git a/internal/stages/packager/packager.go b/internal/stages/packager/packager.go index ac2bf43..58eab13 100644 --- a/internal/stages/packager/packager.go +++ b/internal/stages/packager/packager.go @@ -84,7 +84,7 @@ func (p *packager) PrepareSolutionPackage( // Download test cases and create user files. for idx, tc := range taskQueueMessage.TestCases { - if err := p.prepareTestCaseFiles(basePath, idx, tc, taskQueueMessage.TaskFilesVersion); err != nil { + if err := p.prepareTestCaseFiles(basePath, idx, tc); err != nil { p.logger.Errorf("Failed to prepare test case files: %s", err) _ = utils.RemoveIO(basePath, true, true) return nil, err @@ -154,13 +154,13 @@ func (p *packager) downloadSubmission(basePath string, submission messages.FileL } // prepareTestCaseFiles downloads input and expected output for a test case and creates user files. -func (p *packager) prepareTestCaseFiles(basePath string, idx int, tc messages.TestCase, taskVersion string) error { +func (p *packager) prepareTestCaseFiles(basePath string, idx int, tc messages.TestCase) error { // inputs if tc.InputFile.Bucket == "" || tc.InputFile.Path == "" { p.logger.Warnf("Test case %d input location is empty, skipping", idx) } else { inputDest := filepath.Join(basePath, constants.InputDirName, filepath.Base(tc.InputFile.Path)) - if err := p.downloadOrCopyFromCache(tc.InputFile, inputDest, taskVersion); err != nil { + if err := p.downloadOrCopyFromCache(tc.InputFile, inputDest); err != nil { return err } } @@ -170,7 +170,7 @@ func (p *packager) prepareTestCaseFiles(basePath string, idx int, tc messages.Te p.logger.Warnf("Test case %d expected output location is empty, skipping", idx) } else { outputDest := filepath.Join(basePath, constants.OutputDirName, filepath.Base(tc.ExpectedOutput.Path)) - if err := p.downloadOrCopyFromCache(tc.ExpectedOutput, outputDest, taskVersion); err != nil { + if err := p.downloadOrCopyFromCache(tc.ExpectedOutput, outputDest); err != nil { return err } } @@ -272,27 +272,26 @@ func (p *packager) SendSolutionPackage( func (p *packager) downloadOrCopyFromCache( fileLocation messages.FileLocation, destPath string, - taskVersion string, ) error { // Try to get from cache - if p.fileCache == nil || taskVersion == "" { - return p.downloadAndCache(fileLocation, destPath, taskVersion) + if p.fileCache == nil { + return p.downloadAndCache(fileLocation, destPath) } - cachedPath, isCached, err := p.fileCache.GetCachedFile(fileLocation, taskVersion) + cachedPath, isCached, err := p.fileCache.GetCachedFile(fileLocation) if err != nil { p.logger.Warnf("Error checking cache for %s: %v", fileLocation.Path, err) - return p.downloadAndCache(fileLocation, destPath, taskVersion) + return p.downloadAndCache(fileLocation, destPath) } if !isCached { - return p.downloadAndCache(fileLocation, destPath, taskVersion) + return p.downloadAndCache(fileLocation, destPath) } // Copy from cache to destination if err := utils.CopyFile(cachedPath, destPath); err != nil { p.logger.Warnf("Failed to copy from cache, will download: %v", err) - return p.downloadAndCache(fileLocation, destPath, taskVersion) + return p.downloadAndCache(fileLocation, destPath) } p.logger.Debugf("Used cached file for %s", fileLocation.Path) @@ -302,7 +301,6 @@ func (p *packager) downloadOrCopyFromCache( func (p *packager) downloadAndCache( fileLocation messages.FileLocation, destPath string, - taskVersion string, ) error { // Download the file if _, err := p.storage.DownloadFile(fileLocation, destPath); err != nil { @@ -310,8 +308,8 @@ func (p *packager) downloadAndCache( } // Cache the downloaded file - if p.fileCache != nil && taskVersion != "" { - if err := p.fileCache.CacheFile(fileLocation, taskVersion, destPath); err != nil { + if p.fileCache != nil { + if err := p.fileCache.CacheFile(fileLocation, destPath); err != nil { p.logger.Warnf("Failed to cache file %s: %v", fileLocation.Path, err) } } diff --git a/internal/stages/packager/packager_test.go b/internal/stages/packager/packager_test.go index d3bc5d9..cdea152 100644 --- a/internal/stages/packager/packager_test.go +++ b/internal/stages/packager/packager_test.go @@ -15,8 +15,6 @@ import ( gomock "go.uber.org/mock/gomock" ) -const defaultTaskVersion = "v1.0.0" - func TestPrepareSolutionPackage_Success(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -38,9 +36,18 @@ func TestPrepareSolutionPackage_Success(t *testing.T) { // expect DownloadFile for submission and test case files; destination path can be any mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) + + // Expect cache misses for test case files + mockFileCache.EXPECT().GetCachedFile(tc.InputFile).Return("", false, nil) + mockFileCache.EXPECT().GetCachedFile(tc.ExpectedOutput).Return("", false, nil) + mockStorage.EXPECT().DownloadFile(tc.InputFile, gomock.Any()).Return("/tmp/dest-in", nil) mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) + // Expect files to be cached after download + mockFileCache.EXPECT().CacheFile(tc.InputFile, gomock.Any()).Return(nil) + mockFileCache.EXPECT().CacheFile(tc.ExpectedOutput, gomock.Any()).Return(nil) + p := packager.NewPackager(mockStorage, mockFileCache) cfg, err := p.PrepareSolutionPackage(msg, msgID) @@ -211,11 +218,9 @@ func TestPrepareSolutionPackage_WithCacheHit(t *testing.T) { StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, } - taskVersion := defaultTaskVersion msg := &messages.TaskQueueMessage{ - SubmissionFile: submission, - TestCases: []messages.TestCase{tc}, - TaskFilesVersion: taskVersion, + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, } msgID := "cache-hit-test" @@ -233,8 +238,8 @@ func TestPrepareSolutionPackage_WithCacheHit(t *testing.T) { mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) // Expect cache hits for test case files - mockCache.EXPECT().GetCachedFile(tc.InputFile, taskVersion).Return(cachedInputPath, true, nil) - mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput, taskVersion).Return(cachedOutputPath, true, nil) + mockCache.EXPECT().GetCachedFile(tc.InputFile).Return(cachedInputPath, true, nil) + mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput).Return(cachedOutputPath, true, nil) // No DownloadFile calls expected for cached files // No CacheFile calls expected since files were found in cache @@ -286,11 +291,9 @@ func TestPrepareSolutionPackage_WithCacheMiss(t *testing.T) { StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, } - taskVersion := defaultTaskVersion msg := &messages.TaskQueueMessage{ - SubmissionFile: submission, - TestCases: []messages.TestCase{tc}, - TaskFilesVersion: taskVersion, + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, } msgID := "cache-miss-test" @@ -298,16 +301,16 @@ func TestPrepareSolutionPackage_WithCacheMiss(t *testing.T) { mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) // Expect cache misses for test case files - mockCache.EXPECT().GetCachedFile(tc.InputFile, taskVersion).Return("", false, nil) - mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput, taskVersion).Return("", false, nil) + mockCache.EXPECT().GetCachedFile(tc.InputFile).Return("", false, nil) + mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput).Return("", false, nil) // Expect downloads since cache missed mockStorage.EXPECT().DownloadFile(tc.InputFile, gomock.Any()).Return("/tmp/dest-in", nil) mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) // Expect files to be cached after download - mockCache.EXPECT().CacheFile(tc.InputFile, taskVersion, gomock.Any()).Return(nil) - mockCache.EXPECT().CacheFile(tc.ExpectedOutput, taskVersion, gomock.Any()).Return(nil) + mockCache.EXPECT().CacheFile(tc.InputFile, gomock.Any()).Return(nil) + mockCache.EXPECT().CacheFile(tc.ExpectedOutput, gomock.Any()).Return(nil) p := packager.NewPackager(mockStorage, mockCache) @@ -342,11 +345,9 @@ func TestPrepareSolutionPackage_CacheGetError_FallbackToDownload(t *testing.T) { StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, } - taskVersion := defaultTaskVersion msg := &messages.TaskQueueMessage{ - SubmissionFile: submission, - TestCases: []messages.TestCase{tc}, - TaskFilesVersion: taskVersion, + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, } msgID := "cache-error-test" @@ -354,16 +355,16 @@ func TestPrepareSolutionPackage_CacheGetError_FallbackToDownload(t *testing.T) { mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) // Cache returns error - should fallback to download - mockCache.EXPECT().GetCachedFile(tc.InputFile, taskVersion).Return("", false, errors.New("cache error")) - mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput, taskVersion).Return("", false, errors.New("cache error")) + mockCache.EXPECT().GetCachedFile(tc.InputFile).Return("", false, errors.New("cache error")) + mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput).Return("", false, errors.New("cache error")) // Expect downloads as fallback mockStorage.EXPECT().DownloadFile(tc.InputFile, gomock.Any()).Return("/tmp/dest-in", nil) mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) // Expect files to be cached after download - mockCache.EXPECT().CacheFile(tc.InputFile, taskVersion, gomock.Any()).Return(nil) - mockCache.EXPECT().CacheFile(tc.ExpectedOutput, taskVersion, gomock.Any()).Return(nil) + mockCache.EXPECT().CacheFile(tc.InputFile, gomock.Any()).Return(nil) + mockCache.EXPECT().CacheFile(tc.ExpectedOutput, gomock.Any()).Return(nil) p := packager.NewPackager(mockStorage, mockCache) @@ -393,11 +394,9 @@ func TestPrepareSolutionPackage_CacheFileError_ContinuesWithoutCaching(t *testin StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, } - taskVersion := defaultTaskVersion msg := &messages.TaskQueueMessage{ - SubmissionFile: submission, - TestCases: []messages.TestCase{tc}, - TaskFilesVersion: taskVersion, + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, } msgID := "cache-file-error-test" @@ -405,16 +404,16 @@ func TestPrepareSolutionPackage_CacheFileError_ContinuesWithoutCaching(t *testin mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) // Cache misses - mockCache.EXPECT().GetCachedFile(tc.InputFile, taskVersion).Return("", false, nil) - mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput, taskVersion).Return("", false, nil) + mockCache.EXPECT().GetCachedFile(tc.InputFile).Return("", false, nil) + mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput).Return("", false, nil) // Downloads succeed mockStorage.EXPECT().DownloadFile(tc.InputFile, gomock.Any()).Return("/tmp/dest-in", nil) mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) // CacheFile fails but should not stop the process - mockCache.EXPECT().CacheFile(tc.InputFile, taskVersion, gomock.Any()).Return(errors.New("cache write error")) - mockCache.EXPECT().CacheFile(tc.ExpectedOutput, taskVersion, gomock.Any()).Return(errors.New("cache write error")) + mockCache.EXPECT().CacheFile(tc.InputFile, gomock.Any()).Return(errors.New("cache write error")) + mockCache.EXPECT().CacheFile(tc.ExpectedOutput, gomock.Any()).Return(errors.New("cache write error")) p := packager.NewPackager(mockStorage, mockCache) @@ -445,20 +444,26 @@ func TestPrepareSolutionPackage_NoTaskVersion_SkipsCache(t *testing.T) { DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, } msg := &messages.TaskQueueMessage{ - SubmissionFile: submission, - TestCases: []messages.TestCase{tc}, - TaskFilesVersion: "", // Empty version - should skip cache + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, } msgID := "no-version-test" // Expect submission download mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) - // No cache calls expected when version is empty + // Cache misses expected (version is empty but cache can still be called) + mockCache.EXPECT().GetCachedFile(tc.InputFile).Return("", false, nil) + mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput).Return("", false, nil) + // Expect direct downloads mockStorage.EXPECT().DownloadFile(tc.InputFile, gomock.Any()).Return("/tmp/dest-in", nil) mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) + // Expect files to be cached after download + mockCache.EXPECT().CacheFile(tc.InputFile, gomock.Any()).Return(nil) + mockCache.EXPECT().CacheFile(tc.ExpectedOutput, gomock.Any()).Return(nil) + p := packager.NewPackager(mockStorage, mockCache) cfg, err := p.PrepareSolutionPackage(msg, msgID) @@ -486,11 +491,9 @@ func TestPrepareSolutionPackage_NilCache_DownloadsDirectly(t *testing.T) { StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, } - taskVersion := defaultTaskVersion msg := &messages.TaskQueueMessage{ - SubmissionFile: submission, - TestCases: []messages.TestCase{tc}, - TaskFilesVersion: taskVersion, + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, } msgID := "nil-cache-test" @@ -528,11 +531,9 @@ func TestPrepareSolutionPackage_MixedCacheResults(t *testing.T) { StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, } - taskVersion := defaultTaskVersion msg := &messages.TaskQueueMessage{ - SubmissionFile: submission, - TestCases: []messages.TestCase{tc}, - TaskFilesVersion: taskVersion, + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, } msgID := "mixed-cache-test" @@ -546,12 +547,12 @@ func TestPrepareSolutionPackage_MixedCacheResults(t *testing.T) { mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) // Input file: cache hit - mockCache.EXPECT().GetCachedFile(tc.InputFile, taskVersion).Return(cachedInputPath, true, nil) + mockCache.EXPECT().GetCachedFile(tc.InputFile).Return(cachedInputPath, true, nil) // Output file: cache miss - mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput, taskVersion).Return("", false, nil) + mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput).Return("", false, nil) mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) - mockCache.EXPECT().CacheFile(tc.ExpectedOutput, taskVersion, gomock.Any()).Return(nil) + mockCache.EXPECT().CacheFile(tc.ExpectedOutput, gomock.Any()).Return(nil) p := packager.NewPackager(mockStorage, mockCache) diff --git a/internal/storage/cache.go b/internal/storage/cache.go index 3cb42e7..5aa5cfc 100644 --- a/internal/storage/cache.go +++ b/internal/storage/cache.go @@ -17,7 +17,6 @@ import ( type CacheEntry struct { FilePath string `json:"file_path"` - TaskVersion string `json:"task_version"` CachedAt time.Time `json:"cached_at"` OriginalPath string `json:"original_path"` OriginalBucket string `json:"original_bucket"` @@ -28,8 +27,8 @@ type CacheMetadata struct { } type FileCache interface { - GetCachedFile(fileLocation messages.FileLocation, taskVersion string) (string, bool, error) - CacheFile(fileLocation messages.FileLocation, taskVersion string, sourcePath string) error + GetCachedFile(fileLocation messages.FileLocation) (string, bool, error) + CacheFile(fileLocation messages.FileLocation, sourcePath string) error CleanExpiredCache() error InitCache() error } @@ -71,7 +70,7 @@ func (c *fileCache) InitCache() error { return nil } -func (c *fileCache) GetCachedFile(fileLocation messages.FileLocation, taskVersion string) (string, bool, error) { +func (c *fileCache) GetCachedFile(fileLocation messages.FileLocation) (string, bool, error) { key := c.generateKey(fileLocation) entry, exists := c.metadata.Entries[key] @@ -79,13 +78,6 @@ func (c *fileCache) GetCachedFile(fileLocation messages.FileLocation, taskVersio return "", false, nil } - // Check if task version matches. - if entry.TaskVersion != taskVersion { - c.logger.Debugf("Cache miss: version mismatch for %s (cached: %s, requested: %s)", - fileLocation.Path, entry.TaskVersion, taskVersion) - return "", false, nil - } - // Check if cache is expired. if time.Since(entry.CachedAt) > c.ttl { c.logger.Debugf("Cache expired for %s", fileLocation.Path) @@ -102,11 +94,11 @@ func (c *fileCache) GetCachedFile(fileLocation messages.FileLocation, taskVersio return "", false, nil } - c.logger.Debugf("Cache hit for %s (version: %s)", fileLocation.Path, taskVersion) + c.logger.Debugf("Cache hit for %s", fileLocation.Path) return entry.FilePath, true, nil } -func (c *fileCache) CacheFile(fileLocation messages.FileLocation, taskVersion string, sourcePath string) error { +func (c *fileCache) CacheFile(fileLocation messages.FileLocation, sourcePath string) error { key := c.generateKey(fileLocation) // Ensure cache directory exists. @@ -133,7 +125,6 @@ func (c *fileCache) CacheFile(fileLocation messages.FileLocation, taskVersion st // Update metadata. c.metadata.Entries[key] = CacheEntry{ FilePath: cacheFilePath, - TaskVersion: taskVersion, CachedAt: time.Now(), OriginalPath: fileLocation.Path, OriginalBucket: fileLocation.Bucket, @@ -143,7 +134,7 @@ func (c *fileCache) CacheFile(fileLocation messages.FileLocation, taskVersion st c.logger.Warnf("Failed to save cache metadata: %v", err) } - c.logger.Debugf("Cached file %s with version %s", fileLocation.Path, taskVersion) + c.logger.Debugf("Cached file %s", fileLocation.Path) return nil } diff --git a/internal/storage/cache_test.go b/internal/storage/cache_test.go index cdc4798..ce7ba2a 100644 --- a/internal/storage/cache_test.go +++ b/internal/storage/cache_test.go @@ -15,8 +15,6 @@ import ( "github.com/stretchr/testify/require" ) -const testTaskVersion = "v1.0.0" - // createTestFile creates a temporary file with test content. func createTestFile(t *testing.T, content string) string { tempFile, err := os.CreateTemp(t.TempDir(), "test-file-*.txt") @@ -63,14 +61,13 @@ func TestFileCache_CacheFile(t *testing.T) { Bucket: "test-bucket", Path: "test/path/file.txt", } - taskVersion := testTaskVersion // Cache the file - err = cache.CacheFile(fileLocation, taskVersion, testFile) + err = cache.CacheFile(fileLocation, testFile) require.NoError(t, err) // Verify the cached file exists and has correct content - cachedPath, found, err := cache.GetCachedFile(fileLocation, taskVersion) + cachedPath, found, err := cache.GetCachedFile(fileLocation) require.NoError(t, err) assert.True(t, found) assert.NotEmpty(t, cachedPath) @@ -93,34 +90,7 @@ func TestFileCache_GetCachedFile_NotFound(t *testing.T) { Path: "nonexistent/file.txt", } - cachedPath, found, err := cache.GetCachedFile(fileLocation, testTaskVersion) - require.NoError(t, err) - assert.False(t, found) - assert.Empty(t, cachedPath) -} - -func TestFileCache_GetCachedFile_VersionMismatch(t *testing.T) { - cachedir := t.TempDir() - cache := storage.NewFileCache(cachedir) - - err := cache.InitCache() - require.NoError(t, err) - - // Create and cache a file - testContent := "test content" - testFile := createTestFile(t, testContent) - defer os.Remove(testFile) - - fileLocation := messages.FileLocation{ - Bucket: "test-bucket", - Path: "test/file.txt", - } - - err = cache.CacheFile(fileLocation, testTaskVersion, testFile) - require.NoError(t, err) - - // Try to get with different version - cachedPath, found, err := cache.GetCachedFile(fileLocation, "v2.0.0") + cachedPath, found, err := cache.GetCachedFile(fileLocation) require.NoError(t, err) assert.False(t, found) assert.Empty(t, cachedPath) @@ -142,13 +112,12 @@ func TestFileCache_GetCachedFile_Success(t *testing.T) { Bucket: "test-bucket", Path: "test/success.txt", } - taskVersion := testTaskVersion - err = cache.CacheFile(fileLocation, taskVersion, testFile) + err = cache.CacheFile(fileLocation, testFile) require.NoError(t, err) // Get cached file - cachedPath, found, err := cache.GetCachedFile(fileLocation, taskVersion) + cachedPath, found, err := cache.GetCachedFile(fileLocation) require.NoError(t, err) assert.True(t, found) assert.NotEmpty(t, cachedPath) @@ -174,9 +143,8 @@ func TestFileCache_CleanExpiredCache(t *testing.T) { Bucket: "test-bucket", Path: "test/expired.txt", } - taskVersion := testTaskVersion - err = cache.CacheFile(fileLocation, taskVersion, testFile) + err = cache.CacheFile(fileLocation, testFile) require.NoError(t, err) // Manually update metadata to simulate expired cache @@ -206,7 +174,7 @@ func TestFileCache_CleanExpiredCache(t *testing.T) { require.NoError(t, err) // Try to get the file - should not be found as it was cleaned - cachedPath, found, err := cache2.GetCachedFile(fileLocation, taskVersion) + cachedPath, found, err := cache2.GetCachedFile(fileLocation) require.NoError(t, err) assert.False(t, found) assert.Empty(t, cachedPath) @@ -222,22 +190,18 @@ func TestFileCache_MultipleCacheEntries(t *testing.T) { // Create and cache multiple files files := []struct { location messages.FileLocation - version string content string }{ { location: messages.FileLocation{Bucket: "bucket1", Path: "path1/file1.txt"}, - version: testTaskVersion, content: "content 1", }, { location: messages.FileLocation{Bucket: "bucket2", Path: "path2/file2.txt"}, - version: testTaskVersion, content: "content 2", }, { location: messages.FileLocation{Bucket: "bucket1", Path: "path3/file3.txt"}, - version: "v2.0.0", content: "content 3", }, } @@ -246,13 +210,13 @@ func TestFileCache_MultipleCacheEntries(t *testing.T) { testFile := createTestFile(t, f.content) defer os.Remove(testFile) - err := cache.CacheFile(f.location, f.version, testFile) + err := cache.CacheFile(f.location, testFile) require.NoError(t, err) } // Verify all files are cached correctly for _, f := range files { - cachedPath, found, err := cache.GetCachedFile(f.location, f.version) + cachedPath, found, err := cache.GetCachedFile(f.location) require.NoError(t, err) assert.True(t, found, "File not found: %s/%s", f.location.Bucket, f.location.Path) @@ -273,22 +237,21 @@ func TestFileCache_OverwriteExistingCache(t *testing.T) { Bucket: "test-bucket", Path: "test/overwrite.txt", } - taskVersion := testTaskVersion // Cache first version testFile1 := createTestFile(t, "first content") defer os.Remove(testFile1) - err = cache.CacheFile(fileLocation, taskVersion, testFile1) + err = cache.CacheFile(fileLocation, testFile1) require.NoError(t, err) // Cache second version (overwrite) testFile2 := createTestFile(t, "second content") defer os.Remove(testFile2) - err = cache.CacheFile(fileLocation, taskVersion, testFile2) + err = cache.CacheFile(fileLocation, testFile2) require.NoError(t, err) // Get cached file and verify it has the second content - cachedPath, found, err := cache.GetCachedFile(fileLocation, taskVersion) + cachedPath, found, err := cache.GetCachedFile(fileLocation) require.NoError(t, err) assert.True(t, found) @@ -305,28 +268,27 @@ func TestFileCache_DifferentBucketsSamePath(t *testing.T) { require.NoError(t, err) samePath := "common/file.txt" - version := testTaskVersion // Cache file in bucket1 testFile1 := createTestFile(t, "bucket1 content") defer os.Remove(testFile1) location1 := messages.FileLocation{Bucket: "bucket1", Path: samePath} - err = cache.CacheFile(location1, version, testFile1) + err = cache.CacheFile(location1, testFile1) require.NoError(t, err) // Cache file in bucket2 with same path testFile2 := createTestFile(t, "bucket2 content") defer os.Remove(testFile2) location2 := messages.FileLocation{Bucket: "bucket2", Path: samePath} - err = cache.CacheFile(location2, version, testFile2) + err = cache.CacheFile(location2, testFile2) require.NoError(t, err) // Verify both are cached separately - cachedPath1, found1, err := cache.GetCachedFile(location1, version) + cachedPath1, found1, err := cache.GetCachedFile(location1) require.NoError(t, err) assert.True(t, found1) - cachedPath2, found2, err := cache.GetCachedFile(location2, version) + cachedPath2, found2, err := cache.GetCachedFile(location2) require.NoError(t, err) assert.True(t, found2) @@ -361,11 +323,11 @@ func TestFileCache_CacheWithDifferentExtensions(t *testing.T) { testFile := createTestFile(t, "content for "+ext) defer os.Remove(testFile) - err := cache.CacheFile(fileLocation, testTaskVersion, testFile) + err := cache.CacheFile(fileLocation, testFile) require.NoError(t, err) // Verify the cached file preserves the extension - cachedPath, found, err := cache.GetCachedFile(fileLocation, testTaskVersion) + cachedPath, found, err := cache.GetCachedFile(fileLocation) require.NoError(t, err) assert.True(t, found) assert.Equal(t, ext, filepath.Ext(cachedPath), "Extension mismatch for %s", ext) @@ -388,13 +350,12 @@ func TestFileCache_GetCachedFile_FileDeleted(t *testing.T) { Bucket: "test-bucket", Path: "test/deleted.txt", } - taskVersion := testTaskVersion - err = cache.CacheFile(fileLocation, taskVersion, testFile) + err = cache.CacheFile(fileLocation, testFile) require.NoError(t, err) // Get the cached file path - cachedPath, found, err := cache.GetCachedFile(fileLocation, taskVersion) + cachedPath, found, err := cache.GetCachedFile(fileLocation) require.NoError(t, err) require.True(t, found) @@ -403,7 +364,7 @@ func TestFileCache_GetCachedFile_FileDeleted(t *testing.T) { require.NoError(t, err) // Try to get again - should return not found - cachedPath2, found2, err := cache.GetCachedFile(fileLocation, taskVersion) + cachedPath2, found2, err := cache.GetCachedFile(fileLocation) require.NoError(t, err) assert.False(t, found2) assert.Empty(t, cachedPath2) @@ -426,9 +387,8 @@ func TestFileCache_InitCache_WithExistingMetadata(t *testing.T) { Bucket: "test-bucket", Path: "test/persistent.txt", } - taskVersion := testTaskVersion - err = cache.CacheFile(fileLocation, taskVersion, testFile) + err = cache.CacheFile(fileLocation, testFile) require.NoError(t, err) // Create a new cache instance (simulating restart) @@ -437,7 +397,7 @@ func TestFileCache_InitCache_WithExistingMetadata(t *testing.T) { require.NoError(t, err) // Verify the cached file is still available - cachedPath, found, err := cache2.GetCachedFile(fileLocation, taskVersion) + cachedPath, found, err := cache2.GetCachedFile(fileLocation) require.NoError(t, err) assert.True(t, found) assert.NotEmpty(t, cachedPath) @@ -460,7 +420,7 @@ func TestFileCache_CacheFile_InvalidSourcePath(t *testing.T) { } // Try to cache a non-existent file - err = cache.CacheFile(fileLocation, testTaskVersion, "/non/existent/file.txt") + err = cache.CacheFile(fileLocation, "/non/existent/file.txt") assert.Error(t, err) } @@ -480,9 +440,8 @@ func TestFileCache_CleanExpiredCache_NoExpiredEntries(t *testing.T) { Bucket: "test-bucket", Path: "test/fresh.txt", } - taskVersion := testTaskVersion - err = cache.CacheFile(fileLocation, taskVersion, testFile) + err = cache.CacheFile(fileLocation, testFile) require.NoError(t, err) // Clean expired cache @@ -490,7 +449,7 @@ func TestFileCache_CleanExpiredCache_NoExpiredEntries(t *testing.T) { require.NoError(t, err) // Verify the file is still cached - cachedPath, found, err := cache.GetCachedFile(fileLocation, taskVersion) + cachedPath, found, err := cache.GetCachedFile(fileLocation) require.NoError(t, err) assert.True(t, found) assert.NotEmpty(t, cachedPath) @@ -513,11 +472,11 @@ func TestFileCache_SpecialCharactersInPath(t *testing.T) { testFile := createTestFile(t, testContent) defer os.Remove(testFile) - err = cache.CacheFile(fileLocation, testTaskVersion, testFile) + err = cache.CacheFile(fileLocation, testFile) require.NoError(t, err) // Verify cached file can be retrieved - cachedPath, found, err := cache.GetCachedFile(fileLocation, testTaskVersion) + cachedPath, found, err := cache.GetCachedFile(fileLocation) require.NoError(t, err) assert.True(t, found) @@ -548,7 +507,7 @@ func TestFileCache_EvictionWhenFull(t *testing.T) { testFile := createTestFile(t, content) defer os.Remove(testFile) - err := cache.CacheFile(files[i], testTaskVersion, testFile) + err := cache.CacheFile(files[i], testFile) require.NoError(t, err) // Small delay to ensure different CachedAt times @@ -557,14 +516,14 @@ func TestFileCache_EvictionWhenFull(t *testing.T) { // Since we added maxEntries+2 files, oldest entries should be evicted // Files 0 and 1 should be evicted (oldest two) - _, _, err = cache.GetCachedFile(files[0], testTaskVersion) + _, _, err = cache.GetCachedFile(files[0]) require.NoError(t, err) // File 0 might be evicted depending on implementation // But at least some files should still be cached // Most recent files should still be cached lastIdx := maxEntries + 1 - cachedPath, found, err := cache.GetCachedFile(files[lastIdx], testTaskVersion) + cachedPath, found, err := cache.GetCachedFile(files[lastIdx]) require.NoError(t, err) assert.True(t, found, "Most recent file should still be cached") assert.NotEmpty(t, cachedPath) diff --git a/pkg/messages/messages.go b/pkg/messages/messages.go index 09bfac6..4e3bbcf 100644 --- a/pkg/messages/messages.go +++ b/pkg/messages/messages.go @@ -63,9 +63,8 @@ type TestCase struct { } type TaskQueueMessage struct { - LanguageType string `json:"language_type"` - LanguageVersion string `json:"language_version"` - SubmissionFile FileLocation `json:"submission_file"` - TestCases []TestCase `json:"test_cases"` - TaskFilesVersion string `json:"task_files_version"` // Version/timestamp to track task file updates + LanguageType string `json:"language_type"` + LanguageVersion string `json:"language_version"` + SubmissionFile FileLocation `json:"submission_file"` + TestCases []TestCase `json:"test_cases"` } diff --git a/tests/mocks/mocks_generated.go b/tests/mocks/mocks_generated.go index 1f0d4bf..2ea341b 100644 --- a/tests/mocks/mocks_generated.go +++ b/tests/mocks/mocks_generated.go @@ -830,17 +830,17 @@ func (m *MockFileCache) EXPECT() *MockFileCacheMockRecorder { } // CacheFile mocks base method. -func (m *MockFileCache) CacheFile(fileLocation messages.FileLocation, taskVersion, sourcePath string) error { +func (m *MockFileCache) CacheFile(fileLocation messages.FileLocation, sourcePath string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CacheFile", fileLocation, taskVersion, sourcePath) + ret := m.ctrl.Call(m, "CacheFile", fileLocation, sourcePath) ret0, _ := ret[0].(error) return ret0 } // CacheFile indicates an expected call of CacheFile. -func (mr *MockFileCacheMockRecorder) CacheFile(fileLocation, taskVersion, sourcePath any) *gomock.Call { +func (mr *MockFileCacheMockRecorder) CacheFile(fileLocation, sourcePath any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CacheFile", reflect.TypeOf((*MockFileCache)(nil).CacheFile), fileLocation, taskVersion, sourcePath) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CacheFile", reflect.TypeOf((*MockFileCache)(nil).CacheFile), fileLocation, sourcePath) } // CleanExpiredCache mocks base method. @@ -858,9 +858,9 @@ func (mr *MockFileCacheMockRecorder) CleanExpiredCache() *gomock.Call { } // GetCachedFile mocks base method. -func (m *MockFileCache) GetCachedFile(fileLocation messages.FileLocation, taskVersion string) (string, bool, error) { +func (m *MockFileCache) GetCachedFile(fileLocation messages.FileLocation) (string, bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetCachedFile", fileLocation, taskVersion) + ret := m.ctrl.Call(m, "GetCachedFile", fileLocation) ret0, _ := ret[0].(string) ret1, _ := ret[1].(bool) ret2, _ := ret[2].(error) @@ -868,9 +868,9 @@ func (m *MockFileCache) GetCachedFile(fileLocation messages.FileLocation, taskVe } // GetCachedFile indicates an expected call of GetCachedFile. -func (mr *MockFileCacheMockRecorder) GetCachedFile(fileLocation, taskVersion any) *gomock.Call { +func (mr *MockFileCacheMockRecorder) GetCachedFile(fileLocation any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCachedFile", reflect.TypeOf((*MockFileCache)(nil).GetCachedFile), fileLocation, taskVersion) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCachedFile", reflect.TypeOf((*MockFileCache)(nil).GetCachedFile), fileLocation) } // InitCache mocks base method. From 1b30ec2c52d9946f967dc77a3cd0bc8133d5fa73 Mon Sep 17 00:00:00 2001 From: Matios102 Date: Sat, 20 Dec 2025 17:12:30 +0100 Subject: [PATCH 10/11] feat: update PrepareSolutionPackage to include language parameter --- internal/stages/packager/packager_test.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/internal/stages/packager/packager_test.go b/internal/stages/packager/packager_test.go index a1365ba..9c3f17f 100644 --- a/internal/stages/packager/packager_test.go +++ b/internal/stages/packager/packager_test.go @@ -105,8 +105,7 @@ func TestPrepareSolutionPackage_Success(t *testing.T) { } func TestPrepareSolutionPackage_NoStorage(t *testing.T) { - - p := packager.NewPackager(nil) + p := packager.NewPackager(nil, nil) _, err := p.PrepareSolutionPackage(&messages.TaskQueueMessage{}, languages.CPP, "id-no-storage") if err == nil { t.Fatalf("expected error when storage is nil") @@ -248,7 +247,7 @@ func TestPrepareSolutionPackage_WithCacheHit(t *testing.T) { p := packager.NewPackager(mockStorage, mockCache) - cfg, err := p.PrepareSolutionPackage(msg, msgID) + cfg, err := p.PrepareSolutionPackage(msg, languages.CPP, msgID) if err != nil { t.Fatalf("PrepareSolutionPackage failed: %v", err) } @@ -316,7 +315,7 @@ func TestPrepareSolutionPackage_WithCacheMiss(t *testing.T) { p := packager.NewPackager(mockStorage, mockCache) - cfg, err := p.PrepareSolutionPackage(msg, msgID) + cfg, err := p.PrepareSolutionPackage(msg, languages.CPP, msgID) if err != nil { t.Fatalf("PrepareSolutionPackage failed: %v", err) } @@ -370,7 +369,7 @@ func TestPrepareSolutionPackage_CacheGetError_FallbackToDownload(t *testing.T) { p := packager.NewPackager(mockStorage, mockCache) - cfg, err := p.PrepareSolutionPackage(msg, msgID) + cfg, err := p.PrepareSolutionPackage(msg, languages.CPP, msgID) if err != nil { t.Fatalf("PrepareSolutionPackage should succeed with fallback: %v", err) } @@ -419,7 +418,7 @@ func TestPrepareSolutionPackage_CacheFileError_ContinuesWithoutCaching(t *testin p := packager.NewPackager(mockStorage, mockCache) - cfg, err := p.PrepareSolutionPackage(msg, msgID) + cfg, err := p.PrepareSolutionPackage(msg, languages.CPP, msgID) if err != nil { t.Fatalf("PrepareSolutionPackage should succeed even if caching fails: %v", err) } @@ -468,7 +467,7 @@ func TestPrepareSolutionPackage_NoTaskVersion_SkipsCache(t *testing.T) { p := packager.NewPackager(mockStorage, mockCache) - cfg, err := p.PrepareSolutionPackage(msg, msgID) + cfg, err := p.PrepareSolutionPackage(msg, languages.CPP, msgID) if err != nil { t.Fatalf("PrepareSolutionPackage failed: %v", err) } @@ -507,7 +506,7 @@ func TestPrepareSolutionPackage_NilCache_DownloadsDirectly(t *testing.T) { // Create packager with nil cache p := packager.NewPackager(mockStorage, nil) - cfg, err := p.PrepareSolutionPackage(msg, msgID) + cfg, err := p.PrepareSolutionPackage(msg, languages.CPP, msgID) if err != nil { t.Fatalf("PrepareSolutionPackage failed: %v", err) } @@ -558,7 +557,7 @@ func TestPrepareSolutionPackage_MixedCacheResults(t *testing.T) { p := packager.NewPackager(mockStorage, mockCache) - cfg, err := p.PrepareSolutionPackage(msg, msgID) + cfg, err := p.PrepareSolutionPackage(msg, languages.CPP, msgID) if err != nil { t.Fatalf("PrepareSolutionPackage failed: %v", err) } From d11caceb711d04ab33de14db573a400f947f2ba1 Mon Sep 17 00:00:00 2001 From: Matios102 Date: Mon, 29 Dec 2025 15:16:19 +0100 Subject: [PATCH 11/11] remove task cache volume and cache metadata file --- docker-compose.yaml | 2 - internal/storage/cache.go | 61 ++-------------------- internal/storage/cache_test.go | 92 ---------------------------------- 3 files changed, 3 insertions(+), 152 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 3f8479a..03f377a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -15,7 +15,6 @@ services: container_name: go_app volumes: - /var/run/docker.sock:/var/run/docker.sock - - worker_cache:/tmp/worker-cache environment: - RABBITMQ_HOST=rabbitmq - RABBITMQ_USER=guest @@ -33,4 +32,3 @@ services: volumes: rabbitmq_data: - worker_cache: diff --git a/internal/storage/cache.go b/internal/storage/cache.go index 5aa5cfc..8bedb74 100644 --- a/internal/storage/cache.go +++ b/internal/storage/cache.go @@ -3,7 +3,6 @@ package storage import ( "crypto/sha256" "encoding/hex" - "encoding/json" "fmt" "os" "path/filepath" @@ -12,6 +11,7 @@ import ( "github.com/mini-maxit/worker/internal/logger" "github.com/mini-maxit/worker/pkg/constants" "github.com/mini-maxit/worker/pkg/messages" + "github.com/mini-maxit/worker/utils" "go.uber.org/zap" ) @@ -38,7 +38,6 @@ type fileCache struct { cacheDirPath string ttl time.Duration metadata *CacheMetadata - metadataPath string } func NewFileCache(cacheDirPath string) FileCache { @@ -48,21 +47,15 @@ func NewFileCache(cacheDirPath string) FileCache { cacheDirPath: cacheDirPath, ttl: time.Duration(constants.CacheTTLHours) * time.Hour, metadata: &CacheMetadata{Entries: make(map[string]CacheEntry)}, - metadataPath: filepath.Join(cacheDirPath, constants.CacheMetadataFile), } } -// InitCache initializes the cache directory and loads metadata. +// InitCache initializes the cache directory. func (c *fileCache) InitCache() error { if err := os.MkdirAll(c.cacheDirPath, 0755); err != nil { return fmt.Errorf("failed to create cache directory: %w", err) } - if err := c.loadMetadata(); err != nil { - c.logger.Warnf("Failed to load cache metadata, starting fresh: %v", err) - c.metadata = &CacheMetadata{Entries: make(map[string]CacheEntry)} - } - if err := c.CleanExpiredCache(); err != nil { c.logger.Warnf("Failed to clean expired cache: %v", err) } @@ -82,7 +75,6 @@ func (c *fileCache) GetCachedFile(fileLocation messages.FileLocation) (string, b if time.Since(entry.CachedAt) > c.ttl { c.logger.Debugf("Cache expired for %s", fileLocation.Path) delete(c.metadata.Entries, key) - _ = c.saveMetadata() return "", false, nil } @@ -90,7 +82,6 @@ func (c *fileCache) GetCachedFile(fileLocation messages.FileLocation) (string, b if _, err := os.Stat(entry.FilePath); os.IsNotExist(err) { c.logger.Debugf("Cached file no longer exists: %s", entry.FilePath) delete(c.metadata.Entries, key) - _ = c.saveMetadata() return "", false, nil } @@ -118,7 +109,7 @@ func (c *fileCache) CacheFile(fileLocation messages.FileLocation, sourcePath str cacheFilePath := filepath.Join(c.cacheDirPath, cacheFileName) // Copy the file to cache. - if err := c.copyFile(sourcePath, cacheFilePath); err != nil { + if err := utils.CopyFile(sourcePath, cacheFilePath); err != nil { return fmt.Errorf("failed to copy file to cache: %w", err) } @@ -130,10 +121,6 @@ func (c *fileCache) CacheFile(fileLocation messages.FileLocation, sourcePath str OriginalBucket: fileLocation.Bucket, } - if err := c.saveMetadata(); err != nil { - c.logger.Warnf("Failed to save cache metadata: %v", err) - } - c.logger.Debugf("Cached file %s", fileLocation.Path) return nil } @@ -158,7 +145,6 @@ func (c *fileCache) CleanExpiredCache() error { if len(toDelete) > 0 { c.logger.Infof("Cleaned %d expired cache entries", len(toDelete)) - return c.saveMetadata() } return nil @@ -203,44 +189,3 @@ func (c *fileCache) generateCacheFileName(fileLocation messages.FileLocation) st ext := filepath.Ext(fileLocation.Path) return fmt.Sprintf("%s%s", key, ext) } - -func (c *fileCache) copyFile(src, dst string) error { - sourceFile, err := os.Open(src) - if err != nil { - return err - } - defer sourceFile.Close() - - destFile, err := os.Create(dst) - if err != nil { - return err - } - defer destFile.Close() - - if _, err := destFile.ReadFrom(sourceFile); err != nil { - return err - } - - return destFile.Sync() -} - -func (c *fileCache) loadMetadata() error { - data, err := os.ReadFile(c.metadataPath) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - - return json.Unmarshal(data, c.metadata) -} - -func (c *fileCache) saveMetadata() error { - data, err := json.MarshalIndent(c.metadata, "", " ") - if err != nil { - return err - } - - return os.WriteFile(c.metadataPath, data, 0644) -} diff --git a/internal/storage/cache_test.go b/internal/storage/cache_test.go index ce7ba2a..019a737 100644 --- a/internal/storage/cache_test.go +++ b/internal/storage/cache_test.go @@ -1,7 +1,6 @@ package storage_test import ( - "encoding/json" "fmt" "os" "path/filepath" @@ -9,7 +8,6 @@ import ( "time" "github.com/mini-maxit/worker/internal/storage" - "github.com/mini-maxit/worker/pkg/constants" "github.com/mini-maxit/worker/pkg/messages" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -127,59 +125,6 @@ func TestFileCache_GetCachedFile_Success(t *testing.T) { require.NoError(t, err) } -func TestFileCache_CleanExpiredCache(t *testing.T) { - cachedir := t.TempDir() - cache := storage.NewFileCache(cachedir) - - err := cache.InitCache() - require.NoError(t, err) - - // Create and cache a file - testContent := "expired cache test" - testFile := createTestFile(t, testContent) - defer os.Remove(testFile) - - fileLocation := messages.FileLocation{ - Bucket: "test-bucket", - Path: "test/expired.txt", - } - - err = cache.CacheFile(fileLocation, testFile) - require.NoError(t, err) - - // Manually update metadata to simulate expired cache - metadataPath := filepath.Join(cachedir, constants.CacheMetadataFile) - data, err := os.ReadFile(metadataPath) - require.NoError(t, err) - - var metadata storage.CacheMetadata - err = json.Unmarshal(data, &metadata) - require.NoError(t, err) - - // Set all entries to be expired (more than 24 hours old) - for key, entry := range metadata.Entries { - entry.CachedAt = time.Now().Add(-25 * time.Hour) - metadata.Entries[key] = entry - } - - // Save modified metadata - modifiedData, err := json.MarshalIndent(metadata, "", " ") - require.NoError(t, err) - err = os.WriteFile(metadataPath, modifiedData, 0644) - require.NoError(t, err) - - // Create new cache instance to reload metadata - cache2 := storage.NewFileCache(cachedir) - err = cache2.InitCache() - require.NoError(t, err) - - // Try to get the file - should not be found as it was cleaned - cachedPath, found, err := cache2.GetCachedFile(fileLocation) - require.NoError(t, err) - assert.False(t, found) - assert.Empty(t, cachedPath) -} - func TestFileCache_MultipleCacheEntries(t *testing.T) { cachedir := t.TempDir() cache := storage.NewFileCache(cachedir) @@ -370,43 +315,6 @@ func TestFileCache_GetCachedFile_FileDeleted(t *testing.T) { assert.Empty(t, cachedPath2) } -func TestFileCache_InitCache_WithExistingMetadata(t *testing.T) { - cachedir := t.TempDir() - cache := storage.NewFileCache(cachedir) - - // First initialization - err := cache.InitCache() - require.NoError(t, err) - - // Cache a file - testContent := "persistent metadata test" - testFile := createTestFile(t, testContent) - defer os.Remove(testFile) - - fileLocation := messages.FileLocation{ - Bucket: "test-bucket", - Path: "test/persistent.txt", - } - - err = cache.CacheFile(fileLocation, testFile) - require.NoError(t, err) - - // Create a new cache instance (simulating restart) - cache2 := storage.NewFileCache(cachedir) - err = cache2.InitCache() - require.NoError(t, err) - - // Verify the cached file is still available - cachedPath, found, err := cache2.GetCachedFile(fileLocation) - require.NoError(t, err) - assert.True(t, found) - assert.NotEmpty(t, cachedPath) - - content, err := os.ReadFile(cachedPath) - require.NoError(t, err) - assert.Equal(t, testContent, string(content)) -} - func TestFileCache_CacheFile_InvalidSourcePath(t *testing.T) { cachedir := t.TempDir() cache := storage.NewFileCache(cachedir)