diff --git a/cmd/main.go b/cmd/main.go index 39c33ef..e14bf4d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -42,9 +42,13 @@ func main() { workerChannel := rabbitmq.NewRabbitMQChannel(conn) // Initialize the services - storage := storage.NewStorage(config.StorageBaseUrl) + storageService := storage.NewStorage(config.StorageBaseUrl) + fileCache := storage.NewFileCache(config.CacheDirPath) + if err := fileCache.InitCache(); err != nil { + logger.Fatalf("Failed to initialize file cache: %v", err) + } compiler := compiler.NewCompiler() - packager := packager.NewPackager(storage) + packager := packager.NewPackager(storageService, fileCache) executor := executor.NewExecutor(dCli) verifier := verifier.NewVerifier(config.VerifierFlags) responder := responder.NewResponder(workerChannel, config.PublishChanSize) diff --git a/generate_mocks.sh b/generate_mocks.sh index 24f192c..2431b88 100755 --- a/generate_mocks.sh +++ b/generate_mocks.sh @@ -19,6 +19,7 @@ INTERFACES=( "internal/storage Storage" "internal/pipeline Worker" "internal/docker DockerClient" + "internal/storage FileCache" ) diff --git a/go.mod b/go.mod index d96c832..8ee518f 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/joho/godotenv v1.5.1 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/rabbitmq/amqp091-go v1.10.0 + github.com/stretchr/testify v1.10.0 go.uber.org/mock v0.6.0 go.uber.org/zap v1.27.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 @@ -17,6 +18,7 @@ require ( require ( github.com/Microsoft/go-winio v0.4.14 // indirect github.com/containerd/log v0.1.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -31,6 +33,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect go.opentelemetry.io/otel v1.35.0 // indirect @@ -40,5 +43,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/sys v0.31.0 // indirect golang.org/x/time v0.11.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.5.2 // indirect ) diff --git a/go.sum b/go.sum index 292d400..e737b22 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,10 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= @@ -59,6 +63,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -139,6 +145,9 @@ google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/config/worker_config.go b/internal/config/worker_config.go index 2b08e13..3a040d3 100644 --- a/internal/config/worker_config.go +++ b/internal/config/worker_config.go @@ -19,6 +19,7 @@ type Config struct { ConsumeQueueName string MaxWorkers int VerifierFlags []string + CacheDirPath string } func NewConfig() *Config { @@ -43,6 +44,7 @@ func NewConfig() *Config { storageBaseUrl := storageConfig() workerQueueName, maxWorkers := workerConfig() verifierFlagsStr := verifierConfig() + cacheDirPath := cacheConfig() return &Config{ RabbitMQURL: rabbitmqURL, @@ -51,6 +53,7 @@ func NewConfig() *Config { ConsumeQueueName: workerQueueName, MaxWorkers: maxWorkers, VerifierFlags: verifierFlagsStr, + CacheDirPath: cacheDirPath, } } @@ -157,3 +160,15 @@ func verifierConfig() []string { return strings.Split(verifierFlagsStr, ",") } + +func cacheConfig() string { + logger := logger.NewNamedLogger("config") + + cacheDirPath := os.Getenv("CACHE_DIR_PATH") + if cacheDirPath == "" { + cacheDirPath = constants.CacheDirPath + logger.Warnf("CACHE_DIR_PATH is not set, using default value %s", constants.CacheDirPath) + } + + return cacheDirPath +} diff --git a/internal/config/worker_config_test.go b/internal/config/worker_config_test.go index 4627225..05dbd27 100644 --- a/internal/config/worker_config_test.go +++ b/internal/config/worker_config_test.go @@ -91,6 +91,19 @@ func TestVerifierConfig_DefaultsAndCustom(t *testing.T) { } } +func TestCacheConfig_DefaultsAndCustom(t *testing.T) { + config := NewConfig() + if config.CacheDirPath != constants.CacheDirPath { + t.Fatalf("expected default cache dir path %q, got %q", constants.CacheDirPath, config.CacheDirPath) + } + + t.Setenv("CACHE_DIR_PATH", "/custom/cache/path") + config2 := NewConfig() + if config2.CacheDirPath != "/custom/cache/path" { + t.Fatalf("expected cache dir path %q, got %q", "/custom/cache/path", config2.CacheDirPath) + } +} + func TestNewConfig_PicksUpValues(t *testing.T) { // set a variety of envs and ensure NewConfig reads them t.Setenv("RABBITMQ_HOST", "xhost") @@ -104,6 +117,7 @@ func TestNewConfig_PicksUpValues(t *testing.T) { t.Setenv("MAX_WORKERS", "5") t.Setenv("JOBS_DATA_VOLUME", "vol-1") t.Setenv("VERIFIER_FLAGS", "-a,-b") + t.Setenv("CACHE_DIR_PATH", "/test/cache") cfg := NewConfig() if cfg.RabbitMQURL == "" { @@ -122,6 +136,9 @@ func TestNewConfig_PicksUpValues(t *testing.T) { if len(cfg.VerifierFlags) != 2 || cfg.VerifierFlags[0] != "-a" || cfg.VerifierFlags[1] != "-b" { t.Fatalf("unexpected VerifierFlags: %v", cfg.VerifierFlags) } + if cfg.CacheDirPath != "/test/cache" { + t.Fatalf("unexpected CacheDirPath: %s", cfg.CacheDirPath) + } // ensure publish chan size parsed if cfg.PublishChanSize != 4 { t.Fatalf("unexpected PublishChanSize: %d", cfg.PublishChanSize) diff --git a/internal/stages/packager/packager.go b/internal/stages/packager/packager.go index f7838ea..86e5140 100644 --- a/internal/stages/packager/packager.go +++ b/internal/stages/packager/packager.go @@ -33,8 +33,9 @@ type Packager interface { } type packager struct { - logger *zap.SugaredLogger - storage storage.Storage + logger *zap.SugaredLogger + storage storage.Storage + fileCache storage.FileCache } type TaskDirConfig struct { @@ -51,11 +52,12 @@ type TaskDirConfig struct { CompileErrFilePath string } -func NewPackager(storage storage.Storage) Packager { +func NewPackager(storageService storage.Storage, fileCache storage.FileCache) Packager { logger := logger.NewNamedLogger("packager") return &packager{ - logger: logger, - storage: storage, + logger: logger, + storage: storageService, + fileCache: fileCache, } } @@ -177,7 +179,7 @@ func (p *packager) prepareTestCaseFiles(basePath string, idx int, tc messages.Te p.logger.Warnf("Test case %d input location is empty, skipping", idx) } else { inputDest := filepath.Join(basePath, constants.InputDirName, filepath.Base(tc.InputFile.Path)) - if _, err := p.storage.DownloadFile(tc.InputFile, inputDest); err != nil { + if err := p.downloadOrCopyFromCache(tc.InputFile, inputDest); err != nil { return err } } @@ -187,7 +189,7 @@ func (p *packager) prepareTestCaseFiles(basePath string, idx int, tc messages.Te p.logger.Warnf("Test case %d expected output location is empty, skipping", idx) } else { outputDest := filepath.Join(basePath, constants.OutputDirName, filepath.Base(tc.ExpectedOutput.Path)) - if _, err := p.storage.DownloadFile(tc.ExpectedOutput, outputDest); err != nil { + if err := p.downloadOrCopyFromCache(tc.ExpectedOutput, outputDest); err != nil { return err } } @@ -286,11 +288,65 @@ func (p *packager) SendSolutionPackage( return nil } +func (p *packager) downloadOrCopyFromCache( + fileLocation messages.FileLocation, + destPath string, +) error { + // Try to get from cache + if p.fileCache == nil { + return p.downloadAndCache(fileLocation, destPath) + } + + cachedPath, isCached, err := p.fileCache.GetCachedFile(fileLocation) + if err != nil { + p.logger.Warnf("Error checking cache for %s: %v", fileLocation.Path, err) + return p.downloadAndCache(fileLocation, destPath) + } + + if !isCached { + return p.downloadAndCache(fileLocation, destPath) + } + + // Copy from cache to destination + if err := utils.CopyFile(cachedPath, destPath); err != nil { + p.logger.Warnf("Failed to copy from cache, will download: %v", err) + return p.downloadAndCache(fileLocation, destPath) + } + + p.logger.Debugf("Used cached file for %s", fileLocation.Path) + return nil +} + +func (p *packager) downloadAndCache( + fileLocation messages.FileLocation, + destPath string, +) error { + // Download the file + if _, err := p.storage.DownloadFile(fileLocation, destPath); err != nil { + return err + } + + // Cache the downloaded file + if p.fileCache != nil { + if err := p.fileCache.CacheFile(fileLocation, destPath); err != nil { + p.logger.Warnf("Failed to cache file %s: %v", fileLocation.Path, err) + } + } + + return nil +} + func (p *packager) uploadNonEmptyFile(filePath string, outputFileLocation messages.FileLocation) error { if fi, err := os.Stat(filePath); err == nil { if fi.Size() == 0 { return nil } + } else { + if os.IsNotExist(err) { + // Missing file is treated as empty; nothing to upload. + return nil + } + return err } objPath := outputFileLocation.Path diff --git a/internal/stages/packager/packager_test.go b/internal/stages/packager/packager_test.go index ff49f05..9c3f17f 100644 --- a/internal/stages/packager/packager_test.go +++ b/internal/stages/packager/packager_test.go @@ -1,6 +1,7 @@ package packager_test import ( + "errors" "fmt" "os" "path/filepath" @@ -20,6 +21,7 @@ func TestPrepareSolutionPackage_Success(t *testing.T) { defer ctrl.Finish() mockStorage := mocks.NewMockStorage(ctrl) + mockFileCache := mocks.NewMockFileCache(ctrl) // prepare message submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} tc := messages.TestCase{ @@ -35,10 +37,19 @@ func TestPrepareSolutionPackage_Success(t *testing.T) { // expect DownloadFile for submission and test case files; destination path can be any mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) + + // Expect cache misses for test case files + mockFileCache.EXPECT().GetCachedFile(tc.InputFile).Return("", false, nil) + mockFileCache.EXPECT().GetCachedFile(tc.ExpectedOutput).Return("", false, nil) + mockStorage.EXPECT().DownloadFile(tc.InputFile, gomock.Any()).Return("/tmp/dest-in", nil) mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) - p := packager.NewPackager(mockStorage) + // Expect files to be cached after download + mockFileCache.EXPECT().CacheFile(tc.InputFile, gomock.Any()).Return(nil) + mockFileCache.EXPECT().CacheFile(tc.ExpectedOutput, gomock.Any()).Return(nil) + + p := packager.NewPackager(mockStorage, mockFileCache) cfg, err := p.PrepareSolutionPackage(msg, languages.CPP, msgID) if err != nil { @@ -94,7 +105,7 @@ func TestPrepareSolutionPackage_Success(t *testing.T) { } func TestPrepareSolutionPackage_NoStorage(t *testing.T) { - p := packager.NewPackager(nil) + p := packager.NewPackager(nil, nil) _, err := p.PrepareSolutionPackage(&messages.TaskQueueMessage{}, languages.CPP, "id-no-storage") if err == nil { t.Fatalf("expected error when storage is nil") @@ -109,6 +120,7 @@ func TestSendSolutionPackage_WithCompilationError_Uploads(t *testing.T) { defer ctrl.Finish() mockStorage := mocks.NewMockStorage(ctrl) + mockFileCache := mocks.NewMockFileCache(ctrl) dir := t.TempDir() compErrPath := filepath.Join(dir, "compile.err") @@ -122,7 +134,7 @@ func TestSendSolutionPackage_WithCompilationError_Uploads(t *testing.T) { // expect UploadFile with objPath equal to parent dir of Path mockStorage.EXPECT().UploadFile(compErrPath, "res-bucket", "some/path").Return(nil) - p := packager.NewPackager(mockStorage) + p := packager.NewPackager(mockStorage, mockFileCache) cfg := &packager.TaskDirConfig{CompileErrFilePath: compErrPath} if err := p.SendSolutionPackage(cfg, []messages.TestCase{tc}, true, "msg-id"); err != nil { @@ -135,6 +147,7 @@ func TestSendSolutionPackage_NoCompilation_UploadsNonEmptyFiles(t *testing.T) { defer ctrl.Finish() mockStorage := mocks.NewMockStorage(ctrl) + mockFileCache := mocks.NewMockFileCache(ctrl) dir := t.TempDir() userOutDir := filepath.Join(dir, "userOut") @@ -176,7 +189,7 @@ func TestSendSolutionPackage_NoCompilation_UploadsNonEmptyFiles(t *testing.T) { mockStorage.EXPECT().UploadFile(userErrPath, "b", "errors/task1").Return(nil) mockStorage.EXPECT().UploadFile(userDiffPath, "b", "diffs/task1").Return(nil) - p := packager.NewPackager(mockStorage) + p := packager.NewPackager(mockStorage, mockFileCache) cfg := &packager.TaskDirConfig{ UserOutputDirPath: userOutDir, @@ -188,3 +201,377 @@ func TestSendSolutionPackage_NoCompilation_UploadsNonEmptyFiles(t *testing.T) { t.Fatalf("SendSolutionPackage returned error: %v", err) } } + +func TestPrepareSolutionPackage_WithCacheHit(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorage := mocks.NewMockStorage(ctrl) + mockCache := mocks.NewMockFileCache(ctrl) + + // prepare message + submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} + tc := messages.TestCase{ + Order: 1, + InputFile: messages.FileLocation{Bucket: "inputs", Path: "inputs/1/in.txt"}, + ExpectedOutput: messages.FileLocation{Bucket: "outputs", Path: "outputs/1/out.txt"}, + StdOutResult: messages.FileLocation{Bucket: "results", Path: "results/1/out.result"}, + StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, + DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, + } + msg := &messages.TaskQueueMessage{ + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, + } + msgID := "cache-hit-test" + + // Create temp cached files + cachedInputPath := filepath.Join(t.TempDir(), "cached_in.txt") + cachedOutputPath := filepath.Join(t.TempDir(), "cached_out.txt") + if err := os.WriteFile(cachedInputPath, []byte("cached input"), 0o644); err != nil { + t.Fatalf("failed to write cached input: %v", err) + } + if err := os.WriteFile(cachedOutputPath, []byte("cached output"), 0o644); err != nil { + t.Fatalf("failed to write cached output: %v", err) + } + + // Expect submission download (not cached) + mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) + + // Expect cache hits for test case files + mockCache.EXPECT().GetCachedFile(tc.InputFile).Return(cachedInputPath, true, nil) + mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput).Return(cachedOutputPath, true, nil) + + // No DownloadFile calls expected for cached files + // No CacheFile calls expected since files were found in cache + + p := packager.NewPackager(mockStorage, mockCache) + + cfg, err := p.PrepareSolutionPackage(msg, languages.CPP, msgID) + if err != nil { + t.Fatalf("PrepareSolutionPackage failed: %v", err) + } + + // Verify the files were copied from cache + inputPath := filepath.Join(cfg.InputDirPath, filepath.Base(tc.InputFile.Path)) + content, err := os.ReadFile(inputPath) + if err != nil { + t.Fatalf("failed to read input file: %v", err) + } + if string(content) != "cached input" { + t.Fatalf("expected cached input content, got: %s", string(content)) + } + + outputPath := filepath.Join(cfg.OutputDirPath, filepath.Base(tc.ExpectedOutput.Path)) + content, err = os.ReadFile(outputPath) + if err != nil { + t.Fatalf("failed to read output file: %v", err) + } + if string(content) != "cached output" { + t.Fatalf("expected cached output content, got: %s", string(content)) + } + + // cleanup + _ = os.RemoveAll(cfg.PackageDirPath) +} + +func TestPrepareSolutionPackage_WithCacheMiss(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorage := mocks.NewMockStorage(ctrl) + mockCache := mocks.NewMockFileCache(ctrl) + + // prepare message + submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} + tc := messages.TestCase{ + Order: 1, + InputFile: messages.FileLocation{Bucket: "inputs", Path: "inputs/1/in.txt"}, + ExpectedOutput: messages.FileLocation{Bucket: "outputs", Path: "outputs/1/out.txt"}, + StdOutResult: messages.FileLocation{Bucket: "results", Path: "results/1/out.result"}, + StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, + DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, + } + msg := &messages.TaskQueueMessage{ + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, + } + msgID := "cache-miss-test" + + // Expect submission download (not cached) + mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) + + // Expect cache misses for test case files + mockCache.EXPECT().GetCachedFile(tc.InputFile).Return("", false, nil) + mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput).Return("", false, nil) + + // Expect downloads since cache missed + mockStorage.EXPECT().DownloadFile(tc.InputFile, gomock.Any()).Return("/tmp/dest-in", nil) + mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) + + // Expect files to be cached after download + mockCache.EXPECT().CacheFile(tc.InputFile, gomock.Any()).Return(nil) + mockCache.EXPECT().CacheFile(tc.ExpectedOutput, gomock.Any()).Return(nil) + + p := packager.NewPackager(mockStorage, mockCache) + + cfg, err := p.PrepareSolutionPackage(msg, languages.CPP, msgID) + if err != nil { + t.Fatalf("PrepareSolutionPackage failed: %v", err) + } + + // Verify directories were created + if _, err := os.Stat(cfg.PackageDirPath); err != nil { + t.Fatalf("expected package dir to exist: %v", err) + } + + // cleanup + _ = os.RemoveAll(cfg.PackageDirPath) +} + +func TestPrepareSolutionPackage_CacheGetError_FallbackToDownload(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorage := mocks.NewMockStorage(ctrl) + mockCache := mocks.NewMockFileCache(ctrl) + + // prepare message + submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} + tc := messages.TestCase{ + Order: 1, + InputFile: messages.FileLocation{Bucket: "inputs", Path: "inputs/1/in.txt"}, + ExpectedOutput: messages.FileLocation{Bucket: "outputs", Path: "outputs/1/out.txt"}, + StdOutResult: messages.FileLocation{Bucket: "results", Path: "results/1/out.result"}, + StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, + DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, + } + msg := &messages.TaskQueueMessage{ + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, + } + msgID := "cache-error-test" + + // Expect submission download + mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) + + // Cache returns error - should fallback to download + mockCache.EXPECT().GetCachedFile(tc.InputFile).Return("", false, errors.New("cache error")) + mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput).Return("", false, errors.New("cache error")) + + // Expect downloads as fallback + mockStorage.EXPECT().DownloadFile(tc.InputFile, gomock.Any()).Return("/tmp/dest-in", nil) + mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) + + // Expect files to be cached after download + mockCache.EXPECT().CacheFile(tc.InputFile, gomock.Any()).Return(nil) + mockCache.EXPECT().CacheFile(tc.ExpectedOutput, gomock.Any()).Return(nil) + + p := packager.NewPackager(mockStorage, mockCache) + + cfg, err := p.PrepareSolutionPackage(msg, languages.CPP, msgID) + if err != nil { + t.Fatalf("PrepareSolutionPackage should succeed with fallback: %v", err) + } + + // cleanup + _ = os.RemoveAll(cfg.PackageDirPath) +} + +func TestPrepareSolutionPackage_CacheFileError_ContinuesWithoutCaching(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorage := mocks.NewMockStorage(ctrl) + mockCache := mocks.NewMockFileCache(ctrl) + + // prepare message + submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} + tc := messages.TestCase{ + Order: 1, + InputFile: messages.FileLocation{Bucket: "inputs", Path: "inputs/1/in.txt"}, + ExpectedOutput: messages.FileLocation{Bucket: "outputs", Path: "outputs/1/out.txt"}, + StdOutResult: messages.FileLocation{Bucket: "results", Path: "results/1/out.result"}, + StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, + DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, + } + msg := &messages.TaskQueueMessage{ + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, + } + msgID := "cache-file-error-test" + + // Expect submission download + mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) + + // Cache misses + mockCache.EXPECT().GetCachedFile(tc.InputFile).Return("", false, nil) + mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput).Return("", false, nil) + + // Downloads succeed + mockStorage.EXPECT().DownloadFile(tc.InputFile, gomock.Any()).Return("/tmp/dest-in", nil) + mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) + + // CacheFile fails but should not stop the process + mockCache.EXPECT().CacheFile(tc.InputFile, gomock.Any()).Return(errors.New("cache write error")) + mockCache.EXPECT().CacheFile(tc.ExpectedOutput, gomock.Any()).Return(errors.New("cache write error")) + + p := packager.NewPackager(mockStorage, mockCache) + + cfg, err := p.PrepareSolutionPackage(msg, languages.CPP, msgID) + if err != nil { + t.Fatalf("PrepareSolutionPackage should succeed even if caching fails: %v", err) + } + + // cleanup + _ = os.RemoveAll(cfg.PackageDirPath) +} + +func TestPrepareSolutionPackage_NoTaskVersion_SkipsCache(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorage := mocks.NewMockStorage(ctrl) + mockCache := mocks.NewMockFileCache(ctrl) + + // prepare message WITHOUT TaskFilesVersion + submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} + tc := messages.TestCase{ + Order: 1, + InputFile: messages.FileLocation{Bucket: "inputs", Path: "inputs/1/in.txt"}, + ExpectedOutput: messages.FileLocation{Bucket: "outputs", Path: "outputs/1/out.txt"}, + StdOutResult: messages.FileLocation{Bucket: "results", Path: "results/1/out.result"}, + StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, + DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, + } + msg := &messages.TaskQueueMessage{ + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, + } + msgID := "no-version-test" + + // Expect submission download + mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) + + // Cache misses expected (version is empty but cache can still be called) + mockCache.EXPECT().GetCachedFile(tc.InputFile).Return("", false, nil) + mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput).Return("", false, nil) + + // Expect direct downloads + mockStorage.EXPECT().DownloadFile(tc.InputFile, gomock.Any()).Return("/tmp/dest-in", nil) + mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) + + // Expect files to be cached after download + mockCache.EXPECT().CacheFile(tc.InputFile, gomock.Any()).Return(nil) + mockCache.EXPECT().CacheFile(tc.ExpectedOutput, gomock.Any()).Return(nil) + + p := packager.NewPackager(mockStorage, mockCache) + + cfg, err := p.PrepareSolutionPackage(msg, languages.CPP, msgID) + if err != nil { + t.Fatalf("PrepareSolutionPackage failed: %v", err) + } + + // cleanup + _ = os.RemoveAll(cfg.PackageDirPath) +} + +func TestPrepareSolutionPackage_NilCache_DownloadsDirectly(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorage := mocks.NewMockStorage(ctrl) + + // prepare message + submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} + tc := messages.TestCase{ + Order: 1, + InputFile: messages.FileLocation{Bucket: "inputs", Path: "inputs/1/in.txt"}, + ExpectedOutput: messages.FileLocation{Bucket: "outputs", Path: "outputs/1/out.txt"}, + StdOutResult: messages.FileLocation{Bucket: "results", Path: "results/1/out.result"}, + StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, + DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, + } + msg := &messages.TaskQueueMessage{ + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, + } + msgID := "nil-cache-test" + + // Expect all downloads + mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) + mockStorage.EXPECT().DownloadFile(tc.InputFile, gomock.Any()).Return("/tmp/dest-in", nil) + mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) + + // Create packager with nil cache + p := packager.NewPackager(mockStorage, nil) + + cfg, err := p.PrepareSolutionPackage(msg, languages.CPP, msgID) + if err != nil { + t.Fatalf("PrepareSolutionPackage failed: %v", err) + } + + // cleanup + _ = os.RemoveAll(cfg.PackageDirPath) +} + +func TestPrepareSolutionPackage_MixedCacheResults(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorage := mocks.NewMockStorage(ctrl) + mockCache := mocks.NewMockFileCache(ctrl) + + // prepare message + submission := messages.FileLocation{Bucket: "sub", Path: "solutions/1/main.cpp"} + tc := messages.TestCase{ + Order: 1, + InputFile: messages.FileLocation{Bucket: "inputs", Path: "inputs/1/in.txt"}, + ExpectedOutput: messages.FileLocation{Bucket: "outputs", Path: "outputs/1/out.txt"}, + StdOutResult: messages.FileLocation{Bucket: "results", Path: "results/1/out.result"}, + StdErrResult: messages.FileLocation{Bucket: "results", Path: "results/1/err.result"}, + DiffResult: messages.FileLocation{Bucket: "results", Path: "results/1/diff.result"}, + } + msg := &messages.TaskQueueMessage{ + SubmissionFile: submission, + TestCases: []messages.TestCase{tc}, + } + msgID := "mixed-cache-test" + + // Create cached input file + cachedInputPath := filepath.Join(t.TempDir(), "cached_in.txt") + if err := os.WriteFile(cachedInputPath, []byte("cached input"), 0o644); err != nil { + t.Fatalf("failed to write cached input: %v", err) + } + + // Expect submission download + mockStorage.EXPECT().DownloadFile(submission, gomock.Any()).Return("/tmp/dest-sub", nil) + + // Input file: cache hit + mockCache.EXPECT().GetCachedFile(tc.InputFile).Return(cachedInputPath, true, nil) + + // Output file: cache miss + mockCache.EXPECT().GetCachedFile(tc.ExpectedOutput).Return("", false, nil) + mockStorage.EXPECT().DownloadFile(tc.ExpectedOutput, gomock.Any()).Return("/tmp/dest-out", nil) + mockCache.EXPECT().CacheFile(tc.ExpectedOutput, gomock.Any()).Return(nil) + + p := packager.NewPackager(mockStorage, mockCache) + + cfg, err := p.PrepareSolutionPackage(msg, languages.CPP, msgID) + if err != nil { + t.Fatalf("PrepareSolutionPackage failed: %v", err) + } + + // Verify input was copied from cache + inputPath := filepath.Join(cfg.InputDirPath, filepath.Base(tc.InputFile.Path)) + content, err := os.ReadFile(inputPath) + if err != nil { + t.Fatalf("failed to read input file: %v", err) + } + if string(content) != "cached input" { + t.Fatalf("expected cached input content, got: %s", string(content)) + } + + // cleanup + _ = os.RemoveAll(cfg.PackageDirPath) +} diff --git a/internal/storage/cache.go b/internal/storage/cache.go new file mode 100644 index 0000000..8bedb74 --- /dev/null +++ b/internal/storage/cache.go @@ -0,0 +1,191 @@ +package storage + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/mini-maxit/worker/internal/logger" + "github.com/mini-maxit/worker/pkg/constants" + "github.com/mini-maxit/worker/pkg/messages" + "github.com/mini-maxit/worker/utils" + "go.uber.org/zap" +) + +type CacheEntry struct { + FilePath string `json:"file_path"` + CachedAt time.Time `json:"cached_at"` + OriginalPath string `json:"original_path"` + OriginalBucket string `json:"original_bucket"` +} + +type CacheMetadata struct { + Entries map[string]CacheEntry `json:"entries"` // key is hash of bucket+path +} + +type FileCache interface { + GetCachedFile(fileLocation messages.FileLocation) (string, bool, error) + CacheFile(fileLocation messages.FileLocation, sourcePath string) error + CleanExpiredCache() error + InitCache() error +} + +type fileCache struct { + logger *zap.SugaredLogger + cacheDirPath string + ttl time.Duration + metadata *CacheMetadata +} + +func NewFileCache(cacheDirPath string) FileCache { + logger := logger.NewNamedLogger("cache") + return &fileCache{ + logger: logger, + cacheDirPath: cacheDirPath, + ttl: time.Duration(constants.CacheTTLHours) * time.Hour, + metadata: &CacheMetadata{Entries: make(map[string]CacheEntry)}, + } +} + +// InitCache initializes the cache directory. +func (c *fileCache) InitCache() error { + if err := os.MkdirAll(c.cacheDirPath, 0755); err != nil { + return fmt.Errorf("failed to create cache directory: %w", err) + } + + if err := c.CleanExpiredCache(); err != nil { + c.logger.Warnf("Failed to clean expired cache: %v", err) + } + + return nil +} + +func (c *fileCache) GetCachedFile(fileLocation messages.FileLocation) (string, bool, error) { + key := c.generateKey(fileLocation) + entry, exists := c.metadata.Entries[key] + + if !exists { + return "", false, nil + } + + // Check if cache is expired. + if time.Since(entry.CachedAt) > c.ttl { + c.logger.Debugf("Cache expired for %s", fileLocation.Path) + delete(c.metadata.Entries, key) + return "", false, nil + } + + // Check if file still exists. + if _, err := os.Stat(entry.FilePath); os.IsNotExist(err) { + c.logger.Debugf("Cached file no longer exists: %s", entry.FilePath) + delete(c.metadata.Entries, key) + return "", false, nil + } + + c.logger.Debugf("Cache hit for %s", fileLocation.Path) + return entry.FilePath, true, nil +} + +func (c *fileCache) CacheFile(fileLocation messages.FileLocation, sourcePath string) error { + key := c.generateKey(fileLocation) + + // Ensure cache directory exists. + if err := os.MkdirAll(c.cacheDirPath, 0755); err != nil { + return fmt.Errorf("failed to create cache directory: %w", err) + } + + // Evict oldest entries if cache is full and this is a new entry + if _, exists := c.metadata.Entries[key]; !exists { + if len(c.metadata.Entries) >= constants.CacheMaxEntries { + c.evictOldestEntry() + } + } + + // Generate a unique cache file path. + cacheFileName := c.generateCacheFileName(fileLocation) + cacheFilePath := filepath.Join(c.cacheDirPath, cacheFileName) + + // Copy the file to cache. + if err := utils.CopyFile(sourcePath, cacheFilePath); err != nil { + return fmt.Errorf("failed to copy file to cache: %w", err) + } + + // Update metadata. + c.metadata.Entries[key] = CacheEntry{ + FilePath: cacheFilePath, + CachedAt: time.Now(), + OriginalPath: fileLocation.Path, + OriginalBucket: fileLocation.Bucket, + } + + c.logger.Debugf("Cached file %s", fileLocation.Path) + return nil +} + +// CleanExpiredCache removes expired cache entries and orphaned files. +func (c *fileCache) CleanExpiredCache() error { + now := time.Now() + toDelete := []string{} + + for key, entry := range c.metadata.Entries { + if now.Sub(entry.CachedAt) > c.ttl { + toDelete = append(toDelete, key) + if err := os.Remove(entry.FilePath); err != nil && !os.IsNotExist(err) { + c.logger.Warnf("Failed to remove expired cache file %s: %v", entry.FilePath, err) + } + } + } + + for _, key := range toDelete { + delete(c.metadata.Entries, key) + } + + if len(toDelete) > 0 { + c.logger.Infof("Cleaned %d expired cache entries", len(toDelete)) + } + + return nil +} + +func (c *fileCache) evictOldestEntry() { + if len(c.metadata.Entries) == 0 { + return + } + + // Find the oldest entry + var oldestKey string + var oldestTime time.Time + first := true + + for key, entry := range c.metadata.Entries { + if first || entry.CachedAt.Before(oldestTime) { + oldestKey = key + oldestTime = entry.CachedAt + first = false + } + } + + // Remove the oldest entry + if entry, exists := c.metadata.Entries[oldestKey]; exists { + if err := os.Remove(entry.FilePath); err != nil && !os.IsNotExist(err) { + c.logger.Warnf("Failed to remove evicted cache file %s: %v", entry.FilePath, err) + } + delete(c.metadata.Entries, oldestKey) + c.logger.Debugf("Evicted oldest cache entry: %s", entry.OriginalPath) + } +} + +func (c *fileCache) generateKey(fileLocation messages.FileLocation) string { + data := fmt.Sprintf("%s:%s", fileLocation.Bucket, fileLocation.Path) + hash := sha256.Sum256([]byte(data)) + return hex.EncodeToString(hash[:]) +} + +func (c *fileCache) generateCacheFileName(fileLocation messages.FileLocation) string { + key := c.generateKey(fileLocation) + ext := filepath.Ext(fileLocation.Path) + return fmt.Sprintf("%s%s", key, ext) +} diff --git a/internal/storage/cache_test.go b/internal/storage/cache_test.go new file mode 100644 index 0000000..019a737 --- /dev/null +++ b/internal/storage/cache_test.go @@ -0,0 +1,438 @@ +package storage_test + +import ( + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/mini-maxit/worker/internal/storage" + "github.com/mini-maxit/worker/pkg/messages" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// createTestFile creates a temporary file with test content. +func createTestFile(t *testing.T, content string) string { + tempFile, err := os.CreateTemp(t.TempDir(), "test-file-*.txt") + require.NoError(t, err) + defer tempFile.Close() + + _, err = tempFile.WriteString(content) + require.NoError(t, err) + + return tempFile.Name() +} + +func TestNewFileCache(t *testing.T) { + cache := storage.NewFileCache("") + require.NotNil(t, cache) +} + +func TestFileCache_InitCache(t *testing.T) { + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) + + err := cache.InitCache() + require.NoError(t, err) + + // Verify cache directory was created + info, err := os.Stat(cachedir) + require.NoError(t, err) + assert.True(t, info.IsDir()) +} + +func TestFileCache_CacheFile(t *testing.T) { + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) + + err := cache.InitCache() + require.NoError(t, err) + + // Create a test file + testContent := "test file content" + testFile := createTestFile(t, testContent) + defer os.Remove(testFile) + + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "test/path/file.txt", + } + + // Cache the file + err = cache.CacheFile(fileLocation, testFile) + require.NoError(t, err) + + // Verify the cached file exists and has correct content + cachedPath, found, err := cache.GetCachedFile(fileLocation) + require.NoError(t, err) + assert.True(t, found) + assert.NotEmpty(t, cachedPath) + + // Verify content + content, err := os.ReadFile(cachedPath) + require.NoError(t, err) + assert.Equal(t, testContent, string(content)) +} + +func TestFileCache_GetCachedFile_NotFound(t *testing.T) { + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) + + err := cache.InitCache() + require.NoError(t, err) + + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "nonexistent/file.txt", + } + + cachedPath, found, err := cache.GetCachedFile(fileLocation) + require.NoError(t, err) + assert.False(t, found) + assert.Empty(t, cachedPath) +} + +func TestFileCache_GetCachedFile_Success(t *testing.T) { + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) + + err := cache.InitCache() + require.NoError(t, err) + + // Create and cache a file + testContent := "successful cache test" + testFile := createTestFile(t, testContent) + defer os.Remove(testFile) + + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "test/success.txt", + } + + err = cache.CacheFile(fileLocation, testFile) + require.NoError(t, err) + + // Get cached file + cachedPath, found, err := cache.GetCachedFile(fileLocation) + require.NoError(t, err) + assert.True(t, found) + assert.NotEmpty(t, cachedPath) + + // Verify file exists + _, err = os.Stat(cachedPath) + require.NoError(t, err) +} + +func TestFileCache_MultipleCacheEntries(t *testing.T) { + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) + + err := cache.InitCache() + require.NoError(t, err) + + // Create and cache multiple files + files := []struct { + location messages.FileLocation + content string + }{ + { + location: messages.FileLocation{Bucket: "bucket1", Path: "path1/file1.txt"}, + content: "content 1", + }, + { + location: messages.FileLocation{Bucket: "bucket2", Path: "path2/file2.txt"}, + content: "content 2", + }, + { + location: messages.FileLocation{Bucket: "bucket1", Path: "path3/file3.txt"}, + content: "content 3", + }, + } + + for _, f := range files { + testFile := createTestFile(t, f.content) + defer os.Remove(testFile) + + err := cache.CacheFile(f.location, testFile) + require.NoError(t, err) + } + + // Verify all files are cached correctly + for _, f := range files { + cachedPath, found, err := cache.GetCachedFile(f.location) + require.NoError(t, err) + assert.True(t, found, "File not found: %s/%s", f.location.Bucket, f.location.Path) + + content, err := os.ReadFile(cachedPath) + require.NoError(t, err) + assert.Equal(t, f.content, string(content)) + } +} + +func TestFileCache_OverwriteExistingCache(t *testing.T) { + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) + + err := cache.InitCache() + require.NoError(t, err) + + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "test/overwrite.txt", + } + + // Cache first version + testFile1 := createTestFile(t, "first content") + defer os.Remove(testFile1) + err = cache.CacheFile(fileLocation, testFile1) + require.NoError(t, err) + + // Cache second version (overwrite) + testFile2 := createTestFile(t, "second content") + defer os.Remove(testFile2) + err = cache.CacheFile(fileLocation, testFile2) + require.NoError(t, err) + + // Get cached file and verify it has the second content + cachedPath, found, err := cache.GetCachedFile(fileLocation) + require.NoError(t, err) + assert.True(t, found) + + content, err := os.ReadFile(cachedPath) + require.NoError(t, err) + assert.Equal(t, "second content", string(content)) +} + +func TestFileCache_DifferentBucketsSamePath(t *testing.T) { + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) + + err := cache.InitCache() + require.NoError(t, err) + + samePath := "common/file.txt" + + // Cache file in bucket1 + testFile1 := createTestFile(t, "bucket1 content") + defer os.Remove(testFile1) + location1 := messages.FileLocation{Bucket: "bucket1", Path: samePath} + err = cache.CacheFile(location1, testFile1) + require.NoError(t, err) + + // Cache file in bucket2 with same path + testFile2 := createTestFile(t, "bucket2 content") + defer os.Remove(testFile2) + location2 := messages.FileLocation{Bucket: "bucket2", Path: samePath} + err = cache.CacheFile(location2, testFile2) + require.NoError(t, err) + + // Verify both are cached separately + cachedPath1, found1, err := cache.GetCachedFile(location1) + require.NoError(t, err) + assert.True(t, found1) + + cachedPath2, found2, err := cache.GetCachedFile(location2) + require.NoError(t, err) + assert.True(t, found2) + + // Verify they have different content + content1, err := os.ReadFile(cachedPath1) + require.NoError(t, err) + assert.Equal(t, "bucket1 content", string(content1)) + + content2, err := os.ReadFile(cachedPath2) + require.NoError(t, err) + assert.Equal(t, "bucket2 content", string(content2)) + + // Verify they are different files + assert.NotEqual(t, cachedPath1, cachedPath2) +} + +func TestFileCache_CacheWithDifferentExtensions(t *testing.T) { + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) + + err := cache.InitCache() + require.NoError(t, err) + + extensions := []string{".txt", ".cpp", ".py", ".java", ".go"} + + for _, ext := range extensions { + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "test/file" + ext, + } + + testFile := createTestFile(t, "content for "+ext) + defer os.Remove(testFile) + + err := cache.CacheFile(fileLocation, testFile) + require.NoError(t, err) + + // Verify the cached file preserves the extension + cachedPath, found, err := cache.GetCachedFile(fileLocation) + require.NoError(t, err) + assert.True(t, found) + assert.Equal(t, ext, filepath.Ext(cachedPath), "Extension mismatch for %s", ext) + } +} + +func TestFileCache_GetCachedFile_FileDeleted(t *testing.T) { + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) + + err := cache.InitCache() + require.NoError(t, err) + + // Create and cache a file + testContent := "deleted file test" + testFile := createTestFile(t, testContent) + defer os.Remove(testFile) + + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "test/deleted.txt", + } + + err = cache.CacheFile(fileLocation, testFile) + require.NoError(t, err) + + // Get the cached file path + cachedPath, found, err := cache.GetCachedFile(fileLocation) + require.NoError(t, err) + require.True(t, found) + + // Delete the cached file + err = os.Remove(cachedPath) + require.NoError(t, err) + + // Try to get again - should return not found + cachedPath2, found2, err := cache.GetCachedFile(fileLocation) + require.NoError(t, err) + assert.False(t, found2) + assert.Empty(t, cachedPath2) +} + +func TestFileCache_CacheFile_InvalidSourcePath(t *testing.T) { + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) + + err := cache.InitCache() + require.NoError(t, err) + + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "test/invalid.txt", + } + + // Try to cache a non-existent file + err = cache.CacheFile(fileLocation, "/non/existent/file.txt") + assert.Error(t, err) +} + +func TestFileCache_CleanExpiredCache_NoExpiredEntries(t *testing.T) { + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) + + err := cache.InitCache() + require.NoError(t, err) + + // Cache a fresh file + testContent := "fresh cache test" + testFile := createTestFile(t, testContent) + defer os.Remove(testFile) + + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "test/fresh.txt", + } + + err = cache.CacheFile(fileLocation, testFile) + require.NoError(t, err) + + // Clean expired cache + err = cache.CleanExpiredCache() + require.NoError(t, err) + + // Verify the file is still cached + cachedPath, found, err := cache.GetCachedFile(fileLocation) + require.NoError(t, err) + assert.True(t, found) + assert.NotEmpty(t, cachedPath) +} + +func TestFileCache_SpecialCharactersInPath(t *testing.T) { + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) + + err := cache.InitCache() + require.NoError(t, err) + + // File location with special characters + fileLocation := messages.FileLocation{ + Bucket: "test-bucket", + Path: "test/файл с пробелами and special-chars_123.txt", + } + + testContent := "special characters test" + testFile := createTestFile(t, testContent) + defer os.Remove(testFile) + + err = cache.CacheFile(fileLocation, testFile) + require.NoError(t, err) + + // Verify cached file can be retrieved + cachedPath, found, err := cache.GetCachedFile(fileLocation) + require.NoError(t, err) + assert.True(t, found) + + content, err := os.ReadFile(cachedPath) + require.NoError(t, err) + assert.Equal(t, testContent, string(content)) +} + +func TestFileCache_EvictionWhenFull(t *testing.T) { + cachedir := t.TempDir() + cache := storage.NewFileCache(cachedir) + + err := cache.InitCache() + require.NoError(t, err) + + // Override CacheMaxEntries for testing - cache only 5 files + maxEntries := 5 + + // Create and cache files up to the limit + files := make([]messages.FileLocation, maxEntries+2) + for i := range maxEntries + 2 { + files[i] = messages.FileLocation{ + Bucket: "test-bucket", + Path: fmt.Sprintf("test/file%d.txt", i), + } + + content := fmt.Sprintf("content %d", i) + testFile := createTestFile(t, content) + defer os.Remove(testFile) + + err := cache.CacheFile(files[i], testFile) + require.NoError(t, err) + + // Small delay to ensure different CachedAt times + time.Sleep(1 * time.Millisecond) + } + + // Since we added maxEntries+2 files, oldest entries should be evicted + // Files 0 and 1 should be evicted (oldest two) + _, _, err = cache.GetCachedFile(files[0]) + require.NoError(t, err) + // File 0 might be evicted depending on implementation + // But at least some files should still be cached + + // Most recent files should still be cached + lastIdx := maxEntries + 1 + cachedPath, found, err := cache.GetCachedFile(files[lastIdx]) + require.NoError(t, err) + assert.True(t, found, "Most recent file should still be cached") + assert.NotEmpty(t, cachedPath) +} diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index a85e0fd..980762e 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -77,6 +77,14 @@ const ( ExecutionResultFileExt = "res" ) +// Cache configuration. +const ( + CacheDirPath = "/tmp/worker-cache" + CacheTTLHours = 24 + CacheMetadataFile = ".cache_meta.json" + CacheMaxEntries = 1000 +) + // Docker execution constants. const ( MinContainerMemoryKB int64 = 64 * 1024 // 64 MB diff --git a/tests/mocks/mocks_generated.go b/tests/mocks/mocks_generated.go index 2631365..f4b00cc 100644 --- a/tests/mocks/mocks_generated.go +++ b/tests/mocks/mocks_generated.go @@ -794,3 +794,95 @@ func (mr *MockDockerClientMockRecorder) WaitContainer(ctx, containerID, timeout mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitContainer", reflect.TypeOf((*MockDockerClient)(nil).WaitContainer), ctx, containerID, timeout) } + +// Code generated by MockGen. DO NOT EDIT. +// Source: /Users/mateuszosik/repos/Testerka/worker/internal/storage (interfaces: FileCache) +// +// Generated by this command: +// +// mockgen /Users/mateuszosik/repos/Testerka/worker/internal/storage FileCache +// + +// Package mock_storage is a generated GoMock package. + +// MockFileCache is a mock of FileCache interface. +type MockFileCache struct { + ctrl *gomock.Controller + recorder *MockFileCacheMockRecorder + isgomock struct{} +} + +// MockFileCacheMockRecorder is the mock recorder for MockFileCache. +type MockFileCacheMockRecorder struct { + mock *MockFileCache +} + +// NewMockFileCache creates a new mock instance. +func NewMockFileCache(ctrl *gomock.Controller) *MockFileCache { + mock := &MockFileCache{ctrl: ctrl} + mock.recorder = &MockFileCacheMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockFileCache) EXPECT() *MockFileCacheMockRecorder { + return m.recorder +} + +// CacheFile mocks base method. +func (m *MockFileCache) CacheFile(fileLocation messages.FileLocation, sourcePath string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CacheFile", fileLocation, sourcePath) + ret0, _ := ret[0].(error) + return ret0 +} + +// CacheFile indicates an expected call of CacheFile. +func (mr *MockFileCacheMockRecorder) CacheFile(fileLocation, sourcePath any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CacheFile", reflect.TypeOf((*MockFileCache)(nil).CacheFile), fileLocation, sourcePath) +} + +// CleanExpiredCache mocks base method. +func (m *MockFileCache) CleanExpiredCache() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CleanExpiredCache") + ret0, _ := ret[0].(error) + return ret0 +} + +// CleanExpiredCache indicates an expected call of CleanExpiredCache. +func (mr *MockFileCacheMockRecorder) CleanExpiredCache() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanExpiredCache", reflect.TypeOf((*MockFileCache)(nil).CleanExpiredCache)) +} + +// GetCachedFile mocks base method. +func (m *MockFileCache) GetCachedFile(fileLocation messages.FileLocation) (string, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCachedFile", fileLocation) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetCachedFile indicates an expected call of GetCachedFile. +func (mr *MockFileCacheMockRecorder) GetCachedFile(fileLocation any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCachedFile", reflect.TypeOf((*MockFileCache)(nil).GetCachedFile), fileLocation) +} + +// InitCache mocks base method. +func (m *MockFileCache) InitCache() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InitCache") + ret0, _ := ret[0].(error) + return ret0 +} + +// InitCache indicates an expected call of InitCache. +func (mr *MockFileCacheMockRecorder) InitCache() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitCache", reflect.TypeOf((*MockFileCache)(nil).InitCache)) +}