From b52c2a779b6a230f18713a325042fa1dcff7907a Mon Sep 17 00:00:00 2001 From: Kudriashov Maksim Date: Sun, 2 Mar 2025 18:20:19 +0100 Subject: [PATCH] fix check-duplicates endpoint --- apispec/meme-storage/api.yml | 10 + apispec/meme-storage/client/api.gen.go | 105 +++++ apispec/meme-storage/server/api.gen.go | 74 ++++ .../src/service/ApiHandlerService.go | 385 ++++++++++-------- .../ElasticMetadataStorageServiceImpl.go | 55 ++- 5 files changed, 436 insertions(+), 193 deletions(-) diff --git a/apispec/meme-storage/api.yml b/apispec/meme-storage/api.yml index 8f7f680..192d7fa 100644 --- a/apispec/meme-storage/api.yml +++ b/apispec/meme-storage/api.yml @@ -21,6 +21,16 @@ paths: responses: 200: description: ok + + /api/v1/accounts/{AccountId}/update-ocr/{MemeId}: + post: + parameters: + - $ref: "#/components/parameters/AccountId" + - $ref: "#/components/parameters/MemeId" + operationId: "UpdateOcrOne" + responses: + 200: + description: ok /api/v1/accounts/{AccountId}/meme: get: diff --git a/apispec/meme-storage/client/api.gen.go b/apispec/meme-storage/client/api.gen.go index 214ae63..c446894 100644 --- a/apispec/meme-storage/client/api.gen.go +++ b/apispec/meme-storage/client/api.gen.go @@ -181,6 +181,9 @@ type ClientInterface interface { // UpdateOcr request UpdateOcr(ctx context.Context, accountId AccountId, reqEditors ...RequestEditorFn) (*http.Response, error) + + // UpdateOcrOne request + UpdateOcrOne(ctx context.Context, accountId AccountId, memeId MemeId, reqEditors ...RequestEditorFn) (*http.Response, error) } func (c *Client) CheckDuplicates(ctx context.Context, accountId AccountId, reqEditors ...RequestEditorFn) (*http.Response, error) { @@ -267,6 +270,18 @@ func (c *Client) UpdateOcr(ctx context.Context, accountId AccountId, reqEditors return c.Client.Do(req) } +func (c *Client) UpdateOcrOne(ctx context.Context, accountId AccountId, memeId MemeId, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewUpdateOcrOneRequest(c.Server, accountId, memeId) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + // NewCheckDuplicatesRequest generates requests for CheckDuplicates func NewCheckDuplicatesRequest(server string, accountId AccountId) (*http.Request, error) { var err error @@ -548,6 +563,47 @@ func NewUpdateOcrRequest(server string, accountId AccountId) (*http.Request, err return req, nil } +// NewUpdateOcrOneRequest generates requests for UpdateOcrOne +func NewUpdateOcrOneRequest(server string, accountId AccountId, memeId MemeId) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "AccountId", runtime.ParamLocationPath, accountId) + if err != nil { + return nil, err + } + + var pathParam1 string + + pathParam1, err = runtime.StyleParamWithLocation("simple", false, "MemeId", runtime.ParamLocationPath, memeId) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/api/v1/accounts/%s/update-ocr/%s", pathParam0, pathParam1) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + func (c *Client) applyEditors(ctx context.Context, req *http.Request, additionalEditors []RequestEditorFn) error { for _, r := range c.RequestEditors { if err := r(ctx, req); err != nil { @@ -610,6 +666,9 @@ type ClientWithResponsesInterface interface { // UpdateOcrWithResponse request UpdateOcrWithResponse(ctx context.Context, accountId AccountId, reqEditors ...RequestEditorFn) (*UpdateOcrResponse, error) + + // UpdateOcrOneWithResponse request + UpdateOcrOneWithResponse(ctx context.Context, accountId AccountId, memeId MemeId, reqEditors ...RequestEditorFn) (*UpdateOcrOneResponse, error) } type CheckDuplicatesResponse struct { @@ -742,6 +801,27 @@ func (r UpdateOcrResponse) StatusCode() int { return 0 } +type UpdateOcrOneResponse struct { + Body []byte + HTTPResponse *http.Response +} + +// Status returns HTTPResponse.Status +func (r UpdateOcrOneResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r UpdateOcrOneResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + // CheckDuplicatesWithResponse request returning *CheckDuplicatesResponse func (c *ClientWithResponses) CheckDuplicatesWithResponse(ctx context.Context, accountId AccountId, reqEditors ...RequestEditorFn) (*CheckDuplicatesResponse, error) { rsp, err := c.CheckDuplicates(ctx, accountId, reqEditors...) @@ -804,6 +884,15 @@ func (c *ClientWithResponses) UpdateOcrWithResponse(ctx context.Context, account return ParseUpdateOcrResponse(rsp) } +// UpdateOcrOneWithResponse request returning *UpdateOcrOneResponse +func (c *ClientWithResponses) UpdateOcrOneWithResponse(ctx context.Context, accountId AccountId, memeId MemeId, reqEditors ...RequestEditorFn) (*UpdateOcrOneResponse, error) { + rsp, err := c.UpdateOcrOne(ctx, accountId, memeId, reqEditors...) + if err != nil { + return nil, err + } + return ParseUpdateOcrOneResponse(rsp) +} + // ParseCheckDuplicatesResponse parses an HTTP response from a CheckDuplicatesWithResponse call func ParseCheckDuplicatesResponse(rsp *http.Response) (*CheckDuplicatesResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -939,3 +1028,19 @@ func ParseUpdateOcrResponse(rsp *http.Response) (*UpdateOcrResponse, error) { return response, nil } + +// ParseUpdateOcrOneResponse parses an HTTP response from a UpdateOcrOneWithResponse call +func ParseUpdateOcrOneResponse(rsp *http.Response) (*UpdateOcrOneResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &UpdateOcrOneResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + return response, nil +} diff --git a/apispec/meme-storage/server/api.gen.go b/apispec/meme-storage/server/api.gen.go index 76cc1df..d0baf32 100644 --- a/apispec/meme-storage/server/api.gen.go +++ b/apispec/meme-storage/server/api.gen.go @@ -107,6 +107,9 @@ type ServerInterface interface { // (POST /api/v1/accounts/{AccountId}/update-ocr) UpdateOcr(ctx echo.Context, accountId AccountId) error + + // (POST /api/v1/accounts/{AccountId}/update-ocr/{MemeId}) + UpdateOcrOne(ctx echo.Context, accountId AccountId, memeId MemeId) error } // ServerInterfaceWrapper converts echo contexts to parameters. @@ -249,6 +252,30 @@ func (w *ServerInterfaceWrapper) UpdateOcr(ctx echo.Context) error { return err } +// UpdateOcrOne converts echo context to params. +func (w *ServerInterfaceWrapper) UpdateOcrOne(ctx echo.Context) error { + var err error + // ------------- Path parameter "AccountId" ------------- + var accountId AccountId + + err = runtime.BindStyledParameterWithLocation("simple", false, "AccountId", runtime.ParamLocationPath, ctx.Param("AccountId"), &accountId) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter AccountId: %s", err)) + } + + // ------------- Path parameter "MemeId" ------------- + var memeId MemeId + + err = runtime.BindStyledParameterWithLocation("simple", false, "MemeId", runtime.ParamLocationPath, ctx.Param("MemeId"), &memeId) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter MemeId: %s", err)) + } + + // Invoke the callback with all the unmarshaled arguments + err = w.Handler.UpdateOcrOne(ctx, accountId, memeId) + return err +} + // This is a simple interface which specifies echo.Route addition functions which // are present on both echo.Echo and echo.Group, since we want to allow using // either of them for path registration @@ -283,6 +310,7 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL router.GET(baseURL+"/api/v1/accounts/:AccountId/meme/:MemeId/image/thumb/url", wrapper.GetMemeImageThumbUrl) router.GET(baseURL+"/api/v1/accounts/:AccountId/meme/:MemeId/image/url", wrapper.GetMemeImageUrl) router.POST(baseURL+"/api/v1/accounts/:AccountId/update-ocr", wrapper.UpdateOcr) + router.POST(baseURL+"/api/v1/accounts/:AccountId/update-ocr/:MemeId", wrapper.UpdateOcrOne) } @@ -390,6 +418,23 @@ func (response UpdateOcr200Response) VisitUpdateOcrResponse(w http.ResponseWrite return nil } +type UpdateOcrOneRequestObject struct { + AccountId AccountId `json:"AccountId"` + MemeId MemeId `json:"MemeId"` +} + +type UpdateOcrOneResponseObject interface { + VisitUpdateOcrOneResponse(w http.ResponseWriter) error +} + +type UpdateOcrOne200Response struct { +} + +func (response UpdateOcrOne200Response) VisitUpdateOcrOneResponse(w http.ResponseWriter) error { + w.WriteHeader(200) + return nil +} + // StrictServerInterface represents all server handlers. type StrictServerInterface interface { @@ -410,6 +455,9 @@ type StrictServerInterface interface { // (POST /api/v1/accounts/{AccountId}/update-ocr) UpdateOcr(ctx context.Context, request UpdateOcrRequestObject) (UpdateOcrResponseObject, error) + + // (POST /api/v1/accounts/{AccountId}/update-ocr/{MemeId}) + UpdateOcrOne(ctx context.Context, request UpdateOcrOneRequestObject) (UpdateOcrOneResponseObject, error) } type StrictHandlerFunc = strictecho.StrictEchoHandlerFunc @@ -582,3 +630,29 @@ func (sh *strictHandler) UpdateOcr(ctx echo.Context, accountId AccountId) error } return nil } + +// UpdateOcrOne operation middleware +func (sh *strictHandler) UpdateOcrOne(ctx echo.Context, accountId AccountId, memeId MemeId) error { + var request UpdateOcrOneRequestObject + + request.AccountId = accountId + request.MemeId = memeId + + handler := func(ctx echo.Context, request interface{}) (interface{}, error) { + return sh.ssi.UpdateOcrOne(ctx.Request().Context(), request.(UpdateOcrOneRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "UpdateOcrOne") + } + + response, err := handler(ctx, request) + + if err != nil { + return err + } else if validResponse, ok := response.(UpdateOcrOneResponseObject); ok { + return validResponse.VisitUpdateOcrOneResponse(ctx.Response()) + } else if response != nil { + return fmt.Errorf("unexpected response type: %T", response) + } + return nil +} diff --git a/storage-service/src/service/ApiHandlerService.go b/storage-service/src/service/ApiHandlerService.go index cc08a51..6f5c03d 100644 --- a/storage-service/src/service/ApiHandlerService.go +++ b/storage-service/src/service/ApiHandlerService.go @@ -22,56 +22,128 @@ type ApiHandler struct { validate *validator.Validate } -func (a *ApiHandler) iterateDocuments(ctx context.Context, accountId uuid.UUID, callback func(context.Context, *entity.ElasticMatchedContent) error) error { - items, err := a.metaStorage.Search(ctx, accountId, "", nil, addr(1000)) - for err == nil && len(items) > 0 { - for _, item := range items { - err = callback(ctx, item) - } - if len(items) > 0 { - items, err = a.metaStorage.Search(ctx, accountId, "", &items[len(items)-1].Metadata.Created, addr(1000)) - } +// CreateMeme implements server.StrictServerInterface. +func (a *ApiHandler) CreateMeme( + ctx context.Context, + request server.CreateMemeRequestObject, +) (server.CreateMemeResponseObject, error) { + image := request.Body + + idUuid, _ := uuid.NewRandom() + if len(*image.ImageBase64) == 0 { + return nil, errors.New("image is empty") } + hash := helper.CalcHash(request.Body.ImageBase64) + hashDuplicate, err := a.findHashDuplicates(ctx, hash) if err != nil { - return err + return nil, err } - return nil + + if hashDuplicate != nil { + return a.HandleDuplicate(ctx, server.DuplicateHash, hashDuplicate, request) + } + + reqImage := helper.ImageToEntity2(request.Body) + ocrResult, err := a.ocr.DoOcr(ctx, idUuid, reqImage) + if err != nil { + return nil, err + } + + contentDuplicate, err := a.findContentDuplicates(ctx, ocrResult) + if err != nil { + return nil, err + } + + if contentDuplicate != nil { + return a.HandleDuplicate(ctx, server.DuplicateImage, contentDuplicate, request) + } + + if strings.TrimSpace(ocrResult.OcrText) == "" { + return nil, errors.New("no text on image") + } + + err = a.imageStorage.Save(ctx, idUuid, ocrResult.Image, ocrResult.Thumbnail.Image) + if err != nil { + return nil, err + } + + elasticMetaData := OcrResultToElastic( + idUuid, + request.AccountId, + hash, + time.Now().UnixMicro(), + ocrResult, + ) + + err = a.validate.Struct(elasticMetaData) + if err != nil { + //TODO handle fail + return nil, err + } + + err = a.metaStorage.Save(ctx, elasticMetaData) + if err != nil { + //TODO handle fail + return nil, err + } + + response := server.CreateMeme200JSONResponse{} + helper.ElasticToCreateResponse(elasticMetaData, server.New, &response) + return response, nil +} + +// SearchMeme implements server.StrictServerInterface. +func (a *ApiHandler) SearchMeme(ctx context.Context, request server.SearchMemeRequestObject) (server.SearchMemeResponseObject, error) { + query := request.Params.MemeQuery + + log.Printf("SearchMeme: query=%s", query) + + matchedMetadata, err := a.metaStorage.Search( + ctx, + request.AccountId, + query, + request.Params.SearchAfterSortId, + request.Params.PageSize, + ) + + if err != nil { + return nil, err + } + + response := make(server.SearchMeme200JSONResponse, len(matchedMetadata)) + for index, metadataItem := range matchedMetadata { + + imageThumbUrl, err := a.imageStorage.GetUrlThumb(ctx, metadataItem.Metadata.S3Id) + if err != nil { + return nil, err + } + imageUrl, err := a.imageStorage.GetUrl(ctx, metadataItem.Metadata.S3Id) + if err != nil { + return nil, err + } + + dto := server.SearchMemeDto{} + helper.ElasticToSearchMemeDto(metadataItem, &dto) + dto.ImageUrl = &imageUrl + dto.Thumbnail = new(server.SearchMemeThumb) + dto.Thumbnail.ThumbUrl = &imageThumbUrl + dto.Thumbnail.ThumbHeight = &metadataItem.Metadata.ThumbSize.Height + dto.Thumbnail.ThumbWidth = &metadataItem.Metadata.ThumbSize.Width + response[index] = dto + } + + return response, nil } // CheckDuplicates implements server.StrictServerInterface. func (a *ApiHandler) CheckDuplicates(ctx context.Context, request server.CheckDuplicatesRequestObject) (server.CheckDuplicatesResponseObject, error) { - deleted := map[uuid.UUID]any{} - return server.CheckDuplicates200Response{}, a.iterateDocuments( ctx, request.AccountId, func(ctx2 context.Context, emc *entity.ElasticMatchedContent) error { - id := emc.Metadata.ImageId - embedding := emc.Metadata.EmbeddingV1 - - if _, ok := deleted[id]; ok { - return nil - } - - embeddingFoundImage, err := a.metaStorage.GetByEmbeddingV1(ctx2, embedding, 100) - if err != nil { - log.Printf("Failed to search image embedding duplicates : id=%s, err=%v", id, err) - return nil - } - - for i, item := range embeddingFoundImage { - if i == 0 { - continue - } - if _, ok := deleted[item.ImageId]; ok { - continue - } - a.metaStorage.Delete(ctx2, item.ImageId) - deleted[item.ImageId] = "" - } - return nil + return a.internalCheckDuplicate(ctx2, emc.Metadata) }) } @@ -79,36 +151,12 @@ func (a *ApiHandler) CheckDuplicates(ctx context.Context, request server.CheckDu // UpdateOcr implements server.StrictServerInterface. func (a *ApiHandler) UpdateOcr(ctx context.Context, request server.UpdateOcrRequestObject) (server.UpdateOcrResponseObject, error) { return server.UpdateOcr200Response{}, - a.iterateDocuments(ctx, request.AccountId, func(ctx2 context.Context, emc *entity.ElasticMatchedContent) error { - - id := emc.Metadata.ImageId - accountId := emc.Metadata.AccountId - hash := emc.Metadata.Hash - s3id := emc.Metadata.S3Id - created := emc.Metadata.Created - - log.Printf("UpdateOcr: checking image id=%s", id) - - img, err := a.imageStorage.GetImage(ctx2, s3id) - if err != nil { - log.Printf("Failed to read image from storage : id=%s, err=%v", id, err) - return nil - } - - ocrResult, err := a.ocr.DoOcr(ctx2, id, img) - if err != nil { - log.Printf("Failed to doOcr for image id=%s, err=%v", id, err) - return nil - } - - elasticObject := OcrResultToElastic(id, accountId, hash, created, ocrResult) - err = a.metaStorage.Save(ctx2, elasticObject) - if err != nil { - log.Printf("Failed to save new metadata for image id=%s, err=%v", id, err) - return nil - } - return nil - }) + a.iterateDocuments( + ctx, + request.AccountId, + func(ctx context.Context, emc *entity.ElasticMatchedContent) error { + return a.internalUpdateOcr(ctx, emc.Metadata) + }) } // GetMemeImageThumbUrl implements server.StrictServerInterface. @@ -153,6 +201,95 @@ func (a *ApiHandler) GetMemeImageUrl(ctx context.Context, request server.GetMeme return resp, nil } +// UpdateOcrOne implements server.StrictServerInterface. +func (a *ApiHandler) UpdateOcrOne(ctx context.Context, request server.UpdateOcrOneRequestObject) (server.UpdateOcrOneResponseObject, error) { + memeMetadata, err := a.metaStorage.GetById(ctx, request.MemeId) + if err != nil { + return nil, err + } + + if memeMetadata.AccountId != request.AccountId { + return nil, echo.ErrNotFound + } + + err = a.internalUpdateOcr(ctx, memeMetadata) + return server.UpdateOcrOne200Response{}, err +} + +func (a *ApiHandler) internalCheckDuplicate(ctx context.Context, emc *entity.ElasticImageMetaData) error { + id := emc.ImageId + embedding := emc.EmbeddingV1 + log.Printf("Check-duplicate: imageId: %s", id.String()) + + if embedding == nil { + log.Printf("Check-duplicate: NO EMBEDDING: imageId: %s", id.String()) + return nil + } + + embeddingFoundImage, err := a.metaStorage.GetByEmbeddingV1(ctx, embedding, 10) + if err != nil { + log.Printf("Check-duplicate: failed to search image embedding duplicates : id=%s, err=%v", id, err) + return nil + } + + for i, item := range embeddingFoundImage { + if i == 0 { + continue + } + + a.metaStorage.Delete(ctx, item.ImageId) + } + return nil +} + +func (a *ApiHandler) internalUpdateOcr(ctx context.Context, emc *entity.ElasticImageMetaData) error { + + id := emc.ImageId + accountId := emc.AccountId + hash := emc.Hash + s3id := emc.S3Id + created := emc.Created + + log.Printf("UpdateOcr: checking image id=%s", id) + + img, err := a.imageStorage.GetImage(ctx, s3id) + if err != nil { + log.Printf("Failed to read image from storage : id=%s, err=%v", id, err) + return nil + } + + ocrResult, err := a.ocr.DoOcr(ctx, id, img) + if err != nil { + log.Printf("Failed to doOcr for image id=%s, err=%v", id, err) + return nil + } + + elasticObject := OcrResultToElastic(id, accountId, hash, created, ocrResult) + err = a.metaStorage.Save(ctx, elasticObject) + if err != nil { + log.Printf("Failed to save new metadata for image id=%s, err=%v", id, err) + return nil + } + return nil +} + +func (a *ApiHandler) iterateDocuments(ctx context.Context, accountId uuid.UUID, callback func(context.Context, *entity.ElasticMatchedContent) error) error { + items, err := a.metaStorage.Search(ctx, accountId, "", nil, addr(1000)) + for err == nil && len(items) > 0 { + for _, item := range items { + err = callback(ctx, item) + } + if len(items) > 0 { + items, err = a.metaStorage.Search(ctx, accountId, "", &items[len(items)-1].Metadata.Created, addr(1000)) + } + } + + if err != nil { + return err + } + return nil +} + func (a *ApiHandler) findHashDuplicates( ctx context.Context, hash string, @@ -204,120 +341,6 @@ func (a *ApiHandler) HandleDuplicate( return response, nil } -// CreateMeme implements server.StrictServerInterface. -func (a *ApiHandler) CreateMeme( - ctx context.Context, - request server.CreateMemeRequestObject, -) (server.CreateMemeResponseObject, error) { - image := request.Body - - idUuid, _ := uuid.NewRandom() - if len(*image.ImageBase64) == 0 { - return nil, errors.New("image is empty") - } - - hash := helper.CalcHash(request.Body.ImageBase64) - hashDuplicate, err := a.findHashDuplicates(ctx, hash) - if err != nil { - return nil, err - } - - if hashDuplicate != nil { - return a.HandleDuplicate(ctx, server.DuplicateHash, hashDuplicate, request) - } - - reqImage := helper.ImageToEntity2(request.Body) - ocrResult, err := a.ocr.DoOcr(ctx, idUuid, reqImage) - if err != nil { - return nil, err - } - - contentDuplicate, err := a.findContentDuplicates(ctx, ocrResult) - if err != nil { - return nil, err - } - - if contentDuplicate != nil { - return a.HandleDuplicate(ctx, server.DuplicateImage, contentDuplicate, request) - } - - if strings.TrimSpace(ocrResult.OcrText) == "" { - return nil, errors.New("no text on image") - } - - err = a.imageStorage.Save(ctx, idUuid, ocrResult.Image, ocrResult.Thumbnail.Image) - if err != nil { - return nil, err - } - - elasticMetaData := OcrResultToElastic( - idUuid, - request.AccountId, - hash, - time.Now().UnixMicro(), - ocrResult, - ) - - err = a.validate.Struct(elasticMetaData) - if err != nil { - //TODO handle fail - return nil, err - } - - err = a.metaStorage.Save(ctx, elasticMetaData) - if err != nil { - //TODO handle fail - return nil, err - } - - response := server.CreateMeme200JSONResponse{} - helper.ElasticToCreateResponse(elasticMetaData, server.New, &response) - return response, nil -} - -// SearchMeme implements server.StrictServerInterface. -func (a *ApiHandler) SearchMeme(ctx context.Context, request server.SearchMemeRequestObject) (server.SearchMemeResponseObject, error) { - query := request.Params.MemeQuery - - log.Printf("SearchMeme: query=%s", query) - - matchedMetadata, err := a.metaStorage.Search( - ctx, - request.AccountId, - query, - request.Params.SearchAfterSortId, - request.Params.PageSize, - ) - - if err != nil { - return nil, err - } - - response := make(server.SearchMeme200JSONResponse, len(matchedMetadata)) - for index, metadataItem := range matchedMetadata { - - imageThumbUrl, err := a.imageStorage.GetUrlThumb(ctx, metadataItem.Metadata.S3Id) - if err != nil { - return nil, err - } - imageUrl, err := a.imageStorage.GetUrl(ctx, metadataItem.Metadata.S3Id) - if err != nil { - return nil, err - } - - dto := server.SearchMemeDto{} - helper.ElasticToSearchMemeDto(metadataItem, &dto) - dto.ImageUrl = &imageUrl - dto.Thumbnail = new(server.SearchMemeThumb) - dto.Thumbnail.ThumbUrl = &imageThumbUrl - dto.Thumbnail.ThumbHeight = &metadataItem.Metadata.ThumbSize.Height - dto.Thumbnail.ThumbWidth = &metadataItem.Metadata.ThumbSize.Width - response[index] = dto - } - - return response, nil -} - func OcrResultToElastic(idUuid uuid.UUID, accountId uuid.UUID, hash string, created int64, ocrResult *OcrProcessedResult) *entity.ElasticImageMetaData { return &entity.ElasticImageMetaData{ ImageId: idUuid, diff --git a/storage-service/src/service/ElasticMetadataStorageServiceImpl.go b/storage-service/src/service/ElasticMetadataStorageServiceImpl.go index 11ddee1..596fdd0 100644 --- a/storage-service/src/service/ElasticMetadataStorageServiceImpl.go +++ b/storage-service/src/service/ElasticMetadataStorageServiceImpl.go @@ -110,7 +110,7 @@ func (e *ElasticMetadataStorageServiceImpl) Delete(ctx context.Context, id uuid. Delete(INDEX_NAME, id.String()). Do(ctx) if err != nil { - return err + return fmt.Errorf("elastic failed to delete: %w", err) } log.Printf("Delete metadata document: elastic, id=%s response=%s", id, render.Render(response)) @@ -132,7 +132,8 @@ func (e *ElasticMetadataStorageServiceImpl) processKnn( Knn(query). TrackScores(true) - return search.Do(ctx) + knn, err := search.Do(ctx) + return errWrap(knn, "elastic failed to knn: %w", err) } // Search implements MetadataStorageService. @@ -170,7 +171,8 @@ func (e *ElasticMetadataStorageServiceImpl) runSearchQuery( search = search.Size(*pageSize) } - return search.Do(ctx) + resp, err := search.Do(ctx) + return errWrap(resp, "elastic failed to search: %w", err) } func (e *ElasticMetadataStorageServiceImpl) searchQueryInternal( @@ -221,7 +223,10 @@ func (e *ElasticMetadataStorageServiceImpl) Search( result, err := e.searchQueryInternal(ctx, accountId, queryString, sortIdAfter, pageSize) if err != nil { - return nil, err + return nil, + fmt.Errorf( + "search failed: accountId: %s queryString: %s sortIdAfter: %v error: %w", + accountId.String(), queryString, sortIdAfter, err) } resultsSize := len(result.Hits.Hits) @@ -230,11 +235,17 @@ func (e *ElasticMetadataStorageServiceImpl) Search( for index := range resultsSize { item, err := unmarhalSearchResultToMatchedContent(index, result) if err != nil { - return nil, err + return nil, + fmt.Errorf( + "search result unmarshall failed: accountId: %s queryString: %s sortIdAfter: %v error: %w", + accountId.String(), queryString, sortIdAfter, err) } err = e.validate.Struct(item) if err != nil { - return nil, err + return nil, + fmt.Errorf( + "search result vaildation failed: accountId: %s queryString: %s sortIdAfter: %v error: %w", + accountId.String(), queryString, sortIdAfter, err) } results[index] = item } @@ -243,6 +254,8 @@ func (e *ElasticMetadataStorageServiceImpl) Search( // GetById implements MetadataStorageService. func (e *ElasticMetadataStorageServiceImpl) GetById(ctx context.Context, id uuid.UUID) (*entity.ElasticImageMetaData, error) { + log.Printf("GetById: call id: %s", id.String()) + query := types.NewQuery() query.Ids = types.NewIdsQuery() query.Ids.Values = []string{id.String()} @@ -258,7 +271,10 @@ func (e *ElasticMetadataStorageServiceImpl) GetById(ctx context.Context, id uuid data, err := unmarhalSearchResultToElasticEntity(0, result) if err != nil { - return nil, err + return nil, + fmt.Errorf( + "GetById result unmarshall failed: id: %s error: %w", + id.String(), err) } return data, e.validate.Struct(data) @@ -269,6 +285,8 @@ func (e *ElasticMetadataStorageServiceImpl) GetByHash( ctx context.Context, hash string, ) (*entity.ElasticImageMetaData, error) { + log.Printf("GetByHash: call hash: %s", hash) + query := types.NewQuery() query.QueryString = types.NewQueryStringQuery() query.QueryString.Query = fmt.Sprintf("Hash: \"%s\"", hash) @@ -281,7 +299,10 @@ func (e *ElasticMetadataStorageServiceImpl) GetByHash( data, err := unmarhalSearchResultToElasticEntity(0, result) if err != nil { - return nil, err + return nil, + fmt.Errorf( + "GetByHash result unmarshall failed: id: %s error: %w", + hash, err) } if data == nil { @@ -297,6 +318,8 @@ func (e *ElasticMetadataStorageServiceImpl) GetByEmbeddingV1( img *entity.ElasticEmbeddingV1, count int, ) ([]*entity.ElasticImageMetaData, error) { + log.Printf("GetByEmbeddingV1: call") + query := e.embeddingV1KnnQuery(img, count) result, err := e.processKnn(ctx, *query) if err != nil { @@ -313,12 +336,14 @@ func (e *ElasticMetadataStorageServiceImpl) GetByEmbeddingV1( item, err := unmarhalSearchResultToElasticEntity(index, result) if err != nil { - return nil, err + return nil, fmt.Errorf("GetByEmbeddingV1 result unmarshall failed: error: %w", err) } + err = e.validate.Struct(item) if err != nil { - return nil, err + return nil, fmt.Errorf("GetByEmbeddingV1 result vaildation failed: error: %w", err) } + resultsEntity = append(resultsEntity, item) } return resultsEntity, nil @@ -372,8 +397,7 @@ func (e *ElasticMetadataStorageServiceImpl) Save(ctx context.Context, file *enti return err } - log.Printf("Save metadata document: elastic, id=%s response=%s", - file.ImageId, render.Render(response)) + log.Printf("Save metadata document: elastic, id=%s response=%s", file.ImageId, render.Render(response)) return err } @@ -418,6 +442,13 @@ func unmarhalSourceDocument(result json.RawMessage) (*entity.ElasticImageMetaDat func addr[T any](v T) *T { return &v } +func errWrap[T any](v *T, msg string, err error, args ...any) (*T, error) { + if err != nil { + return nil, fmt.Errorf(msg, err, args) + } + return v, err +} + func NewElasticMetadataStorage( config *conf.MetadataStorageConfig, validate *validator.Validate,