Skip to content

Introduced AWS S3 Storage #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sam-laister merged 2 commits into from
Jun 6, 2025
Merged

Introduced AWS S3 Storage #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6117cc5
Implemented openmvgo
sam-laister Jun 6, 2025
38b3c4c
aws s3 implementaation
sam-laister Jun 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 29 additions & 20 deletions app/Dockerfile.dev
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,40 +43,49 @@ RUN git clone --branch develop --depth=1 https://github.com/cdcseacave/openMVS.g
# ------------------------------
# Stage 3: Build Blender (minimal)
# ------------------------------
FROM ubuntu:22.04 AS blender_builder

RUN apt-get update && apt-get install -y --no-install-recommends \
wget xz-utils && \
rm -rf /var/lib/apt/lists/*

RUN wget -q https://download.blender.org/release/Blender4.4/blender-4.4.0-linux-x64.tar.xz --no-check-certificate && \
mkdir -p /opt/blender && \
tar -xf blender-4.4.0-linux-x64.tar.xz -C /opt/blender --strip-components=1 && \
rm blender-4.4.0-linux-x64.tar.xz
FROM linuxserver/blender:4.4.3 AS blender_builder

# -------------------------
# Stage 4: Dev Environment
# -------------------------
FROM ubuntu:latest AS final
FROM ubuntu:22.04 AS final

ENV TZ=UTC
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone

RUN sed -i'' 's/archive\.ubuntu\.com/us\.archive\.ubuntu\.com/' /etc/apt/sources.list

RUN apt-get update && apt-get install -y wget xz-utils libcgal-qt5-dev \
libceres-dev libboost-all-dev libopencv-dev build-essential && \
wget https://download.blender.org/release/Blender4.4/blender-4.4.0-linux-x64.tar.xz && \
wget https://go.dev/dl/go1.23.8.linux-amd64.tar.gz && \
# Create non-root user
RUN groupadd -r appuser && useradd -r -g appuser -s /bin/false appuser

ENV DEBIAN_FRONTEND=noninteractive
ARG DEBIAN_FRONTEND=noninteractive

# Install only runtime dependencies
# Seperate for cache issues
RUN apt-get -y update
RUN apt-get install -y --no-install-recommends \
libcgal-qt5-dev libceres2 libboost-system1.74.0 libboost-filesystem1.74.0 \
build-essential \
libboost-program-options1.74.0 libboost-serialization1.74.0 \
libopencv-core4.5d libopencv-imgproc4.5d libopencv-imgcodecs4.5d \
libjpeg8 libpng16-16 libtiff5 libglu1-mesa libglew2.2 \
libglfw3 libgomp1 ca-certificates curl wget libboost-all-dev libopencv-dev \
xorg && \
apt-get clean && rm -rf /var/lib/apt/lists/*

RUN wget https://go.dev/dl/go1.23.8.linux-amd64.tar.gz && \
tar -C /usr/local -xzf go1.23.8.linux-amd64.tar.gz && \
rm -rf go1.23.8.linux-amd64.tar.gz && \
apt-get clean && rm -rf /var/lib/apt/lists/*

# RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o server
# RUN chmod +x ./server

# Copy OpenMVG and OpenMVS builds from cv_builder
COPY --from=cv_builder /openMVG_build/Linux-x86_64-RELEASE /usr/local/bin
COPY --from=cv_builder /openMVS_build/bin /usr/local/bin

# Copy Blender (only essential parts)
COPY --from=blender_builder /opt/blender/blender /usr/local/bin/blender
COPY --from=blender_builder /opt/blender/4.4/ /opt/blender/4.4/
COPY --from=blender_builder /blender /opt/blender
ENV PATH="/opt/blender:$PATH"

ENV PATH="$PATH:/usr/local/go/bin:/root/go/bin"

Expand Down
47 changes: 36 additions & 11 deletions app/controller/object_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,56 @@ package controller

import (
"fmt"
"io"
"net/http"
"os"
"strconv"

"github.com/Soup666/modelmaker/services"
"github.com/gin-gonic/gin"
)

// AuthController is the controller for handling authentication requests
type ObjectController struct{}
type ObjectController struct {
storageService services.StorageService
}

func NewObjectController() *ObjectController {
return &ObjectController{}
func NewObjectController(storageService services.StorageService) *ObjectController {
return &ObjectController{
storageService: storageService,
}
}

func (c *ObjectController) GetObject(ctx *gin.Context) {
taskId := ctx.Param("taskID")

// Construct the full file path
filePath := fmt.Sprintf("objects/%s/%s", taskId, "mvs/final_model.glb")
// Convert taskId to uint
taskIdInt, err := strconv.ParseUint(taskId, 10, 32)
if err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": "Invalid task ID"})
return
}

// Check if the file exists
if _, err := os.Stat(filePath); os.IsNotExist(err) {
// Get file from object storage
file, err := c.storageService.GetFile(fmt.Sprintf("objects/%d/final.glb", taskIdInt))
if err != nil {
ctx.JSON(http.StatusNotFound, gin.H{"error": "Object not found"})
return
}

// Serve the file
ctx.File(filePath)
defer file.Close()

// Add headers
ctx.Header("Content-Type", "model/gltf-binary")
ctx.Header("Content-Disposition", "attachment; filename=final.glb")

// // Stream the file to the response
// ctx.Stream(func(w io.Writer) bool {
// _, err := io.Copy(w, file)
// return err == nil
// })
// Copy file contents to response writer
_, err = io.Copy(ctx.Writer, file)
if err != nil {
ctx.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to stream file"})
return
}
}
67 changes: 34 additions & 33 deletions app/controller/task_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package controller
import (
"errors"
"fmt"
"io"
"log"
"mime/multipart"
"net/http"
Expand All @@ -23,10 +24,16 @@ type TaskController struct {
TaskService services.TaskService
AppFileService services.AppFileService
VisionService services.VisionService
StorageService services.StorageService
}

func NewTaskController(taskService services.TaskService, appFileService services.AppFileService, visionService services.VisionService) *TaskController {
return &TaskController{TaskService: taskService, AppFileService: appFileService, VisionService: visionService}
func NewTaskController(taskService services.TaskService, appFileService services.AppFileService, visionService services.VisionService, storageService services.StorageService) *TaskController {
return &TaskController{
TaskService: taskService,
AppFileService: appFileService,
VisionService: visionService,
StorageService: storageService,
}
}

func (c *TaskController) GetUnarchivedTasks(ctx *gin.Context) {
Expand Down Expand Up @@ -136,7 +143,6 @@ func (c *TaskController) CreateTask(ctx *gin.Context) {
// @Failure 500 {object} map[string]string
// @Router /tasks/{taskID}/upload [post]
func (c *TaskController) UploadFileToTask(ctx *gin.Context) {

// Get the Task ID from the route
taskIdParam := ctx.Param("taskID")
taskId, err := strconv.Atoi(taskIdParam)
Expand Down Expand Up @@ -164,13 +170,6 @@ func (c *TaskController) UploadFileToTask(ctx *gin.Context) {
return
}

// Define the upload folder
folderPath := fmt.Sprintf("uploads/%d", taskId)
if err := os.MkdirAll(folderPath, os.ModePerm); err != nil {
ctx.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create upload directory"})
return
}

var uploadedImages []model.AppFile
var wg sync.WaitGroup
var mu sync.Mutex
Expand All @@ -192,21 +191,18 @@ func (c *TaskController) UploadFileToTask(ctx *gin.Context) {
return
}

// Generate a unique filename
filename := fmt.Sprintf("%d-%d%s", taskId, index, fileExt)
savePath := filepath.Join(folderPath, filename)

// Save the file
if err := ctx.SaveUploadedFile(file, savePath); err != nil {
// Upload to object storage
url, err := c.StorageService.UploadFile(file, uint(taskId), "upload")
if err != nil {
ctx.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to save file %s", file.Filename)})
hasError = true
return
}

// Save metadata to DB
image := model.AppFile{
Filename: filename,
Url: fmt.Sprintf("/uploads/%d/%s", taskId, filename),
Filename: file.Filename,
Url: url,
TaskId: uint(taskId),
FileType: "upload",
}
Expand All @@ -232,37 +228,42 @@ func (c *TaskController) UploadFileToTask(ctx *gin.Context) {
tx.Commit()

go func() {
// Generate caption
result, err := c.VisionService.AnalyseImage(fmt.Sprintf("./uploads/%d/%s", taskId, uploadedImages[0].Filename), "")
// Get the first image URL from object storage
file, err := c.StorageService.GetFile(fmt.Sprintf("uploads/%d/%s", taskId, uploadedImages[0].Filename))
if err != nil {
log.Printf("Unable to get image for analysis: %v", err)
return
}
defer file.Close()

// Create a temporary file
tempFile, err := os.CreateTemp("", "analysis-*.jpg")
if err != nil {
log.Printf("Unable to analyze the image: %v", err)
log.Printf("Unable to create temp file: %v", err)
return
}
defer os.Remove(tempFile.Name())
defer tempFile.Close()

if err := c.TaskService.UpdateMeta(&task, "ai-description", result); err != nil {
log.Printf("Failed to update task metadata: %v", err)
// Copy the file content
if _, err := io.Copy(tempFile, file); err != nil {
log.Printf("Unable to copy file content: %v", err)
return
}
}()

go func() {
// Generate caption
result, err := c.VisionService.AnalyseImage(fmt.Sprintf("./uploads/%d/%s", taskId, uploadedImages[0].Filename), "categorize the model in this image, use one word only")

result, err := c.VisionService.AnalyseImage(tempFile.Name(), "")
if err != nil {
log.Printf("Unable to analyze the image: %v", err)
return
}

if err := c.TaskService.UpdateMeta(&task, "ai-title", result); err != nil {
if err := c.TaskService.UpdateMeta(&task, "ai-description", result); err != nil {
log.Printf("Failed to update task metadata: %v", err)
}
}()

ctx.JSON(http.StatusOK, gin.H{
"message": "Files uploaded successfully",
"images": uploadedImages,
})
ctx.JSON(http.StatusOK, gin.H{"message": "Files uploaded successfully", "images": uploadedImages})
}

// StartProcess handles the process of starting the photogrammetry process
Expand Down Expand Up @@ -375,7 +376,7 @@ func (c *TaskController) SendMessage(ctx *gin.Context) {
return
}

imagePath := fmt.Sprintf("./uploads/%d/%s", taskId, task.Images[0].Filename)
imagePath := fmt.Sprintf("uploads/%d/%s", taskId, task.Images[0].Filename)

if _, err := os.Stat(imagePath); os.IsNotExist(err) {
log.Printf("Image file does not exist: %v\n", err)
Expand Down
3 changes: 2 additions & 1 deletion app/controller/task_controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,9 @@ func TestTaskController(t *testing.T) {
mockTaskService := new(mocks.MockTaskService)
mockAppFileService := new(mocks.MockAppFileService)
mockVisionService := new(mocks.MockVisionService)
mockStorageService := new(mocks.MockStorageService)

taskController := controller.NewTaskController(mockTaskService, mockAppFileService, mockVisionService)
taskController := controller.NewTaskController(mockTaskService, mockAppFileService, mockVisionService, mockStorageService)

t.Run("GetUnarchivedTasks", func(t *testing.T) {
recorder, c := utils.SetupRecorder()
Expand Down
Loading
Loading