Merge pull request #7 from JackalLabs/refinements

Adding custom path support and better routing

japicore/fidRouteHandling.go

@@ -0,0 +1,120 @@
+package japicore
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/JackalLabs/jackalapi/jutils"
+	"github.com/JackalLabs/jackalgo/handlers/file_io_handler"
+	"github.com/uptrace/bunrouter"
+	"net/http"
+	"strings"
+)
+
+func IpfsHandler(fileIo *file_io_handler.FileIoHandler, queue *FileIoQueue) bunrouter.HandlerFunc {
+	return func(w http.ResponseWriter, req bunrouter.Request) error {
+		var allBytes []byte
+
+		operatingRoot := jutils.LoadEnvVarOrFallback("JAPI_IPFS_ROOT", "s/JAPI/IPFS")
+		gateway := jutils.LoadEnvVarOrFallback("JAPI_IPFS_GATEWAY", "https://ipfs.io/ipfs/")
+		toClone := false
+		cloneHeader := req.Header.Get("J-Clone-Ipfs")
+		if strings.ToLower(cloneHeader) == "true" {
+			toClone = true
+		}
+
+		id := req.Param("id")
+		if len(id) == 0 {
+			warning := "Failed to get IPFS CID"
+			return jutils.ProcessCustomHttpError("processUpload", warning, 500, w)
+		}
+
+		cid := strings.ReplaceAll(id, "/", "_")
+
+		handler, err := fileIo.DownloadFile(fmt.Sprintf("%s/%s", operatingRoot, cid))
+		if err != nil {
+			if !toClone {
+				warning := "IPFS CID Not Found"
+				return jutils.ProcessCustomHttpError("DownloadFile", warning, 404, w)
+			}
+
+			byteBuffer, err := httpGetFileRequest(w, gateway, cid)
+			if err != nil {
+				jutils.ProcessHttpError("httpGetFileRequest", err, 404, w)
+				return err
+			}
+
+			byteReader := bytes.NewReader(byteBuffer.Bytes())
+			workingBytes := jutils.CloneBytes(byteReader)
+			allBytes = jutils.CloneBytes(byteReader)
+
+			fid := processUpload(w, fileIo, workingBytes, cid, operatingRoot, queue)
+			if len(fid) == 0 {
+				warning := "Failed to get FID post-upload"
+				return jutils.ProcessCustomHttpError("IpfsHandler", warning, 500, w)
+			}
+		} else {
+			allBytes = handler.GetFile().Buffer().Bytes()
+		}
+		_, err = w.Write(allBytes)
+		if err != nil {
+			jutils.ProcessError("WWriteError for IpfsHandler", err)
+			return err
+		}
+		return nil
+	}
+}
+
+func DownloadByFidHandler(fileIo *file_io_handler.FileIoHandler) bunrouter.HandlerFunc {
+	return func(w http.ResponseWriter, req bunrouter.Request) error {
+		id := req.Param("id")
+		if len(id) == 0 {
+			warning := "Failed to get FileName"
+			return jutils.ProcessCustomHttpError("DownloadByFidHandler", warning, 404, w)
+		}
+		fid := strings.ReplaceAll(id, "/", "_")
+
+		handler, err := fileIo.DownloadFileFromFid(fid)
+		if err != nil {
+			return err
+		}
+
+		fileBytes := handler.GetFile().Buffer().Bytes()
+		_, err = w.Write(fileBytes)
+		if err != nil {
+			jutils.ProcessError("WWriteError for DownloadByFidHandler", err)
+		}
+		return nil
+	}
+}
+
+func DeleteByFidHandler(fileIo *file_io_handler.FileIoHandler, queue *FileIoQueue) bunrouter.HandlerFunc {
+	return func(w http.ResponseWriter, req bunrouter.Request) error {
+		id := req.Param("id")
+		if len(id) == 0 {
+			warning := "Failed to get FileName"
+			return jutils.ProcessCustomHttpError("processUpload", warning, 400, w)
+		}
+
+		fid := strings.ReplaceAll(id, "/", "_")
+		fmt.Println(fid)
+
+		// TODO - update after deletion by fid is added to jackalgo
+
+		//folder, err := fileIo.DownloadFolder(queue.GetRoot("bulk"))
+		//if err != nil {
+		//	jutils.ProcessHttpError("DeleteFile", err, 404, w)
+		//	return err
+		//}
+		//
+		//err = fileIo.DeleteTargets([]string{fid}, folder)
+		//if err != nil {
+		//	jutils.ProcessHttpError("DeleteFile", err, 500, w)
+		//	return err
+		//}
+
+		//message := createJsonResponse("Deletion complete")
+		message := createJsonResponse("Deletion Not Implemented")
+		condensedWriteJSON(w, message)
+		return nil
+	}
+}

japicore/miscRouteHandling.go

@@ -0,0 +1,41 @@
+package japicore
+
+import (
+	"fmt"
+	"github.com/JackalLabs/jackalapi/jutils"
+	"github.com/uptrace/bunrouter"
+	"net/http"
+)
+
+var (
+	Version = "v0.0.0"
+	Module  = "Jackal API Core"
+)
+
+func Handler() bunrouter.HandlerFunc {
+	return func(w http.ResponseWriter, req bunrouter.Request) error {
+		return nil
+	}
+}
+
+func RouteNotFoundHandler() bunrouter.HandlerFunc {
+	return func(w http.ResponseWriter, req bunrouter.Request) error {
+		warning := fmt.Sprintf("%s is not an availble route", req.URL.Path)
+		return jutils.ProcessCustomHttpError("MethodNotAllowedHandler", warning, 404, w)
+	}
+}
+
+func MethodNotAllowedHandler() bunrouter.HandlerFunc {
+	return func(w http.ResponseWriter, req bunrouter.Request) error {
+		warning := fmt.Sprintf("%s method not availble for \"%s\"", req.URL.Path, req.Method)
+		return jutils.ProcessCustomHttpError("MethodNotAllowedHandler", warning, 405, w)
+	}
+}
+
+func VersionHandler() bunrouter.HandlerFunc {
+	return func(w http.ResponseWriter, req bunrouter.Request) error {
+		message := createJsonResponse("")
+		condensedWriteJSON(w, message)
+		return nil
+	}
+}

japicore/pathRouteHandling.go

@@ -0,0 +1,208 @@
+package japicore
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/JackalLabs/jackalapi/jutils"
+	"github.com/JackalLabs/jackalgo/handlers/file_io_handler"
+	"github.com/uptrace/bunrouter"
+)
+
+func downloadByPathCore(fileIo *file_io_handler.FileIoHandler, operatingRoot string) bunrouter.HandlerFunc {
+	return func(w http.ResponseWriter, req bunrouter.Request) error {
+		location := req.Param("location")
+		if len(location) == 0 {
+			warning := "Failed to get Location"
+			return jutils.ProcessCustomHttpError("DownloadByPathHandler", warning, 404, w)
+		}
+
+		uniquePath := readUniquePath(req)
+		if len(uniquePath) > 0 {
+			operatingRoot += "/" + uniquePath
+		}
+		operatingRoot += "/" + location
+
+		handler, err := fileIo.DownloadFile(operatingRoot)
+		if err != nil {
+			return err
+		}
+
+		fileBytes := handler.GetFile().Buffer().Bytes()
+		_, err = w.Write(fileBytes)
+		if err != nil {
+			jutils.ProcessError("WWriteError for DownloadByPathHandler", err)
+		}
+		return nil
+	}
+}
+
+func deleteByPathCore(fileIo *file_io_handler.FileIoHandler, operatingRoot string) bunrouter.HandlerFunc {
+	return func(w http.ResponseWriter, req bunrouter.Request) error {
+		filename := req.Param("filename")
+		if len(filename) == 0 {
+			warning := "Failed to get FileName"
+			return jutils.ProcessCustomHttpError("processUpload", warning, 400, w)
+		}
+
+		location := req.Param("location")
+		if len(location) == 0 {
+			warning := "Failed to get Location"
+			return jutils.ProcessCustomHttpError("DownloadByPathHandler", warning, 404, w)
+		}
+
+		cleanFilename := strings.ReplaceAll(filename, "/", "_")
+		fmt.Println(cleanFilename)
+
+		uniquePath := readUniquePath(req)
+		if len(uniquePath) > 0 {
+			operatingRoot += "/" + uniquePath
+		}
+		operatingRoot += "/" + location
+
+		folder, err := fileIo.DownloadFolder(operatingRoot)
+		if err != nil {
+			jutils.ProcessHttpError("DeleteFile", err, 404, w)
+			return err
+		}
+
+		err = fileIo.DeleteTargets([]string{cleanFilename}, folder)
+		if err != nil {
+			jutils.ProcessHttpError("DeleteFile", err, 500, w)
+			return err
+		}
+
+		message := createJsonResponse("Deletion complete")
+		condensedWriteJSON(w, message)
+		return nil
+	}
+}
+
+func ImportHandler(fileIo *file_io_handler.FileIoHandler, queue *ScrapeQueue) bunrouter.HandlerFunc {
+	return func(w http.ResponseWriter, req bunrouter.Request) error {
+		operatingRoot := jutils.LoadEnvVarOrFallback("JAPI_BULK_ROOT", "s/JAPI/Bulk")
+
+		uniquePath := readUniquePath(req)
+		if len(uniquePath) > 0 {
+			operatingRoot += "/" + uniquePath
+		}
+
+		var data fileScrape
+		source := req.Header.Get("J-Source-Path")
+		operatingRoot += "/" + source
+
+		err := json.NewDecoder(req.Body).Decode(&data)
+		if err != nil {
+			jutils.ProcessHttpError("JSONDecoder", err, 500, w)
+			return err
+		}
+
+		var wg sync.WaitGroup
+
+		for _, target := range data.Targets {
+			wg.Add(1)
+			queue.Push(fileIo, w, &wg, operatingRoot, target, source)
+		}
+
+		wg.Wait()
+
+		message := createJsonResponse("Import complete")
+		condensedWriteJSON(w, message)
+		return nil
+	}
+}
+
+func DownloadFromBulkByPathHandler(fileIo *file_io_handler.FileIoHandler) bunrouter.HandlerFunc {
+	operatingRoot := jutils.LoadEnvVarOrFallback("JAPI_BULK_ROOT", "s/JAPI/Bulk")
+	return downloadByPathCore(fileIo, operatingRoot)
+}
+
+func DownloadByPathHandler(fileIo *file_io_handler.FileIoHandler) bunrouter.HandlerFunc {
+	operatingRoot := jutils.LoadEnvVarOrFallback("JAPI_OP_ROOT", "s/JAPI")
+	return downloadByPathCore(fileIo, operatingRoot)
+}
+
+func UploadByPathHandler(fileIo *file_io_handler.FileIoHandler, queue *FileIoQueue) bunrouter.HandlerFunc {
+	return func(w http.ResponseWriter, req bunrouter.Request) error {
+		operatingRoot := jutils.LoadEnvVarOrFallback("JAPI_OP_ROOT", "s/JAPI")
+		var byteBuffer bytes.Buffer
+		var wg sync.WaitGroup
+		wg.Add(1)
+		WorkingFileSize := 32 << 30
+
+		envSize := jutils.LoadEnvVarOrFallback("JAPI_MAX_FILE", "")
+		if len(envSize) > 0 {
+			envParse, err := strconv.Atoi(envSize)
+			if err != nil {
+				return err
+			}
+			WorkingFileSize = envParse
+		}
+		MaxFileSize := int64(WorkingFileSize)
+
+		// ParseMultipartForm parses a request body as multipart/form-data
+		err := req.ParseMultipartForm(MaxFileSize) // MAX file size lives here
+		if err != nil {
+			jutils.ProcessHttpError("ParseMultipartForm", err, 400, w)
+			return err
+		}
+
+		// Retrieve the file from form data
+		file, head, err := req.FormFile("file")
+		if err != nil {
+			jutils.ProcessHttpError("FormFileFile", err, 400, w)
+			return err
+		}
+
+		uniquePath := readUniquePath(req)
+		if len(uniquePath) > 0 {
+			operatingRoot += "/" + uniquePath
+		}
+
+		subFolder := req.FormValue("subfolder")
+		if len(subFolder) > 0 {
+			operatingRoot += "/" + subFolder
+		}
+
+		_, err = io.Copy(&byteBuffer, file)
+		if err != nil {
+			jutils.ProcessHttpError("MakeByteBuffer", err, 500, w)
+			return err
+		}
+
+		fid := processUpload(w, fileIo, byteBuffer.Bytes(), head.Filename, operatingRoot, queue)
+		if len(fid) == 0 {
+			warning := "Failed to get FID"
+			return jutils.ProcessCustomHttpError("processUpload", warning, 500, w)
+		}
+
+		successfulUpload := UploadResponse{
+			FID: fid,
+		}
+		err = json.NewEncoder(w).Encode(successfulUpload)
+		if err != nil {
+			jutils.ProcessHttpError("JSONSuccessEncode", err, 500, w)
+			return err
+		}
+
+		message := createJsonResponse("Upload complete")
+		condensedWriteJSON(w, message)
+		return nil
+	}
+}
+
+func DeleteFromBulkByPathHandler(fileIo *file_io_handler.FileIoHandler) bunrouter.HandlerFunc {
+	operatingRoot := jutils.LoadEnvVarOrFallback("JAPI_BULK_ROOT", "s/JAPI/Bulk")
+	return deleteByPathCore(fileIo, operatingRoot)
+}
+
+func DeleteByPathHandler(fileIo *file_io_handler.FileIoHandler) bunrouter.HandlerFunc {
+	operatingRoot := jutils.LoadEnvVarOrFallback("JAPI_OP_ROOT", "s/JAPI")
+	return deleteByPathCore(fileIo, operatingRoot)
+}

japicore/requests.go

@@ -2,7 +2,6 @@ package japicore
 
 import (
 	"bytes"
-	"fmt"
 	"io"
 	"net/http"
 	"net/url"
@@ -22,7 +21,7 @@ func httpGetFileRequest(w http.ResponseWriter, host string, path string) (bytes.
 	}
 
 	url = url.JoinPath(path)
-	fmt.Println(url.String())
+	// fmt.Println(url.String())
 
 	innerReq, err := http.NewRequest("GET", url.String(), nil)
 	if err != nil {